Merge pull request #32 from PureStackStudio/unify-code-languages-for-highlighters

Unified all three languages, 95 extensions, and shared aliases for both highlighters.

Author: koculu

packages/ts-ssg/src/mdx/codeLanguages.test.ts

@@ -0,0 +1,57 @@
+import hljs from 'highlight.js'
+import { describe, expect, it, vi } from 'vitest'
+import { LANGUAGE_BY_EXTENSION } from './codeLanguages'
+import { createMdxHighlighter, DEFAULT_MDX_CODE_LANGS } from './highlight'
+import { createHljsHighlighter } from './highlightjs'
+
+describe('MDX code languages', () => {
+  it('renders every inferred language with the default Shiki grammars', async () => {
+    const highlighter = await createMdxHighlighter()
+    const languages = new Set([
+      ...DEFAULT_MDX_CODE_LANGS,
+      ...Object.values(LANGUAGE_BY_EXTENSION),
+    ])
+    for (const language of languages) {
+      expect(await highlighter.codeToHtml('example', language)).toContain(
+        '<pre',
+      )
+    }
+  })
+
+  it('uses a registered highlight.js grammar for every default language', () => {
+    const engine = hljs.newInstance()
+    for (const language of hljs.listLanguages()) {
+      const grammar = hljs.getLanguage(language)
+      if (grammar) engine.registerLanguage(language, () => grammar)
+    }
+    const highlight = vi.spyOn(engine, 'highlight')
+    const highlighter = createHljsHighlighter(engine)
+    for (const language of DEFAULT_MDX_CODE_LANGS) {
+      highlight.mockClear()
+      highlighter.codeToHtml('example', language)
+      expect(highlight, language).toHaveBeenCalledOnce()
+    }
+  })
+
+  it.each([
+    [' PY3 ', 'python', 'print(42)'],
+    ['cs', 'csharp', 'public class Example {}'],
+    ['c++', 'cpp', 'int main() {}'],
+    ['pwsh', 'powershell', 'Write-Host "Hello"'],
+    ['yml', 'yaml', 'answer: 42'],
+    ['shellscript', 'bash', 'echo "Hello"'],
+    ['objectivec', 'objective-c', '@interface Example'],
+    ['dos', 'bat', 'echo Hello'],
+    ['gql', 'graphql', 'query { user { id } }'],
+  ])('resolves %s with both highlighters', async (alias, language, code) => {
+    // Loading by the alias must also work when defaults are overridden.
+    const shiki = await createMdxHighlighter(undefined, [alias])
+    expect(await shiki.codeToHtml(code, alias)).toBe(
+      await shiki.codeToHtml(code, language),
+    )
+    const highlighter = createHljsHighlighter()
+    const html = highlighter.codeToHtml(code, alias)
+    expect(html).toBe(highlighter.codeToHtml(code, language))
+    expect(html).toContain('<span')
+  })
+})

packages/ts-ssg/src/mdx/codeLanguages.ts

@@ -0,0 +1,103 @@
+interface CodeLanguage {
+  aliases?: string[]
+  extensions?: string[]
+}
+
+// Use Shiki names (including its aliases) as the common language identifiers.
+const codeLanguages: Record<string, CodeLanguage> = {
+  bash: {
+    aliases: ['sh', 'shell', 'shellscript', 'zsh'],
+    extensions: ['.sh', '.bash', '.zsh'],
+  },
+  bat: { aliases: ['batch', 'cmd', 'dos'], extensions: ['.bat', '.cmd'] },
+  c: { extensions: ['.c', '.h'] },
+  cmake: { extensions: ['.cmake'] },
+  cpp: {
+    aliases: ['c++', 'cc', 'cxx', 'h++', 'hpp', 'hxx'],
+    extensions: ['.cc', '.cpp', '.cxx', '.hh', '.hpp', '.hxx'],
+  },
+  csharp: { aliases: ['c#', 'cs'], extensions: ['.cs', '.csx'] },
+  css: { extensions: ['.css'] },
+  dart: { extensions: ['.dart'] },
+  diff: { aliases: ['patch'], extensions: ['.diff', '.patch'] },
+  dockerfile: { aliases: ['docker'], extensions: ['.dockerfile'] },
+  elixir: { aliases: ['ex', 'exs'], extensions: ['.ex', '.exs'] },
+  erlang: { aliases: ['erl'], extensions: ['.erl', '.hrl'] },
+  fsharp: { aliases: ['f#', 'fs'], extensions: ['.fs', '.fsi', '.fsx'] },
+  go: { aliases: ['golang'], extensions: ['.go'] },
+  graphql: { aliases: ['gql'], extensions: ['.graphql', '.gql'] },
+  groovy: { extensions: ['.groovy', '.gradle'] },
+  html: { aliases: ['htm'], extensions: ['.html', '.htm'] },
+  http: { aliases: ['https'], extensions: ['.http', '.rest'] },
+  ini: { aliases: ['properties'], extensions: ['.ini', '.cfg', '.properties'] },
+  java: { extensions: ['.java'] },
+  javascript: {
+    aliases: ['js', 'cjs', 'mjs'],
+    extensions: ['.js', '.cjs', '.mjs'],
+  },
+  json: { extensions: ['.json'] },
+  jsonc: { extensions: ['.jsonc'] },
+  jsx: { extensions: ['.jsx'] },
+  kotlin: { aliases: ['kt', 'kts'], extensions: ['.kt', '.kts'] },
+  less: { extensions: ['.less'] },
+  lua: { extensions: ['.lua'] },
+  makefile: { aliases: ['make', 'mk'], extensions: ['.mk', '.mak'] },
+  markdown: {
+    aliases: ['md', 'mkd', 'mdown'],
+    extensions: ['.md', '.markdown', '.mkd', '.mdown'],
+  },
+  mdx: { aliases: ['rmdx'], extensions: ['.mdx', '.rmdx'] },
+  'objective-c': { aliases: ['objc', 'objectivec'], extensions: ['.m'] },
+  perl: { aliases: ['pl', 'pm'], extensions: ['.pl', '.pm'] },
+  php: { extensions: ['.php', '.phtml'] },
+  powershell: {
+    aliases: ['ps', 'ps1', 'pwsh'],
+    extensions: ['.ps1', '.psm1', '.psd1'],
+  },
+  python: {
+    aliases: ['py', 'py3', 'python3'],
+    extensions: ['.py', '.pyw', '.pyi'],
+  },
+  r: { extensions: ['.r'] },
+  ruby: {
+    aliases: ['rb', 'gemspec'],
+    extensions: ['.rb', '.rake', '.gemspec'],
+  },
+  rust: { aliases: ['rs'], extensions: ['.rs'] },
+  scala: { extensions: ['.scala', '.sc'] },
+  scss: { extensions: ['.scss'] },
+  sql: { extensions: ['.sql'] },
+  swift: { extensions: ['.swift'] },
+  text: { aliases: ['txt', 'plaintext'], extensions: ['.txt', '.text'] },
+  toml: { extensions: ['.toml'] },
+  'ts-tags': { aliases: ['lit'] },
+  tsx: { extensions: ['.tsx'] },
+  typescript: {
+    aliases: ['ts', 'cts', 'mts'],
+    extensions: ['.ts', '.cts', '.mts'],
+  },
+  xml: {
+    aliases: ['svg', 'xhtml'],
+    extensions: ['.xml', '.svg', '.xhtml', '.xsd', '.xsl', '.xslt'],
+  },
+  yaml: { aliases: ['yml'], extensions: ['.yaml', '.yml'] },
+}
+
+export const DEFAULT_CODE_LANGUAGES = Object.keys(codeLanguages)
+
+const languageAliases = new Map(
+  Object.entries(codeLanguages).flatMap(([language, { aliases = [] }]) =>
+    aliases.map((alias) => [alias, language] as const),
+  ),
+)
+
+export const LANGUAGE_BY_EXTENSION: Record<string, string> = Object.fromEntries(
+  Object.entries(codeLanguages).flatMap(([language, { extensions = [] }]) =>
+    extensions.map((extension) => [extension, language]),
+  ),
+)
+
+export function normalizeCodeLanguage(value: string): string {
+  const normalized = value.trim().toLowerCase()
+  return languageAliases.get(normalized) ?? normalized
+}

packages/ts-ssg/src/mdx/highlight.ts

@@ -1,4 +1,5 @@
 import { createHighlighter, type Highlighter } from 'shiki'
+import { DEFAULT_CODE_LANGUAGES, normalizeCodeLanguage } from './codeLanguages'
 
 export interface MdxCodeHighlighter {
   codeToHtml: (code: string, lang?: string) => string | Promise<string>
@@ -20,38 +21,8 @@ export const DEFAULT_MDX_CODE_THEMES: MdxCodeThemes = {
   dark: 'github-dark',
 }
 
-export const DEFAULT_MDX_CODE_LANGS: MdxCodeLangs = [
-  'bash',
-  'sh',
-  'shell',
-  'javascript',
-  'js',
-  'typescript',
-  'ts',
-  'ts-tags',
-  'jsx',
-  'tsx',
-  'json',
-  'html',
-  'css',
-  'markdown',
-  'mdx',
-  'yaml',
-  'yml',
-  'toml',
-  'diff',
-  'csharp',
-  'cpp',
-  'c',
-  'java',
-  'python',
-  'ruby',
-  'go',
-  'rust',
-  'xml',
-]
+export const DEFAULT_MDX_CODE_LANGS: MdxCodeLangs = DEFAULT_CODE_LANGUAGES
 
-const TYPESCRIPT_LANGS = new Set(['typescript', 'ts', 'cts', 'mts'])
 const TYPESCRIPT_WITH_TAGS_LANG = 'ts-tags'
 
 export async function createMdxHighlighter(
@@ -78,28 +49,23 @@ export async function createMdxHighlighter(
 }
 
 function resolveLoadedLanguages(langs: string[]) {
-  const resolved = dedupe(langs.map(normalizeLanguageName))
+  const resolved = dedupe(langs.map(normalizeCodeLanguage))
   if (
-    resolved.some((lang) => TYPESCRIPT_LANGS.has(lang)) &&
-    !resolved.includes(TYPESCRIPT_WITH_TAGS_LANG) &&
-    !resolved.includes('lit')
+    resolved.includes('typescript') &&
+    !resolved.includes(TYPESCRIPT_WITH_TAGS_LANG)
   ) {
     resolved.push(TYPESCRIPT_WITH_TAGS_LANG)
   }
   return resolved
 }
 
 function resolveHighlightLanguage(lang: string | undefined) {
-  const normalized = normalizeLanguageName(lang ?? 'text')
-  return TYPESCRIPT_LANGS.has(normalized)
+  const normalized = normalizeCodeLanguage(lang ?? 'text')
+  return normalized === 'typescript'
     ? TYPESCRIPT_WITH_TAGS_LANG
     : normalized || 'text'
 }
 
-function normalizeLanguageName(value: string) {
-  return value.trim().toLowerCase()
-}
-
 function dedupe(values: string[]) {
   return Array.from(new Set(values.filter((value) => value.length > 0)))
 }

packages/ts-ssg/src/mdx/highlightjs.ts

@@ -1,25 +1,26 @@
 import hljs, { type HLJSApi } from 'highlight.js'
 
+import { normalizeCodeLanguage } from './codeLanguages'
 import type { MdxCodeHighlighter } from './highlight'
 
-const languageAliases: Record<string, string> = {
-  js: 'javascript',
-  ts: 'typescript',
-  sh: 'bash',
-  shell: 'bash',
-  yml: 'yaml',
-  md: 'markdown',
-}
+// Use highlight.js names, or the closest grammar for Shiki-only languages.
+const hljsLanguages = new Map([
+  ['jsonc', 'json'],
+  ['mdx', 'markdown'],
+  ['objective-c', 'objectivec'],
+  ['ts-tags', 'typescript'],
+])
 
 export function createHljsHighlighter(
   highlighter: HLJSApi = hljs,
 ): MdxCodeHighlighter {
   return {
     codeToHtml: (code: string, lang?: string) => {
-      const normalizedLang = normalizeLanguage(lang)
-      if (normalizedLang && highlighter.getLanguage(normalizedLang)) {
+      const normalizedLang = normalizeCodeLanguage(lang ?? '')
+      const highlightLang = hljsLanguages.get(normalizedLang) ?? normalizedLang
+      if (highlightLang && highlighter.getLanguage(highlightLang)) {
         const highlighted = highlighter.highlight(code, {
-          language: normalizedLang,
+          language: highlightLang,
           ignoreIllegals: true,
         })
         return `<pre class="hljs"><code class="hljs language-${highlighted.language}">${highlighted.value}</code></pre>`
@@ -32,13 +33,6 @@ export function createHljsHighlighter(
   }
 }
 
-function normalizeLanguage(lang: string | undefined): string | undefined {
-  if (!lang) return undefined
-  const normalized = lang.trim().toLowerCase()
-  if (!normalized) return undefined
-  return languageAliases[normalized] ?? normalized
-}
-
 function escapeHtml(value: string): string {
   return value
     .replaceAll('&', '&amp;')

packages/ts-ssg/src/mdx/imports.test.ts

@@ -12,6 +12,10 @@ describe('expandImports', () => {
     const files: Record<string, string> = {
       'guides/demo.ts': 'export const answer = 42\n',
       'guides/styles.css': '.demo { color: red; }\n',
+      'guides/example.PY': 'print(42)\n',
+      'guides/example.cs': 'public class Example {}\n',
+      'guides/example.ps1': 'Write-Host "Hello"\n',
+      'guides/example.sql': 'SELECT 42;\n',
       'shared/fence.md': 'Use a fence:\r\n\r\n```ts\r\nconst x = 1\r\n```\r\n',
       'guides/data.unknown': 'plain',
       'shared/_note.mdx':
@@ -58,6 +62,17 @@ describe('expandImports', () => {
     )
   })
 
+  it.each([
+    ['example.PY', 'python', 'print(42)'],
+    ['example.cs', 'csharp', 'public class Example {}'],
+    ['example.ps1', 'powershell', 'Write-Host "Hello"'],
+    ['example.sql', 'sql', 'SELECT 42;'],
+  ])('infers the language for %s', async (file, language, code) => {
+    expect(await expand(`<import-codeblock src="./${file}"/>`)).toBe(
+      `\n\`\`\`${language}\n${code}\n\`\`\`\n`,
+    )
+  })
+
   it('resolves src from the page, reports each import, and keeps line endings uniform', async () => {
     const imports: string[] = []
 

packages/ts-ssg/src/mdx/imports.ts

@@ -7,6 +7,7 @@ import remarkParse from 'remark-parse'
 import { unified } from 'unified'
 import { isSharedContentFile } from '../discover/content'
 import { isContentExt } from '../discover/contentExtensions'
+import { LANGUAGE_BY_EXTENSION } from './codeLanguages'
 import { findMarkupTags } from './regorMarkup'
 
 const IMPORT_TAGS = {
@@ -28,26 +29,6 @@ type ImportTagName = keyof typeof IMPORT_TAGS
 
 const IMPORT_TAG_NAMES = Object.keys(IMPORT_TAGS) as ImportTagName[]
 
-const LANGUAGE_BY_EXTENSION: Record<string, string> = {
-  '.cjs': 'javascript',
-  '.css': 'css',
-  '.cts': 'typescript',
-  '.html': 'html',
-  '.js': 'javascript',
-  '.json': 'json',
-  '.jsx': 'jsx',
-  '.md': 'markdown',
-  '.mdx': 'mdx',
-  '.mjs': 'javascript',
-  '.mts': 'typescript',
-  '.rmdx': 'mdx',
-  '.sh': 'bash',
-  '.ts': 'typescript',
-  '.tsx': 'tsx',
-  '.yaml': 'yaml',
-  '.yml': 'yaml',
-}
-
 const parser = unified().use(remarkParse).use(remarkGfm)
 
 export interface ImportOptions {