Merge syntax-highlight: revealed mermaid + LaTeX math highlighting

Author: wilfreddenton

src/doc_layout.rs

@@ -682,6 +682,7 @@ fn build_ghosts_before(
             &[],
             None,
             &[],
+            &[],
         );
         let layout = engine.build_line_hanging(
             &lr.text,
@@ -1446,6 +1447,40 @@ impl DocLayout {
                 });
 
             let extra = github.map(|g| g.extra_regions(i)).unwrap_or_default();
+            // Byte ranges on this line to LaTeX-highlight: the content of a revealed inline
+            // `$…$` span (caret inside it) or a revealed `$$…$$` block line (caret in the
+            // block / a selection overlapping it — the same predicate that stops the block
+            // from collapsing to an image). Empty when the `math` feature is off.
+            #[cfg(feature = "math")]
+            let latex_ranges: Vec<Range<usize>> = {
+                let mut lr = Vec::new();
+                if let Some(spans) = math_spans.get(&i) {
+                    for s in spans {
+                        if cursor_offset >= s.full_range.start && cursor_offset <= s.full_range.end
+                        {
+                            let a = s.content_range.start.max(range.start);
+                            let b = s.content_range.end.min(range.end);
+                            if a < b {
+                                lr.push(a..b);
+                            }
+                        }
+                    }
+                }
+                for m in &math_blocks {
+                    let revealed = (m.block.start < selection.end && selection.start < m.block.end)
+                        || m.block.contains(&cursor_line_start);
+                    if revealed {
+                        let a = m.content.start.max(range.start);
+                        let b = m.content.end.min(range.end);
+                        if a < b {
+                            lr.push(a..b);
+                        }
+                    }
+                }
+                lr
+            };
+            #[cfg(not(feature = "math"))]
+            let latex_ranges: Vec<Range<usize>> = Vec::new();
             // Reuse the cached render when nothing about this line changed. Lines with
             // GitHub extra regions bypass the cache (validation can change them without
             // a version bump); all others key on (version, line, cursor-on-line).
@@ -1462,6 +1497,7 @@ impl DocLayout {
                         &[],
                         tc,
                         math_spans.get(&i).map_or(&[][..], |v| v.as_slice()),
+                        &latex_ranges,
                     ))
                 })
             } else {
@@ -1475,6 +1511,7 @@ impl DocLayout {
                     &extra,
                     table_ctx.clone(),
                     math_spans.get(&i).map_or(&[][..], |v| v.as_slice()),
+                    &latex_ranges,
                 ))
             };
             #[cfg_attr(not(feature = "math"), allow(unused_mut))]

src/highlight.rs

@@ -6,6 +6,8 @@ use tree_sitter_highlight::{
     Highlight, HighlightConfiguration, HighlightEvent, Highlighter as TSHighlighter,
 };
 
+use crate::tokenize;
+
 pub const HIGHLIGHT_NAMES: &[&str] = &[
     "attribute",
     "boolean",
@@ -43,6 +45,18 @@ pub struct HighlightSpan {
     pub highlight_id: usize,
 }
 
+/// Resolve a capture name (e.g. `"keyword"`, `"operator"`) to its `highlight_id` — the
+/// index into [`HIGHLIGHT_NAMES`] that [`EditorTheme::color_for_highlight`] maps to a
+/// color. For the hand-written tokenizers (mermaid, latex), which emit categories by name
+/// rather than via a tree-sitter query. An unknown name resolves to a sentinel that renders
+/// as the plain foreground.
+pub fn highlight_id(name: &str) -> usize {
+    HIGHLIGHT_NAMES
+        .iter()
+        .position(|&n| n == name)
+        .unwrap_or(usize::MAX)
+}
+
 /// Language keys are stored lowercase. Only allocate when the input actually has an
 /// uppercase byte — the overwhelmingly common case (already-lowercase fence infos)
 /// borrows unchanged.
@@ -54,13 +68,21 @@ fn normalized_lang(lang: &str) -> Cow<'_, str> {
     }
 }
 
-struct LanguageConfig {
-    config: HighlightConfiguration,
+/// How a registered language produces highlight spans. Most languages use a tree-sitter
+/// grammar + query; a few (mermaid, latex) have no publishable grammar crate and are
+/// lexed by a pure-Rust tokenizer instead. Both go through the same `highlight()` entry
+/// point, so callers (the code-block highlight cache, revealed-math styling) never
+/// special-case which backend a language uses.
+enum Backend {
+    /// Boxed — a `HighlightConfiguration` is large, and it lives behind an `Arc` anyway.
+    TreeSitter(Box<HighlightConfiguration>),
+    /// `fn(source) -> spans`, each span's `highlight_id` from [`highlight_id`].
+    Tokenizer(fn(&str) -> Vec<HighlightSpan>),
 }
 
 pub struct Highlighter {
     inner: TSHighlighter,
-    languages: HashMap<String, Arc<LanguageConfig>>,
+    languages: HashMap<String, Arc<Backend>>,
 }
 
 impl Default for Highlighter {
@@ -74,25 +96,40 @@ impl Highlighter {
         let inner = TSHighlighter::new();
         let mut languages = HashMap::new();
 
-        // Register Rust
+        let mut register = |aliases: &[&str], backend: Backend| {
+            let backend = Arc::new(backend);
+            for a in aliases {
+                languages.insert((*a).to_string(), Arc::clone(&backend));
+            }
+        };
+
+        // Tree-sitter grammars.
         if let Some(config) = Self::create_rust_config() {
-            let config = Arc::new(config);
-            languages.insert("rust".to_string(), Arc::clone(&config));
-            languages.insert("rs".to_string(), Arc::clone(&config));
+            register(&["rust", "rs"], Backend::TreeSitter(Box::new(config)));
         }
-
-        // Register Bash
         if let Some(config) = Self::create_bash_config() {
-            let config = Arc::new(config);
-            languages.insert("bash".to_string(), Arc::clone(&config));
-            languages.insert("sh".to_string(), Arc::clone(&config));
-            languages.insert("shell".to_string(), Arc::clone(&config));
+            register(
+                &["bash", "sh", "shell"],
+                Backend::TreeSitter(Box::new(config)),
+            );
         }
 
+        // Tokenizer-backed languages (no publishable grammar crate). Registered ungated:
+        // a ```mermaid / ```latex code fence highlights through the normal code-block path,
+        // and revealed `$…$` math styles via `highlight(content, "latex")`.
+        register(
+            &["mermaid"],
+            Backend::Tokenizer(tokenize::highlight_mermaid),
+        );
+        register(
+            &["latex", "tex"],
+            Backend::Tokenizer(tokenize::highlight_latex),
+        );
+
         Self { inner, languages }
     }
 
-    fn create_rust_config() -> Option<LanguageConfig> {
+    fn create_rust_config() -> Option<HighlightConfiguration> {
         let language = tree_sitter_rust::LANGUAGE.into();
 
         // Use Zed's highlights.scm for better Rust coverage
@@ -116,10 +153,10 @@ impl Highlighter {
         // Configure which highlight names we recognize
         config.configure(HIGHLIGHT_NAMES);
 
-        Some(LanguageConfig { config })
+        Some(config)
     }
 
-    fn create_bash_config() -> Option<LanguageConfig> {
+    fn create_bash_config() -> Option<HighlightConfiguration> {
         let language = tree_sitter_bash::LANGUAGE.into();
         let highlights_query = tree_sitter_bash::HIGHLIGHT_QUERY;
 
@@ -134,21 +171,26 @@ impl Highlighter {
 
         config.configure(HIGHLIGHT_NAMES);
 
-        Some(LanguageConfig { config })
+        Some(config)
     }
 
     pub fn supports_language(&self, lang: &str) -> bool {
         self.languages.contains_key(normalized_lang(lang).as_ref())
     }
 
     pub fn highlight(&mut self, code: &str, language: &str) -> Vec<HighlightSpan> {
-        let Some(lang_config) = self.languages.get(normalized_lang(language).as_ref()) else {
+        let Some(backend) = self.languages.get(normalized_lang(language).as_ref()) else {
             return Vec::new();
         };
 
+        let config = match backend.as_ref() {
+            Backend::Tokenizer(f) => return f(code),
+            Backend::TreeSitter(config) => config.as_ref(),
+        };
+
         // Run the highlighter
         let highlights = match self.inner.highlight(
-            &lang_config.config,
+            config,
             code.as_bytes(),
             None,     // cancellation flag
             |_| None, // injection callback (not used)

src/lib.rs

@@ -64,4 +64,5 @@ pub mod status_bar;
 pub mod table;
 pub mod text_engine;
 pub mod text_input;
+pub mod tokenize;
 pub mod validation;