From 059bcb8f60eb344cf6bc88624efb2e8ff01e4cd6 Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Wed, 29 Jul 2026 16:46:48 +0200
Subject: [PATCH 1/4] new literal module in AtomSplit

---
 tokenizers/atomsplit/Cargo.toml       |  2 +-
 tokenizers/atomsplit/src/lib.rs       |  5 +++
 tokenizers/atomsplit/src/literal.rs   | 53 +++++++++++++++++++++++++++
 tokenizers/atomsplit/tests/literal.rs | 39 ++++++++++++++++++++
 4 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 tokenizers/atomsplit/src/literal.rs
 create mode 100644 tokenizers/atomsplit/tests/literal.rs

diff --git a/tokenizers/atomsplit/Cargo.toml b/tokenizers/atomsplit/Cargo.toml
index 0a9de6a865..fb8947f3f4 100644
--- a/tokenizers/atomsplit/Cargo.toml
+++ b/tokenizers/atomsplit/Cargo.toml
@@ -25,7 +25,7 @@ name = "atomsplit"
 path = "src/lib.rs"
 
 [dependencies]
-memchr = "2.8.2" # SIMD single-byte search; used only in CharDelimiterSplit (1.4–23× vs scalar).
+memchr = "2.8.2" # SIMD search: used for single-byte  search in CharDelimiterSplit (1.4–23× vs scalar), or multi-byte string pattern search in `literal`.
 # NOTE: classify tables live in src/atom_tables.rs (committed, generated). Regenerate after any atom
 # scheme change with `cargo run -p bitmap_gen`. No build script / build-dep — atomsplit builds clean.
 # onig is C (Oniguruma) and fancy-regex pulls it in transitively for benches only — neither
diff --git a/tokenizers/atomsplit/src/lib.rs b/tokenizers/atomsplit/src/lib.rs
index ceefc9296c..5f22f52d00 100644
--- a/tokenizers/atomsplit/src/lib.rs
+++ b/tokenizers/atomsplit/src/lib.rs
@@ -7,6 +7,10 @@
 //! Mistral's tekken are exposed as the [`fsm::fsm_o200k`] / [`fsm::fsm_tekken`] functions rather than
 //! recipe structs).
 //!
+//! For pre-tokenizers that split on single characters (such as the Metaspace `▁` delimiter),
+//! we skip the atom classification pass entirely and search for the raw  bytes instead.
+//! See [`literal`].
+//!
 //! Design: every fsm is *no-push* — it writes spans into a caller-preallocated `&mut [fsm::Span]`
 //! (length ≥ `text.len()`) and returns the token count; there is no `Vec`/allocation on the hot path.
 //!
@@ -17,6 +21,7 @@
 mod atom_tables;
 pub mod classify;
 pub mod fsm;
+pub mod literal;
 pub mod regexes;
 #[cfg(target_arch = "x86_64")]
 mod simd_avx_classify;
diff --git a/tokenizers/atomsplit/src/literal.rs b/tokenizers/atomsplit/src/literal.rs
new file mode 100644
index 0000000000..041694cd4e
--- /dev/null
+++ b/tokenizers/atomsplit/src/literal.rs
@@ -0,0 +1,53 @@
+//! Searching for a literal string, for pre-tokenizers that cut on one exact character.
+//!
+//! The rest of the crate works off atom tags: one SIMD pass gives every character a class, and the
+//! FSMs cut where the class changes ([`crate::fsm`]).
+//!
+//! The atom classification is unnecessary for pre-tokenizers splitting on an exact character or literal string:
+//! We can use a simpler byte search looking at 16 bytes at a time.
+
+use memchr::memmem;
+use std::fmt;
+
+/// The pattern handed to [`Literal::new`] was empty.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EmptyPattern;
+
+impl fmt::Display for EmptyPattern {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("an empty pattern matches everywhere")
+    }
+}
+
+impl std::error::Error for EmptyPattern {}
+
+/// A literal string to split on.
+#[derive(Debug, Clone)]
+pub struct Literal {
+    finder: memmem::Finder<'static>,
+}
+
+impl Literal {
+    /// # Errors
+    /// If `pattern` is empty, which would match everywhere.
+    pub fn new(pattern: &[u8]) -> Result<Self, EmptyPattern> {
+        if pattern.is_empty() {
+            return Err(EmptyPattern);
+        }
+        Ok(Self {
+            finder: memmem::Finder::new(pattern).into_owned(),
+        })
+    }
+
+    /// The string being searched for.
+    #[must_use]
+    pub fn pattern(&self) -> &[u8] {
+        self.finder.needle()
+    }
+
+    /// Byte offset of every match, left to right. Matches never overlap, so `"aa"` is found once in
+    /// `"aaa"` — the same matches a regex engine would report.
+    pub fn matches<'t>(&'t self, text: &'t [u8]) -> impl Iterator<Item = usize> + 't {
+        self.finder.find_iter(text)
+    }
+}
diff --git a/tokenizers/atomsplit/tests/literal.rs b/tokenizers/atomsplit/tests/literal.rs
new file mode 100644
index 0000000000..80978ba42e
--- /dev/null
+++ b/tokenizers/atomsplit/tests/literal.rs
@@ -0,0 +1,39 @@
+//! Tests for the literal search. Kept out of `src/` so the core stays production-only.
+use atomsplit::literal::{EmptyPattern, Literal};
+
+#[test]
+fn finds_every_match_and_nothing_else() {
+    let literal = Literal::new(b"-").unwrap();
+    assert_eq!(literal.matches(b"a-b--c").collect::<Vec<_>>(), [1, 3, 4]);
+    assert_eq!(literal.matches(b"none here").count(), 0);
+    assert_eq!(literal.matches(b"").count(), 0);
+    assert_eq!(literal.pattern(), b"-");
+}
+
+#[test]
+fn a_multi_byte_pattern_only_matches_whole() {
+    // `▁` is U+2581 = E2 96 81. The other characters here share its first byte and nothing else, so a
+    // search for that byte alone would report all of them.
+    let literal = Literal::new("▁".as_bytes()).unwrap();
+    let text = "a—b“c…d▁e";
+    assert_eq!(
+        literal.matches(text.as_bytes()).collect::<Vec<_>>(),
+        [text.find('▁').unwrap()]
+    );
+}
+
+#[test]
+fn matches_do_not_overlap() {
+    let literal = Literal::new(b"aa").unwrap();
+    assert_eq!(literal.matches(b"aaa").collect::<Vec<_>>(), [0]);
+    assert_eq!(literal.matches(b"aaaa").collect::<Vec<_>>(), [0, 2]);
+}
+
+#[test]
+fn an_empty_pattern_is_rejected() {
+    assert_eq!(Literal::new(b"").unwrap_err(), EmptyPattern);
+    assert_eq!(
+        EmptyPattern.to_string(),
+        "an empty pattern matches everywhere"
+    );
+}

From f01e1656141c2ecb11b20cbd37730e13c280a024 Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Wed, 29 Jul 2026 17:15:04 +0200
Subject: [PATCH 2/4] use Literal in Replace normalizer and Split pretok

---
 tokenizers/tk-encode/Cargo.toml               |   9 +-
 .../tk-encode/src/normalizers/replace.rs      | 160 +++++++++++++-----
 .../tk-encode/src/pre_tokenizers/split.rs     | 128 ++++++++++----
 .../tk-encode/src/tokenizer/normalizer.rs     |   3 +-
 tokenizers/tk-encode/src/tokenizer/pattern.rs |  73 +++++---
 .../tk-encode/src/tokenizer/pipeline.rs       |  28 +++
 tokenizers/tk-encode/src/utils/mod.rs         |   7 +-
 tokenizers/tk-encode/src/utils/no_regex.rs    |  14 +-
 tokenizers/tk-encode/tests/pipeline_oracle.rs |   9 +-
 9 files changed, 310 insertions(+), 121 deletions(-)

diff --git a/tokenizers/tk-encode/Cargo.toml b/tokenizers/tk-encode/Cargo.toml
index ba64839a53..5f830e3ee6 100644
--- a/tokenizers/tk-encode/Cargo.toml
+++ b/tokenizers/tk-encode/Cargo.toml
@@ -73,10 +73,11 @@ pcre2 = { version = "0.2", optional = true }
 logos = { version = "0.15", optional = true } # compile-time DFA lexer reference (pure Rust)
 
 [features]
-# `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a `Split` pre-tokenizer with an
-# arbitrary (non-GPT) regex or the `Replace` normalizer — the atomsplit-native pre-tokenizers (GPT-2,
-# cl100k, deepseek, the class family, char-delimiter) need no backend. Without it a stub compiles and
-# those arbitrary-regex paths error at load. Enable with `--features fancy-regex`.
+# `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a *regex* pattern in a `Split`
+# pre-tokenizer or a `Replace` normalizer — the atomsplit-native pre-tokenizers (GPT-2, cl100k,
+# deepseek, the class family, char-delimiter) need no backend, and a plain string pattern is searched
+# for directly. Without it a stub compiles and those regex paths error at load. Enable with
+# `--features fancy-regex`.
 default = ["progressbar"]
 progressbar = ["indicatif"]
 http = ["hf-hub"]
diff --git a/tokenizers/tk-encode/src/normalizers/replace.rs b/tokenizers/tk-encode/src/normalizers/replace.rs
index 48fc581801..cc4545b47a 100644
--- a/tokenizers/tk-encode/src/normalizers/replace.rs
+++ b/tokenizers/tk-encode/src/normalizers/replace.rs
@@ -5,6 +5,7 @@ use crate::tokenizer::Decoder;
 use crate::tokenizer::pattern::Pattern;
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 use crate::utils::SysRegex;
+use atomsplit::literal::Literal;
 use serde::{Deserialize, Serialize};
 
 /// Represents the different patterns that `Replace` can use
@@ -26,7 +27,7 @@ impl From<&str> for ReplacePattern {
     }
 }
 
-/// We use this custom deserializer to provide the value for `regex` for `Replace`
+/// We use this custom deserializer to build the search for `Replace`
 #[doc(hidden)]
 #[derive(Deserialize)]
 #[serde(tag = "type")]
@@ -43,6 +44,27 @@ impl std::convert::TryFrom<ReplaceDeserializer> for Replace {
     }
 }
 
+/// How a [`Replace`] looks for its pattern.
+#[derive(Debug)]
+enum Search {
+    /// A plain string, scanned for directly — no regex engine involved, so this works in every build.
+    Literal(Literal),
+    /// A real regex, which needs the system backend (the `fancy-regex` feature).
+    Regex(SysRegex),
+    /// The empty string, which matches nothing.
+    Nothing,
+}
+
+impl Search {
+    fn find_matches(&self, inside: &str) -> Result<Vec<((usize, usize), bool)>> {
+        match self {
+            Self::Literal(literal) => literal.find_matches(inside),
+            Self::Regex(regex) => regex.find_matches(inside),
+            Self::Nothing => Ok(vec![((0, inside.len()), false)]),
+        }
+    }
+}
+
 /// This normalizer will take a `pattern` (for now only a String)
 /// and replace every occurrence with `content`.
 #[derive(Debug, Serialize, Deserialize)]
@@ -51,7 +73,7 @@ pub struct Replace {
     pattern: ReplacePattern,
     pub content: String,
     #[serde(skip)]
-    regex: SysRegex,
+    search: Search,
 }
 
 impl Clone for Replace {
@@ -69,45 +91,68 @@ impl PartialEq for Replace {
 impl Replace {
     pub fn new<I: Into<ReplacePattern>, C: Into<String>>(pattern: I, content: C) -> Result<Self> {
         let pattern: ReplacePattern = pattern.into();
-        let regex = match &pattern {
-            ReplacePattern::String(s) => SysRegex::new(&regex::escape(s))?,
-            ReplacePattern::Regex(r) => SysRegex::new(r)?,
+        let search = match &pattern {
+            ReplacePattern::String(s) if s.is_empty() => Search::Nothing,
+            ReplacePattern::String(s) => Search::Literal(Literal::new(s.as_bytes())?),
+            ReplacePattern::Regex(r) => Search::Regex(SysRegex::new(r)?),
         };
 
         Ok(Self {
             pattern,
             content: content.into(),
-            regex,
+            search,
         })
     }
 }
 
 impl Normalizer for Replace {
     fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> {
-        normalized.replace(&self.regex, &self.content)
+        match &self.search {
+            Search::Literal(literal) => normalized.replace(literal, &self.content),
+            Search::Regex(regex) => normalized.replace(regex, &self.content),
+            Search::Nothing => Ok(()),
+        }
+    }
+}
+
+/// Builds the text with every match swapped for `content`. Borrows the input back untouched when
+/// nothing matched, which is what keeps the common case free of allocation.
+fn replace_matches<'a>(
+    input: &'a str,
+    content: &str,
+    matches: impl Iterator<Item = (usize, usize)>,
+) -> Cow<'a, str> {
+    let mut replaced: Option<String> = None;
+    let mut last_end = 0;
+    for (start, end) in matches {
+        let replaced: &mut String =
+            replaced.get_or_insert_with(|| String::with_capacity(input.len()));
+        replaced.push_str(&input[last_end..start]);
+        replaced.push_str(content);
+        last_end = end;
+    }
+    match replaced {
+        Some(mut replaced) => {
+            replaced.push_str(&input[last_end..]);
+            Cow::Owned(replaced)
+        }
+        None => Cow::Borrowed(input),
     }
 }
 
 impl pipeline::Normalizer for Replace {
     fn normalize<'a>(&self, input: &'a str) -> Result<Cow<'a, str>> {
-        let iter = self.regex.find_iter(input);
-        let mut replaced: Option<String> = None;
-        let mut last_end = 0;
-
-        for (start, end) in iter {
-            let replaced: &mut String =
-                replaced.get_or_insert_with(|| String::with_capacity(input.len()));
-            replaced.push_str(&input[last_end..start]);
-            replaced.push_str(&self.content);
-            last_end = end;
-        }
-        if let Some(mut replaced) = replaced {
-            if last_end < input.len() {
-                replaced.push_str(&input[last_end..]);
+        Ok(match &self.search {
+            Search::Literal(literal) => {
+                let width = literal.pattern().len();
+                let matches = literal
+                    .matches(input.as_bytes())
+                    .map(|start| (start, start + width));
+                replace_matches(input, &self.content, matches)
             }
-            return Ok(Cow::Owned(replaced));
-        }
-        Ok(input.into())
+            Search::Regex(regex) => replace_matches(input, &self.content, regex.find_iter(input)),
+            Search::Nothing => Cow::Borrowed(input),
+        })
     }
 }
 
@@ -118,7 +163,7 @@ impl Decoder for Replace {
             .map(|token| -> Result<String> {
                 let mut new_token = "".to_string();
 
-                for ((start, stop), is_match) in (&self.regex).find_matches(&token)? {
+                for ((start, stop), is_match) in self.search.find_matches(&token)? {
                     if is_match {
                         new_token.push_str(&self.content);
                     } else {
@@ -131,8 +176,7 @@ impl Decoder for Replace {
     }
 }
 
-// `Replace` needs a system-regex backend (SysRegex) for every test here.
-#[cfg(all(test, feature = "fancy-regex"))]
+#[cfg(test)]
 mod tests {
     use super::*;
 
@@ -148,6 +192,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "fancy-regex")] // a regex pattern needs a system-regex backend
     fn test_replace_regex() {
         let original = "This     is   a         test";
         let normalized = "This is a test";
@@ -162,6 +207,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "fancy-regex")] // the regex half of this needs a system-regex backend
     fn serialization() {
         let replace = Replace::new("Hello", "Hey").unwrap();
         let replace_s = r#"{"type":"Replace","pattern":{"String":"Hello"},"content":"Hey"}"#;
@@ -174,6 +220,37 @@ mod tests {
         assert_eq!(serde_json::from_str::<Replace>(replace_s).unwrap(), replace);
     }
 
+    /// The goal of the literal path: a plain string pattern builds and runs with no regex backend.
+    #[test]
+    fn a_string_pattern_needs_no_backend() {
+        let replace = Replace::new(" ", "▁").unwrap();
+        assert_eq!(
+            pipeline::Normalizer::normalize(&replace, "a b  c").unwrap(),
+            "a▁b▁▁c"
+        );
+        // Nothing to replace: the input is handed back as it is, with nothing allocated.
+        assert!(matches!(
+            pipeline::Normalizer::normalize(&replace, "abc").unwrap(),
+            Cow::Borrowed("abc")
+        ));
+        // An empty pattern would match everywhere, so it matches nowhere instead.
+        let empty = Replace::new("", "x").unwrap();
+        assert_eq!(
+            pipeline::Normalizer::normalize(&empty, "abc").unwrap(),
+            "abc"
+        );
+    }
+
+    /// A config spelling its pattern as a string must also *deserialize* with no backend — the
+    /// regex half of `serialization` above can only run once one is compiled.
+    #[test]
+    fn a_string_pattern_deserializes_with_no_backend() {
+        let replace_s = r#"{"type":"Replace","pattern":{"String":"Hello"},"content":"Hey"}"#;
+        let replace = Replace::new("Hello", "Hey").unwrap();
+        assert_eq!(serde_json::from_str::<Replace>(replace_s).unwrap(), replace);
+        assert_eq!(serde_json::to_string(&replace).unwrap(), replace_s);
+    }
+
     #[test]
     fn test_replace_decode() {
         let original = vec!["hello".to_string(), "_hello".to_string()];
@@ -184,26 +261,27 @@ mod tests {
         );
     }
 
-    #[test]
-    fn pipeline_replace_matches_legacy() {
-        let n = Replace::new("''", "\"").unwrap();
-        for input in &["This is a ''test''", "no quotes", ""] {
+    fn assert_pipeline_matches_legacy(n: &Replace, inputs: &[&str]) {
+        for input in inputs {
             let mut ns = NormalizedString::from(*input);
-            Normalizer::normalize(&n, &mut ns).unwrap(); // legacy oracle
+            Normalizer::normalize(n, &mut ns).unwrap(); // legacy oracle
             assert_eq!(
                 ns.get(),
-                &*pipeline::Normalizer::normalize(&n, input).unwrap()
+                &*pipeline::Normalizer::normalize(n, input).unwrap()
             );
         }
+    }
 
+    #[test]
+    fn pipeline_replace_matches_legacy() {
+        let n = Replace::new("''", "\"").unwrap();
+        assert_pipeline_matches_legacy(&n, &["This is a ''test''", "no quotes", ""]);
+    }
+
+    #[test]
+    #[cfg(feature = "fancy-regex")] // a regex pattern needs a system-regex backend
+    fn pipeline_replace_matches_legacy_for_a_regex() {
         let n = Replace::new(ReplacePattern::Regex(r"\s+".into()), " ").unwrap();
-        for input in &["a   b   c", "single", ""] {
-            let mut ns = NormalizedString::from(*input);
-            Normalizer::normalize(&n, &mut ns).unwrap(); // legacy oracle
-            assert_eq!(
-                ns.get(),
-                &*pipeline::Normalizer::normalize(&n, input).unwrap()
-            );
-        }
+        assert_pipeline_matches_legacy(&n, &["a   b   c", "single", ""]);
     }
 }
diff --git a/tokenizers/tk-encode/src/pre_tokenizers/split.rs b/tokenizers/tk-encode/src/pre_tokenizers/split.rs
index 61c4835594..779c167f10 100644
--- a/tokenizers/tk-encode/src/pre_tokenizers/split.rs
+++ b/tokenizers/tk-encode/src/pre_tokenizers/split.rs
@@ -1,5 +1,6 @@
 use crate::pipeline;
 use crate::utils::{GptFsm, GptFsmPattern, SysRegex, gpt_fsm};
+use atomsplit::literal::Literal;
 use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::tokenizer::{
@@ -26,15 +27,26 @@ impl From<&str> for SplitPattern {
     }
 }
 
+/// How a [`Split`] looks for its pattern.
+#[derive(Debug)]
+pub enum Search {
+    /// A plain string, scanned for directly — no regex engine involved, so this works in every build.
+    Literal(Literal),
+    /// A real regex, which needs the system backend (the `fancy-regex` feature).
+    Regex(SysRegex),
+    /// No way to search: no backend is compiled and the pattern is a regex. Splitting then only works
+    /// through the native FSM below, and errors otherwise.
+    Unavailable,
+}
+
 #[derive(Debug, Serialize)]
 #[serde(tag = "type")]
 pub struct Split {
     pub pattern: SplitPattern,
-    /// System-regex backend for the pattern. `None` only when no backend is compiled *and* the
-    /// pattern is a recognized GPT regex handled natively by `fsm` — so no backend is needed.
-    /// With `fancy-regex` enabled this is always `Some`; the default build has no backend (`fsm` only).
+    /// How the pattern is found. A plain string never needs a backend; a regex does, unless it is one
+    /// of the GPT patterns the native FSM below covers.
     #[serde(skip)]
-    pub regex: Option<SysRegex>,
+    pub search: Search,
     pub behavior: SplitDelimiterBehavior,
     pub invert: bool,
     /// Native `atomsplit` FSM for a recognized GPT regex (gpt2 / cl100k-Llama-3 / o200k), used on the
@@ -93,22 +105,21 @@ impl Split {
             SplitPattern::String(_) => None,
             SplitPattern::Regex(r) => gpt_fsm(r),
         };
-        // Compile a system-regex backend for the pattern. With `fancy-regex` enabled this succeeds and
-        // the legacy path is unchanged; with no backend (the default) it's only fatal when atomsplit
-        // can't cover the pattern (i.e. an arbitrary regex, not a recognized GPT one).
-        let compiled = match &pattern {
-            SplitPattern::String(s) => SysRegex::new(&regex::escape(s)),
-            SplitPattern::Regex(r) => SysRegex::new(r),
-        };
-        let regex = match compiled {
-            Ok(re) => Some(re),
-            Err(_) if fsm.is_some() => None,
-            Err(e) => return Err(e),
+        let search = match &pattern {
+            SplitPattern::String(s) => Search::Literal(Literal::new(s.as_bytes())?),
+            // A regex needs the system backend. Missing it is only fatal when the native FSM cannot
+            // cover this pattern either — which `pre_tokenize` reports, since a recognized GPT
+            // pattern in its usual form splits without any backend.
+            SplitPattern::Regex(r) => match SysRegex::new(r) {
+                Ok(regex) => Search::Regex(regex),
+                Err(_) if fsm.is_some() => Search::Unavailable,
+                Err(e) => return Err(e),
+            },
         };
 
         Ok(Self {
             pattern,
-            regex,
+            search,
             behavior,
             invert,
             fsm,
@@ -133,15 +144,27 @@ impl Split {
 
 impl PreTokenizer for Split {
     fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
-        if let Some(regex) = &self.regex {
-            return if self.invert {
-                pretokenized.split(|_, normalized| normalized.split(Invert(regex), self.behavior))
-            } else {
-                pretokenized.split(|_, normalized| normalized.split(regex, self.behavior))
-            };
+        match &self.search {
+            Search::Literal(literal) => {
+                return if self.invert {
+                    pretokenized
+                        .split(|_, normalized| normalized.split(Invert(literal), self.behavior))
+                } else {
+                    pretokenized.split(|_, normalized| normalized.split(literal, self.behavior))
+                };
+            }
+            Search::Regex(regex) => {
+                return if self.invert {
+                    pretokenized
+                        .split(|_, normalized| normalized.split(Invert(regex), self.behavior))
+                } else {
+                    pretokenized.split(|_, normalized| normalized.split(regex, self.behavior))
+                };
+            }
+            Search::Unavailable => {}
         }
-        // No system-regex backend: only a recognized GPT pattern in its canonical usage (Isolated,
-        // not inverted — how these regexes always ship) can split, via the native atomsplit FSM.
+        // No way to search for the pattern: only a recognized GPT pattern in its canonical usage
+        // (Isolated, not inverted — how these regexes always ship) can split, via the native FSM.
         let fsm = self
             .fsm
             .filter(|_| !self.invert && self.behavior == SplitDelimiterBehavior::Isolated)
@@ -178,15 +201,18 @@ impl pipeline::PreTokenizer for Split {
             );
             return Ok(());
         }
-        // Not a natively-routed GPT regex: fall back to the system-regex backend.
-        let regex = self.regex.as_ref().ok_or_else(|| -> crate::tokenizer::Error {
-            "this `Split` pattern needs a system-regex backend; enable the `fancy-regex` feature"
-                .into()
-        })?;
-        let matches = if self.invert {
-            Invert(regex).find_matches(text)?
-        } else {
-            regex.find_matches(text)?
+        // Not a natively-routed GPT regex: fall-back to Literal or Regex search
+        let matches = match (&self.search, self.invert) {
+            (Search::Literal(literal), false) => literal.find_matches(text)?,
+            (Search::Literal(literal), true) => Invert(literal).find_matches(text)?,
+            (Search::Regex(regex), false) => regex.find_matches(text)?,
+            (Search::Regex(regex), true) => Invert(regex).find_matches(text)?,
+            (Search::Unavailable, _) => {
+                return Err(
+                    "this `Split` pattern needs a system-regex backend; enable the `fancy-regex` feature"
+                        .into(),
+                );
+            }
         };
         pipeline::split_matches(out, matches, self.behavior);
         Ok(())
@@ -523,6 +549,41 @@ mod tests {
         );
     }
 
+    /// The goal of the literal path: a plain string pattern splits with no regex backend, on both the
+    /// legacy and the pipeline path.
+    #[test]
+    fn a_string_pattern_needs_no_backend() {
+        let pretok = Split::new("-", SplitDelimiterBehavior::Removed, false).unwrap();
+        let mut legacy = PreTokenizedString::from("a-b--c");
+        pretok.pre_tokenize(&mut legacy).unwrap();
+        let words: Vec<&str> = legacy
+            .get_splits(OffsetReferential::Original, OffsetType::Byte)
+            .iter()
+            .map(|(word, _, _)| *word)
+            .collect();
+        assert_eq!(words, ["a", "b", "c"]);
+        assert_eq!(
+            pipeline_split(
+                SplitPattern::String("-".into()),
+                SplitDelimiterBehavior::Removed,
+                false,
+                "a-b--c"
+            ),
+            [("a", (0, 1)), ("b", (2, 3)), ("c", (5, 6))]
+        );
+    }
+
+    /// A config spelling its pattern as a string must also *deserialize* with no backend — the
+    /// regex half of `serialization` below can only run once one is compiled.
+    #[test]
+    fn a_string_pattern_deserializes_with_no_backend() {
+        let split_s =
+            r#"{"type":"Split","pattern":{"String":"Hello"},"behavior":"Removed","invert":true}"#;
+        let split = Split::new("Hello", SplitDelimiterBehavior::Removed, true).unwrap();
+        assert_eq!(serde_json::from_str::<Split>(split_s).unwrap(), split);
+        assert_eq!(serde_json::to_string(&split).unwrap(), split_s);
+    }
+
     #[cfg(feature = "fancy-regex")] // needs a system-regex backend
     #[test]
     fn regex_string() {
@@ -548,7 +609,6 @@ mod tests {
         assert_eq!(pretok_str_for_regex, pretok_str_for_string);
     }
 
-    #[cfg(feature = "fancy-regex")] // needs a system-regex backend
     #[test]
     fn invert() {
         let mut pretok_str = PreTokenizedString::from("Hello Hello Hello");
diff --git a/tokenizers/tk-encode/src/tokenizer/normalizer.rs b/tokenizers/tk-encode/src/tokenizer/normalizer.rs
index 5bebd5f7b4..8f1899bdd3 100644
--- a/tokenizers/tk-encode/src/tokenizer/normalizer.rs
+++ b/tokenizers/tk-encode/src/tokenizer/normalizer.rs
@@ -1022,6 +1022,7 @@ impl From<&str> for NormalizedString {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use atomsplit::literal::Literal;
     use regex::Regex;
     use unicode_categories::UnicodeCategories;
 
@@ -1479,7 +1480,7 @@ mod tests {
 
         // Overlapping
         let mut s = NormalizedString::from("aaaab");
-        s.replace("aaa", "b").unwrap();
+        s.replace(&Literal::new(b"aaa").unwrap(), "b").unwrap();
         assert_eq!(s.get(), "bab");
 
         // Regex
diff --git a/tokenizers/tk-encode/src/tokenizer/pattern.rs b/tokenizers/tk-encode/src/tokenizer/pattern.rs
index 5e8821f953..4a1f6054fd 100644
--- a/tokenizers/tk-encode/src/tokenizer/pattern.rs
+++ b/tokenizers/tk-encode/src/tokenizer/pattern.rs
@@ -1,5 +1,6 @@
 use crate::utils::SysRegex;
 use crate::{Offsets, Result};
+use atomsplit::literal::Literal;
 use regex::Regex;
 
 /// Pattern used to split a NormalizedString
@@ -19,25 +20,6 @@ impl Pattern for char {
     }
 }
 
-impl Pattern for &str {
-    fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
-        if self.is_empty() {
-            // If we try to find the matches with an empty string, just don't match anything
-            return Ok(vec![((0, inside.chars().count()), false)]);
-        }
-
-        let re = Regex::new(&regex::escape(self))?;
-        (&re).find_matches(inside)
-    }
-}
-
-impl Pattern for &String {
-    fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
-        let s: &str = self;
-        s.find_matches(inside)
-    }
-}
-
 impl Pattern for &Regex {
     fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
         if inside.is_empty() {
@@ -60,6 +42,30 @@ impl Pattern for &Regex {
     }
 }
 
+/// Searching for a plain string, does not need a regex engine: [`Literal`] scans the bytes.
+impl Pattern for &Literal {
+    fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
+        if inside.is_empty() {
+            return Ok(vec![((0, 0), false)]);
+        }
+
+        let mut prev = 0;
+        let mut splits = Vec::with_capacity(inside.len());
+        for start in self.matches(inside.as_bytes()) {
+            let end = start + self.pattern().len();
+            if prev != start {
+                splits.push(((prev, start), false));
+            }
+            splits.push(((start, end), true));
+            prev = end;
+        }
+        if prev != inside.len() {
+            splits.push(((prev, inside.len()), false))
+        }
+        Ok(splits)
+    }
+}
+
 impl Pattern for &SysRegex {
     fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
         if inside.is_empty() {
@@ -168,17 +174,28 @@ mod tests {
     }
 
     #[test]
-    fn str() {
-        do_test!("aba", "a" => vec![((0, 1), true), ((1, 2), false), ((2, 3), true)]);
-        do_test!("bbbba", "a" => vec![((0, 4), false), ((4, 5), true)]);
-        do_test!("aabbb", "a" => vec![((0, 1), true), ((1, 2), true), ((2, 5), false)]);
-        do_test!("aabbb", "ab" => vec![((0, 1), false), ((1, 3), true), ((3, 5), false)]);
-        do_test!("aabbab", "ab" =>
+    fn literal() {
+        let a = Literal::new(b"a").unwrap();
+        do_test!("aba", &a => vec![((0, 1), true), ((1, 2), false), ((2, 3), true)]);
+        do_test!("bbbba", &a => vec![((0, 4), false), ((4, 5), true)]);
+        do_test!("aabbb", &a => vec![((0, 1), true), ((1, 2), true), ((2, 5), false)]);
+        do_test!("", &a => vec![((0, 0), false)]);
+
+        let ab = Literal::new(b"ab").unwrap();
+        do_test!("aabbb", &ab => vec![((0, 1), false), ((1, 3), true), ((3, 5), false)]);
+        do_test!("aabbab", &ab =>
             vec![((0, 1), false), ((1, 3), true), ((3, 4), false), ((4, 6), true)]
         );
-        do_test!("", "" => vec![((0, 0), false)]);
-        do_test!("aaa", "" => vec![((0, 3), false)]);
-        do_test!("aaa", "b" => vec![((0, 3), false)]);
+
+        let b = Literal::new(b"b").unwrap();
+        do_test!("aaa", &b => vec![((0, 3), false)]);
+
+        // Offsets are byte offsets, over the whole input a multi-byte pattern covers.
+        let metaspace = Literal::new("▁".as_bytes()).unwrap();
+        do_test!("a▁b", &metaspace => vec![((0, 1), false), ((1, 4), true), ((4, 5), false)]);
+
+        // An empty pattern would match everywhere, so there is no `Literal` for it.
+        assert!(Literal::new(b"").is_err());
     }
 
     #[test]
diff --git a/tokenizers/tk-encode/src/tokenizer/pipeline.rs b/tokenizers/tk-encode/src/tokenizer/pipeline.rs
index efb12eb679..0eb1aabab4 100644
--- a/tokenizers/tk-encode/src/tokenizer/pipeline.rs
+++ b/tokenizers/tk-encode/src/tokenizer/pipeline.rs
@@ -1004,6 +1004,34 @@ mod tests {
         }
     }
 
+    /// Test the literal only replace and splits can be run without the fancy-regex feature
+    #[cfg(not(feature = "fancy-regex"))]
+    #[test]
+    fn string_pattern_config_loads_and_encodes_with_no_regex_backend() {
+        let normalizer: NormalizerWrapper =
+            serde_json::from_str(r#"{"type":"Replace","pattern":{"String":" "},"content":"▁"}"#)
+                .unwrap();
+        let pre_tokenizer: PreTokenizerWrapper = serde_json::from_str(
+            r#"{"type":"Split","pattern":{"String":"▁"},"behavior":"MergedWithPrevious","invert":false}"#,
+        )
+        .unwrap();
+
+        let mut tok = wordlevel_tokenizer(vec![("<unk>", 0), ("hello▁", 1), ("world", 2)], None);
+        tok.with_normalizer(Some(normalizer)).unwrap();
+        tok.with_pre_tokenizer(Some(pre_tokenizer));
+
+        let ids: Vec<u32> = PipelineTokenizer::try_from(&tok)
+            .unwrap()
+            .encode("hello world", false)
+            .unwrap()
+            .iter()
+            .map(|t| t.id)
+            .collect();
+        // Not the unk id: both the `Replace` and the `Split` really ran on the literal path.
+        assert_eq!(ids, [1, 2]);
+        assert_pipeline_matches_reference(&tok, "hello world");
+    }
+
     #[test]
     fn segment_iterator_yields_text_and_specials_in_order() {
         let input = "aa<s>bb<s>cc";
diff --git a/tokenizers/tk-encode/src/utils/mod.rs b/tokenizers/tk-encode/src/utils/mod.rs
index 83f39109d1..4003f9e097 100644
--- a/tokenizers/tk-encode/src/utils/mod.rs
+++ b/tokenizers/tk-encode/src/utils/mod.rs
@@ -2,9 +2,10 @@ pub(crate) mod cache;
 #[cfg(feature = "http")]
 pub(crate) mod from_pretrained;
 
-// Optional system-regex backend for arbitrary (non-atomsplit) patterns. With `fancy-regex` off a
-// stub compiles and arbitrary-regex features error at load — the atomsplit-native pre-tokenizers
-// work regardless, so `fancy-regex` is only needed for custom `Split` regexes / `Replace`.
+// Optional system-regex backend, needed only for a *regex* pattern that atomsplit does not cover.
+// With `fancy-regex` off a stub compiles and those patterns error at load. Everything else works
+// regardless: the atomsplit-native pre-tokenizers, and any `Split` or `Replace` whose pattern is a
+// plain string (searched for directly, see `atomsplit::literal`).
 #[cfg(feature = "fancy-regex")]
 mod fancy;
 #[cfg(feature = "fancy-regex")]
diff --git a/tokenizers/tk-encode/src/utils/no_regex.rs b/tokenizers/tk-encode/src/utils/no_regex.rs
index 51a8d05a52..8645b69c30 100644
--- a/tokenizers/tk-encode/src/utils/no_regex.rs
+++ b/tokenizers/tk-encode/src/utils/no_regex.rs
@@ -1,9 +1,10 @@
 //! Stub `SysRegex` for builds with **no** system-regex backend (`fancy-regex` off — the default).
 //!
-//! The type stays present so `Split` / `Replace` still compile, but construction always fails: the
-//! atomsplit-native pre-tokenizers (GPT-2, cl100k, deepseek, the class family, char-delimiter) need
-//! no backend, while a `Split` with an *arbitrary* regex or the `Replace` normalizer error at load
-//! time with a clear message. Enable `fancy-regex` to get a real backend.
+//! The type stays present so `Split` / `Replace` still compile, but construction always fails. Only a
+//! *regex* pattern ever asks for it: the atomsplit-native pre-tokenizers (GPT-2, cl100k, deepseek, the
+//! class family, char-delimiter) need no backend, and a plain string pattern is searched for directly
+//! (`atomsplit::literal`). A regex atomsplit does not cover errors at load time with a clear message.
+//! Enable `fancy-regex` to get a real backend.
 use std::error::Error;
 
 #[derive(Debug)]
@@ -16,8 +17,9 @@ pub struct SysRegex {
 impl SysRegex {
     pub fn new(_regex_str: &str) -> Result<Self, Box<dyn Error + Send + Sync + 'static>> {
         Err(
-            "no system-regex backend compiled: enable the `fancy-regex` feature \
-             to use a `Split` pre-tokenizer with a custom regex, or the `Replace` normalizer"
+            "no system-regex backend compiled: enable the `fancy-regex` feature to use a regex \
+             pattern in a `Split` pre-tokenizer or a `Replace` normalizer (a plain string pattern \
+             needs no backend)"
                 .into(),
         )
     }
diff --git a/tokenizers/tk-encode/tests/pipeline_oracle.rs b/tokenizers/tk-encode/tests/pipeline_oracle.rs
index 33e8e83720..6222084627 100644
--- a/tokenizers/tk-encode/tests/pipeline_oracle.rs
+++ b/tokenizers/tk-encode/tests/pipeline_oracle.rs
@@ -107,10 +107,6 @@ fn check_model(tok_file: &str) {
     );
 }
 
-// The decode oracle's model set (one per decoder archetype) plus mistral-small-4, whose
-// tekken pre-tokenizer is its own encode archetype; whichever files a given checkout has
-// get run (bert-wiki + llama-3 ship with `make test`; the rest with `make bench-models`).
-// Unsupported models skip, not fail.
 #[test]
 fn bert_wiki() {
     check_model("bert-wiki.json");
@@ -136,6 +132,11 @@ fn llama2() {
     check_model("llama-2.json");
 }
 
+#[test]
+fn gemma4() {
+    check_model("gemma-4.json");
+}
+
 #[test]
 fn t5_base() {
     check_model("t5-base.json");

From 1a383917178ecd9aafb9a252d25eb049b77b6ccd Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Wed, 29 Jul 2026 17:37:22 +0200
Subject: [PATCH 3/4] box the Literal finder

`memmem::Finder` carries a few hundred bytes of prefilter state on x86_64 (much
less on aarch64, which is why this only showed up in CI): a `Literal` stored
inline blew `NormalizerWrapper` and `DecoderWrapper` up to 352 bytes through
`Replace`, which `clippy::large_enum_variant` rejects.
---
 tokenizers/atomsplit/src/literal.rs   |  7 +++++--
 tokenizers/atomsplit/tests/literal.rs | 11 +++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/tokenizers/atomsplit/src/literal.rs b/tokenizers/atomsplit/src/literal.rs
index 041694cd4e..c826b2dda6 100644
--- a/tokenizers/atomsplit/src/literal.rs
+++ b/tokenizers/atomsplit/src/literal.rs
@@ -22,9 +22,12 @@ impl fmt::Display for EmptyPattern {
 impl std::error::Error for EmptyPattern {}
 
 /// A literal string to split on.
+///
+/// The finder is boxed because it is large — a few hundred bytes of prefilter state on x86_64 — and
+/// callers store it inside enums whose other variants are tiny.
 #[derive(Debug, Clone)]
 pub struct Literal {
-    finder: memmem::Finder<'static>,
+    finder: Box<memmem::Finder<'static>>,
 }
 
 impl Literal {
@@ -35,7 +38,7 @@ impl Literal {
             return Err(EmptyPattern);
         }
         Ok(Self {
-            finder: memmem::Finder::new(pattern).into_owned(),
+            finder: Box::new(memmem::Finder::new(pattern).into_owned()),
         })
     }
 
diff --git a/tokenizers/atomsplit/tests/literal.rs b/tokenizers/atomsplit/tests/literal.rs
index 80978ba42e..14ceed9a03 100644
--- a/tokenizers/atomsplit/tests/literal.rs
+++ b/tokenizers/atomsplit/tests/literal.rs
@@ -29,6 +29,17 @@ fn matches_do_not_overlap() {
     assert_eq!(literal.matches(b"aaaa").collect::<Vec<_>>(), [0, 2]);
 }
 
+/// A `Literal` is stored inline in the normalizer and decoder enums, where every other variant is a
+/// handful of bytes. `memmem::Finder` itself is a few hundred, so it has to stay behind a pointer.
+#[test]
+fn a_literal_is_pointer_sized() {
+    assert_eq!(
+        size_of::<Literal>(),
+        size_of::<*const u8>(),
+        "a Literal must not carry its finder inline"
+    );
+}
+
 #[test]
 fn an_empty_pattern_is_rejected() {
     assert_eq!(Literal::new(b"").unwrap_err(), EmptyPattern);

From 179799c88daae7aab7b777dad5b593910b904e74 Mon Sep 17 00:00:00 2001
From: SBrandeis <33657802+SBrandeis@users.noreply.github.com>
Date: Wed, 29 Jul 2026 17:37:23 +0200
Subject: [PATCH 4/4] keep the &str / &String patterns, backed by Literal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Python bindings search with a `&String` pattern — `NormalizedString.replace`
and `.split` take a plain `str` — so dropping these impls broke every job that
builds the bindings. Reinstating them through `Literal` keeps the public API and
still drops the regex engine from the literal path: they used to escape the
string and compile a regex on every call.

An empty pattern now covers the input by byte length rather than character
count, like every other impl (the old count sliced mid-character).
---
 tokenizers/tk-encode/src/tokenizer/pattern.rs | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tokenizers/tk-encode/src/tokenizer/pattern.rs b/tokenizers/tk-encode/src/tokenizer/pattern.rs
index 4a1f6054fd..5a147f5f4a 100644
--- a/tokenizers/tk-encode/src/tokenizer/pattern.rs
+++ b/tokenizers/tk-encode/src/tokenizer/pattern.rs
@@ -66,6 +66,25 @@ impl Pattern for &Literal {
     }
 }
 
+/// A plain string is one [`Literal`], built here because the pattern only lives for this call. Prefer
+/// keeping a [`Literal`] around when the same pattern is searched for repeatedly.
+impl Pattern for &str {
+    fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
+        match Literal::new(self.as_bytes()) {
+            Ok(literal) => (&literal).find_matches(inside),
+            // An empty pattern would match everywhere, so it matches nowhere instead.
+            Err(_) => Ok(vec![((0, inside.len()), false)]),
+        }
+    }
+}
+
+impl Pattern for &String {
+    fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
+        let s: &str = self;
+        s.find_matches(inside)
+    }
+}
+
 impl Pattern for &SysRegex {
     fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
         if inside.is_empty() {
@@ -198,6 +217,21 @@ mod tests {
         assert!(Literal::new(b"").is_err());
     }
 
+    #[test]
+    fn str() {
+        do_test!("aba", "a" => vec![((0, 1), true), ((1, 2), false), ((2, 3), true)]);
+        do_test!("aabbab", "ab" =>
+            vec![((0, 1), false), ((1, 3), true), ((3, 4), false), ((4, 6), true)]
+        );
+        do_test!("aaa", "b" => vec![((0, 3), false)]);
+        do_test!("", "" => vec![((0, 0), false)]);
+        // An empty pattern would match everywhere, so it matches nowhere instead.
+        do_test!("aaa", "" => vec![((0, 3), false)]);
+
+        let owned = String::from("ab");
+        do_test!("aabbb", &owned => vec![((0, 1), false), ((1, 3), true), ((3, 5), false)]);
+    }
+
     #[test]
     fn functions() {
         let is_b = |c| c == 'b';