feat(atomsplit): offload literal patterns to atomsplit

tokenizers/atomsplit/Cargo.toml

@@ -25,7 +25,7 @@ name = "atomsplit"
 path = "src/lib.rs"
 
 [dependencies]
-memchr = "2.8.2" # SIMD single-byte search; used only in CharDelimiterSplit (1.4–23× vs scalar).
+memchr = "2.8.2" # SIMD search: used for single-byte  search in CharDelimiterSplit (1.4–23× vs scalar), or multi-byte string pattern search in `literal`.
 # NOTE: classify tables live in src/atom_tables.rs (committed, generated). Regenerate after any atom
 # scheme change with `cargo run -p bitmap_gen`. No build script / build-dep — atomsplit builds clean.
 # onig is C (Oniguruma) and fancy-regex pulls it in transitively for benches only — neither

tokenizers/atomsplit/src/lib.rs

@@ -7,6 +7,10 @@
 //! Mistral's tekken are exposed as the [`fsm::fsm_o200k`] / [`fsm::fsm_tekken`] functions rather than
 //! recipe structs).
 //!
+//! For pre-tokenizers that split on single characters (such as the Metaspace `▁` delimiter),
+//! we skip the atom classification pass entirely and search for the raw  bytes instead.
+//! See [`literal`].
+//!
 //! Design: every fsm is *no-push* — it writes spans into a caller-preallocated `&mut [fsm::Span]`
 //! (length ≥ `text.len()`) and returns the token count; there is no `Vec`/allocation on the hot path.
 //!
@@ -17,6 +21,7 @@
 mod atom_tables;
 pub mod classify;
 pub mod fsm;
+pub mod literal;
 pub mod regexes;
 #[cfg(target_arch = "x86_64")]
 mod simd_avx_classify;

tokenizers/atomsplit/src/literal.rs

@@ -0,0 +1,56 @@
+//! Searching for a literal string, for pre-tokenizers that cut on one exact character.
+//!
+//! The rest of the crate works off atom tags: one SIMD pass gives every character a class, and the
+//! FSMs cut where the class changes ([`crate::fsm`]).
+//!
+//! The atom classification is unnecessary for pre-tokenizers splitting on an exact character or literal string:
+//! We can use a simpler byte search looking at 16 bytes at a time.
+
+use memchr::memmem;
+use std::fmt;
+
+/// The pattern handed to [`Literal::new`] was empty.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EmptyPattern;
+
+impl fmt::Display for EmptyPattern {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("an empty pattern matches everywhere")
+    }
+}
+
+impl std::error::Error for EmptyPattern {}
+
+/// A literal string to split on.
+///
+/// The finder is boxed because it is large — a few hundred bytes of prefilter state on x86_64 — and
+/// callers store it inside enums whose other variants are tiny.
+#[derive(Debug, Clone)]
+pub struct Literal {
+    finder: Box<memmem::Finder<'static>>,
+}
+
+impl Literal {
+    /// # Errors
+    /// If `pattern` is empty, which would match everywhere.
+    pub fn new(pattern: &[u8]) -> Result<Self, EmptyPattern> {
+        if pattern.is_empty() {
+            return Err(EmptyPattern);
+        }
+        Ok(Self {
+            finder: Box::new(memmem::Finder::new(pattern).into_owned()),
+        })
+    }
+
+    /// The string being searched for.
+    #[must_use]
+    pub fn pattern(&self) -> &[u8] {
+        self.finder.needle()
+    }
+
+    /// Byte offset of every match, left to right. Matches never overlap, so `"aa"` is found once in
+    /// `"aaa"` — the same matches a regex engine would report.
+    pub fn matches<'t>(&'t self, text: &'t [u8]) -> impl Iterator<Item = usize> + 't {
+        self.finder.find_iter(text)
+    }
+}

tokenizers/atomsplit/tests/literal.rs

@@ -0,0 +1,50 @@
+//! Tests for the literal search. Kept out of `src/` so the core stays production-only.
+use atomsplit::literal::{EmptyPattern, Literal};
+
+#[test]
+fn finds_every_match_and_nothing_else() {
+    let literal = Literal::new(b"-").unwrap();
+    assert_eq!(literal.matches(b"a-b--c").collect::<Vec<_>>(), [1, 3, 4]);
+    assert_eq!(literal.matches(b"none here").count(), 0);
+    assert_eq!(literal.matches(b"").count(), 0);
+    assert_eq!(literal.pattern(), b"-");
+}
+
+#[test]
+fn a_multi_byte_pattern_only_matches_whole() {
+    // `▁` is U+2581 = E2 96 81. The other characters here share its first byte and nothing else, so a
+    // search for that byte alone would report all of them.
+    let literal = Literal::new("▁".as_bytes()).unwrap();
+    let text = "a—b“c…d▁e";
+    assert_eq!(
+        literal.matches(text.as_bytes()).collect::<Vec<_>>(),
+        [text.find('▁').unwrap()]
+    );
+}
+
+#[test]
+fn matches_do_not_overlap() {
+    let literal = Literal::new(b"aa").unwrap();
+    assert_eq!(literal.matches(b"aaa").collect::<Vec<_>>(), [0]);
+    assert_eq!(literal.matches(b"aaaa").collect::<Vec<_>>(), [0, 2]);
+}
+
+/// A `Literal` is stored inline in the normalizer and decoder enums, where every other variant is a
+/// handful of bytes. `memmem::Finder` itself is a few hundred, so it has to stay behind a pointer.
+#[test]
+fn a_literal_is_pointer_sized() {
+    assert_eq!(
+        size_of::<Literal>(),
+        size_of::<*const u8>(),
+        "a Literal must not carry its finder inline"
+    );
+}
+
+#[test]
+fn an_empty_pattern_is_rejected() {
+    assert_eq!(Literal::new(b"").unwrap_err(), EmptyPattern);
+    assert_eq!(
+        EmptyPattern.to_string(),
+        "an empty pattern matches everywhere"
+    );
+}

tokenizers/tk-encode/Cargo.toml

@@ -73,10 +73,11 @@ pcre2 = { version = "0.2", optional = true }
 logos = { version = "0.15", optional = true } # compile-time DFA lexer reference (pure Rust)
 
 [features]
-# `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a `Split` pre-tokenizer with an
-# arbitrary (non-GPT) regex or the `Replace` normalizer — the atomsplit-native pre-tokenizers (GPT-2,
-# cl100k, deepseek, the class family, char-delimiter) need no backend. Without it a stub compiles and
-# those arbitrary-regex paths error at load. Enable with `--features fancy-regex`.
+# `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a *regex* pattern in a `Split`
+# pre-tokenizer or a `Replace` normalizer — the atomsplit-native pre-tokenizers (GPT-2, cl100k,
+# deepseek, the class family, char-delimiter) need no backend, and a plain string pattern is searched
+# for directly. Without it a stub compiles and those regex paths error at load. Enable with
+# `--features fancy-regex`.
 default = ["progressbar"]
 progressbar = ["indicatif"]
 http = ["hf-hub"]