5D tokenizer with multiverse time travel

Author: kg583

tests/tokenization.py

@@ -135,3 +135,9 @@ def test_backslash(self):
 
         self.assertEqual(TIProgram.encode(r'TI(Disp "A\ and B")TI'), TIProgram.encode(r'TI(Disp "\x41\x40\x42")TI'))
         self.assertEqual(TIProgram.encode(r'TI(Send("A\ and \B"))TI'), TIProgram.encode(r'TI(Send("A and B"))TI'))
+
+    def test_equations(self):
+        self.assertEqual(TIProgram.encode(r'"sin(X)"->{Y1}'), b'\x2A\xc2X\x11\x2A\x04\x5E\x10')
+        self.assertEqual(TIProgram.encode(r'String>Equ("sin(X)",{r1}'), b'\xBB\x56\x2A\xc2X\x11\x2A\x2B\x5E\x40')
+        self.assertEqual(TIProgram.encode(r'"sin(X)->{X2T}'), TIProgram.encode(r'"\sin(X)->{X2T}'))
+        self.assertNotEqual(TIProgram.encode(r'"sin(X)->|u')[:-2], TIProgram.encode(r'"sin(X)->UU')[:-2])

tivars/tokenizer/encoder.py

@@ -11,6 +11,7 @@
 from tivars.models import *
 from tivars.token import *
 from tivars.trie import *
+from tivars.util import *
 from .state import *
 
 
@@ -70,38 +71,46 @@ def tokenize(string: str, *, trie: TITokenTrie = None, mode: str = None, normali
     trie = trie or TI_84PCE.tokens.tries[None]
     mode = mode or "smart"
 
-    tokens = []
-    index = 0
-
     match mode:
         case "max":
-            stack = [MaxMode()]
-            
+            steps = [([], string, 0, [MaxMode()])]
+
         case "min" | "string":
-            stack = [MinMode()]
+            steps = [([], string, 0, [MinMode()])]
 
         case "smart":
-            stack = [SmartMode()]
+            steps = [([], string, 0, [SmartMode()])]
 
         case _:
             raise ValueError(f"unrecognized tokenization mode: '{mode}'")
 
-    while string:
+    while steps:
+        tokens, string, index, stack = steps.pop(0)
+
         try:
-            token, remainder, contexts = stack.pop().munch(string, trie)
-            stack += contexts
+            state = stack.pop()
+            if not string:
+                if state.accept:
+                    return tokens
 
-        except ValueError:
-            raise ValueError(f"could not tokenize input at position {index}: '{string[:12]}'")
+                else:
+                    continue
+
+            if isinstance(state, IllegalState):
+                continue
 
         except IndexError:
-            raise ValueError(f"stack consumed at position {index}: '{string[:12]}'")
+            raise ValueError(f"stack consumed at position {index}: '{trim_string(string, 12)}'")
+
+        try:
+            token, remainder, timelines = state.munch(string, trie)
+            for contexts in timelines:
+                steps.append((tokens + [token], remainder, index + len(string) - len(remainder), stack + contexts))
 
-        tokens.append(token)
-        index += len(string) - len(remainder)
-        string = remainder
+        except (IndexError, ValueError):
+            raise ValueError(f"failed to tokenize input at position {index}: '{trim_string(string, 12)}'")
 
-    return tokens
+    raise ValueError(f"all tokenization attempts failed; last segment was '{trim_string(string, 12)}'")
 
 
 def unparse(tokens: Sequence[TIToken]) -> bytes:

tivars/tokenizer/state.py

@@ -1,5 +1,5 @@
 """
-Encoder states
+NFA implementation for context-aware tokenization with lookahead
 """
 
 
@@ -16,26 +16,29 @@ class TokenizerState:
     Each state represents some encoding context which affects tokenization.
     """
 
-    mode: int = 0
-    """
-    Whether to munch maximally (``0``) or minimally (``-1``)
-    """
-
     max_length: int = None
     """
     The maximum number of tokens to emit before leaving this state
     """
 
-    def __init__(self, length: int = 0):
+    def __init__(self, mode: int, accept: bool = True, length: int = 0):
+        """
+        :param mode: Whether to munch maximally (``0``) or minimally (``-1``)
+        :param accept: Whether this state can end a timeline (defaults to ``True``)
+        :param length: The current length of the input this state is going to process (defaults to ``0``)
+        """
+
+        self.mode = mode
+        self.accept = accept
         self.length = length
 
-    def munch(self, string: str, trie: TITokenTrie) -> tuple[TIToken, str, list['TokenizerState']]:
+    def munch(self, string: str, trie: TITokenTrie) -> tuple[TIToken, str, list[list['TokenizerState']]]:
         """
-        Munch the input string and determine the resulting token, encoder state, and remainder of the string
+        Munch the input string and determine the resulting token, tokenizer timelines, and remainder of the string
 
         :param string: The text string to tokenize
         :param trie: The `TokenTrie` object to use for tokenization
-        :return: A tuple of the output `Token`, the remainder of ``string``, and a list of states to add to the stack
+        :return: A tuple of the output `Token`, the remainder of ``string``, and a list of timelines
         """
 
         # Is this a byte literal?
@@ -44,7 +47,7 @@ def munch(self, string: str, trie: TITokenTrie) -> tuple[TIToken, str, list['Tok
             string, remainder = string[:length], string[length:]
             token = IllegalToken(bytes.fromhex(string.lstrip(r"\ux")))
 
-            return token, remainder, self.next(token)
+            return token, remainder, self.next(token, remainder)
 
         # Is this a var prefix?
         for leading_byte, prefix in TIToken.var_prefixes.items():
@@ -53,95 +56,106 @@ def munch(self, string: str, trie: TITokenTrie) -> tuple[TIToken, str, list['Tok
                 string, remainder = string[:length], string[length:]
                 token = IllegalToken(bytes([leading_byte, int(string[-2:], 16)]))
 
-                return token, remainder, self.next(token)
+                return token, remainder, self.next(token, remainder)
 
         # Is there a token separator?
         if string.startswith(("␟", " ", "‌")):
             string = string[1:]
 
         # Is there a backslash?
         if string.startswith("\\"):
-            string = string[1:]
-            self.mode = 0
+            tokens = trie.match(string[1:])
+            token, remainder = tokens[0]
 
-        tokens = trie.match(string)
-        if not tokens:
-            raise ValueError("no tokenization options exist")
+        else:
+            tokens = trie.match(string)
 
-        # Is this a glyph?
-        if string[0] in punctuation and len(tokens) > 1:
-            tokens.pop()
+            # Is this a glyph?
+            if string[0] in punctuation and len(tokens) > 1:
+                tokens.pop()
 
-        token, remainder = tokens[self.mode]
+            token, remainder = tokens[self.mode]
 
         # Are we out of tokens?
         if self.length == self.max_length:
-            return token, remainder, []
+            return token, remainder, [[]]
 
-        return token, remainder, self.next(token)
+        return token, remainder, self.next(token, remainder)
 
-    def next(self, token: TIToken) -> list['TokenizerState']:
+    def next(self, token: TIToken, remainder: str) -> list[list['TokenizerState']]:
         """
-        Determines the next tokenizer state given a token
+        Determines the next tokenizer timelines given a token
 
         The current state is popped from the stack, and the states returned by this method are pushed.
 
-        If the list of returned states is...
-            - empty, then the tokenizer is exiting the current state.
-            - length one, then the tokenizer's current state is being replaced by a new state.
-            - length two, then the tokenizer is entering a new state, able to exit back to this one.
+        1. The current state is popped from the stack.
+        2. All possible timelines are determined, each a list of states.
+        3. For each separate timeline, those states are added its stack.
+
+        If a list of states in a timeline is...
+            - empty, then the timeline is exiting the current state.
+            - length one, then the timeline's current state is being replaced by a new state.
+            - length two, then the timeline is entering a new state, able to exit back to this one.
 
         :param token: The current token
-        :return: A list of tokenizer states to add to the stack
+        :param remainder: The remaining string content to tokenize
+        :return: A list of timelines (each a list of states)
         """
 
-        return [type(self)(self.length + 1)]
+        return [[type(self)(self.mode, self.accept, self.length + 1)]]
+
+
+class IllegalState(TokenizerState):
+    """
+    Tokenizer state which indicates its timeline must be pruned
+    """
 
 
 class MaxMode(TokenizerState):
     """
     Maximal munching mode
     """
 
-    mode = 0
+    def __init__(self, mode: int = 0, accept: bool = True, length: int = 0):
+        super().__init__(mode, accept, length)
 
 
 class MinMode(TokenizerState):
     """
     Minimal munching mode
     """
 
-    mode = -1
+    def __init__(self, mode: int = -1, accept: bool = True, length: int = 0):
+        super().__init__(mode, accept, length)
 
 
 class Line(TokenizerState):
     """
     State which is always exited after a line break or STO
     """
 
-    def next(self, token: TIToken) -> list[TokenizerState]:
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
         match token.bits:
+                 # STO (→)  Line break
             case b'\x04' | b'\x3F':
-                return []
+                return [[]]
 
             case _:
-                return super().next(token)
+                return super().next(token, remainder)
 
 
-class Name(Line):
+class Name(MinMode, Line):
     """
     Valid var identifiers
     """
 
-    mode = -1
-
-    def next(self, token: TIToken) -> list[TokenizerState]:
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
         #  Digits                              Uppercase letters (and theta)
         if b'\x30' <= token.bits <= b'\x39' or b'\x41' <= token.bits <= b'\x5B':
-            return super().next(token)
+            return super().next(token, remainder)
 
         else:
-            return []
+            return [[]]
 
 
 class ListName(Name):
@@ -165,72 +179,90 @@ class String(Line):
     Strings
     """
 
-    mode = -1
-
-    def next(self, token: TIToken) -> list[TokenizerState]:
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
         match token.bits:
+            case b'\x04':
+                return [[StringTarget(self.mode, self.accept)]]
+
             case b'\x2A':
-                return []
+                return [[StringSto(self.mode, self.accept)]]
 
             case _:
-                return super().next(token)
+                return super().next(token, remainder)
 
 
-class MaxString(String):
+class StringStart(Line):
     """
-    Maximally munched string
+    Opening quote of a string
     """
 
-    mode = 0
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
+        match token.bits:
+            case b'\x2A':
+                return [[String(self.mode, self.accept)]]
+
+            case _:
+                return [[]]
 
 
-class MaxStart(Line):
+class StringSto(Line):
     """
-    State to initialize `MaxString`
-
-    If any token besides ``"`` is encountered, this state is immediately exited to avoid cluttering the stack.
+    STO immediately following a string
     """
 
-    mode = 0
-
-    def next(self, token: TIToken) -> list[TokenizerState]:
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
         match token.bits:
-            case b'\x2A':
-                return [MaxString()]
+            case b'\x04':
+                return [[StringTarget(self.mode, self.accept)]]
+
+            case _:
+                return [[]] if self.accept else [[IllegalState(0)]]
+
+
+class StringTarget(Line):
+    """
+    STO target of a string
+    """
+
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
+        match self.mode, token.bits.startswith(b'\x5E'), self.accept:
+            case (0, True, _) | (0, _, True) | (-1, False, _):
+                return [[]]
 
             case _:
-                return []
+                return [[IllegalState(0)]]
 
 
 class SmartMode(TokenizerState):
     """
     Smart tokenization mode
     """
 
-    mode = 0
+    def __init__(self, mode: int = 0, accept: bool = True, length: int = 0):
+        super().__init__(mode, accept, length)
 
-    def next(self, token: TIToken) -> list[TokenizerState]:
+    def next(self, token: TIToken, remainder: str) -> list[list[TokenizerState]]:
         match token.bits:
             #    "
             case b'\x2A':
-                return [self, String()]
+                return [[self, String(0, False)], [self, String(-1)]]
 
             #    prgm
             case b'\x5F':
-                return [self, ProgramName()]
+                return [[self, ProgramName()]]
 
             #    Send(     String>Equ(
             case b'\xE7' | b'\xBB\x56':
-                return [self, MaxStart()]
+                return [[self, StringStart(0)]]
 
             #    |L
             case b'\xEB':
-                return [self, ListName()]
+                return [[self, ListName()]]
 
             case _:
-                return super().next(token)
+                return super().next(token, remainder)
 
 
-__all__ = ["TokenizerState", "MaxMode", "MinMode", "SmartMode",
+__all__ = ["TokenizerState", "IllegalState", "MaxMode", "MinMode", "SmartMode",
            "Line", "Name", "ListName", "ProgramName",
-           "String", "MaxString", "MaxStart"]
+           "String", "StringStart", "StringSto", "StringTarget"]