Update docs

Author: kg583

tivars/tokenizer/decoder.py

@@ -16,9 +16,9 @@ def parse(bytestream: bytes, *, tokens: TITokens = None) -> list[TIToken]:
     Parses a byte stream into a list of `TIToken` objects
 
     Each token is represented using one of three different representations formats, dictated by ``mode``:
-        - ``display``: Represents the tokens with Unicode characters matching the calculator's display
-        - ``accessible``: Represents the tokens with ASCII-only equivalents, often requiring multi-character glyphs
-        - ``ti_ascii``: Represents the tokens with their internal font indices (returns a ``bytes`` object)
+    - ``display``: Represents the tokens with Unicode characters matching the calculator's display
+    - ``accessible``: Represents the tokens with ASCII-only equivalents, often requiring multi-character glyphs
+    - ``ti_ascii``: Represents the tokens with their internal font indices (returns a ``bytes`` object)
 
     :param bytestream: The token bytes to parse
     :param tokens: The `TITokens` object to use for decoding (defaults to the TI-84+CE tokens)

tivars/tokenizer/encoder.py

@@ -31,34 +31,34 @@ def normalize(string: str):
 
 
 def tokenize(string: str, *, trie: TITokenTrie = None, mode: str = None, normalize: bool = True) -> list[TIToken]:
-    r"""
+    """
     Tokenizes a string of tokens represented as text into a list of `TIToken` objects
 
     Tokenization is performed using one of three procedures, dictated by ``mode``:
-        - ``max``: Always munch maximally, i.e. consume the most input possible to produce a token
-        - ``min``: Always munch minimally, i.e. consume the least input possible, which is often single characters
-        - ``smart``: Munch maximally or minimally depending on context
+    - ``max``: Always munch maximally, i.e. consume the most input possible to produce a token
+    - ``min``: Always munch minimally, i.e. consume the least input possible, which is often single characters
+    - ``smart``: Munch maximally or minimally depending on context
 
     The ``smart`` tokenization mode uses the following contexts, munching maximally otherwise:
-        - Strings: munch minimally, except when interpolating using ``Send(`` or storing to an equation
-        - Program names: munch minimally up to 8 tokens
-        - List names: munch minimally up to 5 tokens
+    - Strings: munch minimally, except when interpolating using ``Send(`` or storing to an equation
+    - Program names: munch minimally up to 8 tokens
+    - List names: munch minimally up to 5 tokens
 
     In all modes:
-        - Standard glyphs can be used for substituting Unicode symbols
-        - `\xXX` and `\uUUUU` output the denoted bytes exactly, regardless of validity
-        - `\ABCD` maximally munches `ABCD`, regardless of surrounding context
-        - Certain unprintable characters act as a hard separator for tokens
-            - U+001F (unit separator): `␟`
-            - U+200A (hair space): ` `
-            - U+200C (zero width non-joiner): `‌`
+    - Standard glyphs can be used for substituting Unicode symbols
+    - `\\xXX` and `\\uUUUU` output the denoted bytes exactly, regardless of validity
+    - `\\ABCD` maximally munches `ABCD`, regardless of surrounding context
+    - Certain unprintable characters act as a hard separator for tokens
+        - U+001F (unit separator): `␟`
+        - U+200A (hair space): ` `
+        - U+200C (zero width non-joiner): `‌`
 
     For reference, here are the tokenization modes utilized by popular IDEs and other software:
-        - SourceCoder: ``max``
-        - TokenIDE: ``max``
-        - TI Connect CE: ¯\_(ツ)_/¯
-        - TI-Planet Project Builder: ``smart``
-        - tivars_lib_cpp: ``smart``
+    - SourceCoder: ``max``
+    - TokenIDE: ``max``
+    - TI Connect CE: ¯\\_(ツ)_/¯
+    - TI-Planet Project Builder: ``smart``
+    - tivars_lib_cpp: ``smart``
 
     :param string: The text string to encode
     :param trie: The `TokenTrie` object to use for tokenization (defaults to the TI-84+CE trie)

tivars/tokenizer/state.py

@@ -86,16 +86,14 @@ def next(self, token: TIToken, remainder: str) -> list[list['TokenizerState']]:
         """
         Determines the next tokenizer timelines given a token
 
-        The current state is popped from the stack, and the states returned by this method are pushed.
-
         1. The current state is popped from the stack.
         2. All possible timelines are determined, each a list of states.
         3. For each separate timeline, those states are added its stack.
 
         If a list of states in a timeline is...
-            - empty, then the timeline is exiting the current state.
-            - length one, then the timeline's current state is being replaced by a new state.
-            - length two, then the timeline is entering a new state, able to exit back to this one.
+        - empty, then the timeline is exiting the current state.
+        - length one, then the timeline's current state is being replaced by a new state.
+        - length two, then the timeline is entering a new state, able to exit back to this one.
 
         :param token: The current token
         :param remainder: The remaining string content to tokenize