Merge pull request #129 from FSoft-AI4Code/fix/tiktoken-special-tokens

fix: count literal special tokens like <|endoftext|> as plain text

Author: anhnh2002

codewiki/src/be/utils.py

@@ -56,7 +56,9 @@ def count_tokens(text: str) -> int:
     """
     Count the number of tokens in a text.
     """
-    length = len(enc.encode(text))
+    # source code may contain literal special tokens such as "<|endoftext|>"
+    # (e.g. CLIP tokenizers); count them as plain text instead of raising
+    length = len(enc.encode(text, disallowed_special=()))
     # logger.debug(f"Number of tokens: {length}")
     return length