There was an error while loading. Please reload this page.
1 parent 4fb4af1 commit fa1bdddCopy full SHA for fa1bddd
1 file changed
model2vec/tokenizer/tokenizer.py
@@ -46,7 +46,7 @@ def clean_and_create_vocabulary(
46
added_tokens_to_add: list[str] = []
47
seen_added = set()
48
for token in vocabulary_to_add:
49
- preprocessed = preprocessor.preprocess(token)
+ preprocessed = preprocessor.preprocess(token, had_initial_subword_prefix=True)
50
if len(preprocessed) < 1:
51
logger.warning(f"Token '{token}' was empty after preprocessing.")
52
n_empty += 1
0 commit comments