V0.2.0 has been released

Author: sefineh-ai

.gitignore

@@ -44,7 +44,9 @@ env/
 htmlcov/
 .tox/
 .mypy_cache/
-
+data_crawler
+scripts/
+amh_bpe_sample.json
 # Local config
 *.env
 cleaned_amharic.txt

MANIFEST.in

@@ -1,7 +1,5 @@
 include README.md
 include LICENSE
 include pyproject.toml
-include amharic_tokenizer/amh_bpe.json
-recursive-include amharic_tokenizer *.pyx *.pxd *.pxi *.py *.so *.cpp
-recursive-include examples *.py
+recursive-include amharic_tokenizer *.pyx *.pxd *.pxi *.py *.so *.cpp *.json
 recursive-include tests *.py

README.md

@@ -84,7 +84,7 @@ amh-tokenizer train cleaned_amharic.txt amh_bpe --num-merges 50000 --verbose --l
 from amharic_tokenizer import AmharicTokenizer
 
 # Load a trained model
-tok = AmharicTokenizer.load("amh_bpe")  # use full path if needed
+tok = AmharicTokenizer.load("amh_bpe_v0.2.0")
 
 text = "ኢትዮጵያ ጥሩ ናት።"