diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index b722b494..78c6b943 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -44,7 +44,7 @@ jobs: - name: Checkout source code uses: actions/checkout@v6 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} - id: check_build_trigger name: Check build trigger run: bash build_tools/github/check_build_trigger.sh @@ -56,13 +56,11 @@ jobs: needs: check_build_trigger if: needs.check_build_trigger.outputs.build strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false + fail-fast: false # If false, all jobs continue even if one fails matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: ["3.14"] # Check with PyO3's maximum supported version - env: - CIBW_BUILD: "" # blank, let cibuildwheel build all supported platforms + # cibuildwheel will manage multiple Python versions internally steps: - name: Checkout source code @@ -92,17 +90,19 @@ jobs: package-dir: nlpo3-python output-dir: wheelhouse env: - CIBW_BUILD_VERBOSITY: 1 # See CIBW_BUILD, CIBW_SKIP, CIBW_ARCHS and other build selectors at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip - CIBW_SKIP: "*-musllinux_i686" + CIBW_ENABLE: "graalpy pypy" # Enable GraalPy and PyPy builds + CIBW_BUILD: "cp* gp* pp*" # Build for CPython, GraalPy and PyPy + CIBW_BUILD_VERBOSITY: 1 + CIBW_SKIP: "*-musllinux_i686 gp*-win_*" CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_ENVIRONMENT_MACOS: | MACOSX_DEPLOYMENT_TARGET=10.12 PATH="$HOME/.cargo/bin:$PATH" CC=/usr/bin/clang CXX=/usr/bin/clang++ - CIBW_ARCHS_LINUX: "auto" + CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ENVIRONMENT_LINUX: PATH="$HOME/.cargo/bin:$PATH" CIBW_BEFORE_BUILD_LINUX: | pip install --upgrade setuptools-rust @@ -112,13 +112,20 @@ jobs: # and the container does not have Rust. CIBW_ARCHS_WINDOWS: "AMD64 x86" + - name: List wheel files + run: | + echo "Files to upload (binary wheels):" + find ./wheelhouse -maxdepth 1 -type f -name '*.whl' -print | sort || true + echo "Count: $(find ./wheelhouse -maxdepth 1 -type f -name '*.whl' -print | wc -l || true)" + shell: bash - name: Store artifacts uses: actions/upload-artifact@v6 with: path: ./wheelhouse/*.whl - name: wheelhouse-${{ matrix.os }}-${{ matrix.python-version }} + name: ${{ github.job }}-wheelhouse-${{ matrix.os }}-${{ github.run_id }} if-no-files-found: warn overwrite: true + retention-days: 1 build_sdist: name: Build source distribution @@ -131,16 +138,25 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: "3.14" # Check with PyO3's maximum supported version + python-version: "3.14" # Ideally matches with the python-version in build_wheels cache: "pip" - name: Build source distribution run: | cd nlpo3-python bash ../build_tools/github/build_source.sh + - name: List sdist files + run: | + echo "Files to upload (sdist):" + find nlpo3-python/dist -maxdepth 1 -type f -name '*.tar.gz' -print | sort || true + echo "Count: $(find nlpo3-python/dist -maxdepth 1 -type f -name '*.tar.gz' -print | wc -l || true)" + shell: bash - name: Store artifacts uses: actions/upload-artifact@v6 with: path: nlpo3-python/dist/*.tar.gz + name: sdist + overwrite: true + retention-days: 1 publish_pypi: name: Publish to PyPI @@ -151,11 +167,11 @@ jobs: # Alternatively, upload to PyPI on every tag starting with 'v': #if: github.event_name == 'push' && startsWith(github.event.ref, 'v') steps: - - name: Retrieve artifacts + - name: Retrieve all artifacts uses: actions/download-artifact@v6 with: - name: artifact path: dist + merge-multiple: true - name: Publish package uses: pypa/gh-action-pypi-publish@v1.13.0 with: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4b88d426..cfe6fd39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,11 +43,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -58,7 +58,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # â„šī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -72,4 +72,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/test-nlpo3-cli.yml b/.github/workflows/test-nlpo3-cli.yml index 808ee06f..e81a7f2b 100644 --- a/.github/workflows/test-nlpo3-cli.yml +++ b/.github/workflows/test-nlpo3-cli.yml @@ -33,28 +33,19 @@ jobs: uses: actions/checkout@v6 - name: Setup Rust toolchain - non-win32 - uses: actions-rs/toolchain@v1 - with: - override: true - profile: minimal - toolchain: stable - if: ${{ !startsWith(matrix.os, 'windows') || matrix.bitness != '32' }} + uses: actions-rust-lang/setup-rust-toolchain@v1 + if: "!startsWith(matrix.os, 'windows') || matrix.bitness != '32'" - name: Setup Rust toolchain - win32 - uses: actions-rs/toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@v1 with: - override: true - profile: minimal - toolchain: stable target: i686-pc-windows-msvc if: startsWith(matrix.os, 'windows') && matrix.bitness == '32' - - name: Setup Rust dependencies - uses: actions-rs/cargo@v1 - with: - command: check + - name: Test - non-win32 + run: cargo test + if: "!startsWith(matrix.os, 'windows') || matrix.bitness != '32'" - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test + - name: Test - win32 + run: cargo test --target i686-pc-windows-msvc + if: startsWith(matrix.os, 'windows') && matrix.bitness == '32' diff --git a/.github/workflows/test-nlpo3-python.yml b/.github/workflows/test-nlpo3-python.yml index a394c830..37b9ee3f 100644 --- a/.github/workflows/test-nlpo3-python.yml +++ b/.github/workflows/test-nlpo3-python.yml @@ -24,7 +24,7 @@ defaults: jobs: test: strategy: - fail-fast: false + fail-fast: true matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: ["3.14", "3.13", "3.12", "3.11", "3.10", "3.9"] diff --git a/README.md b/README.md index ba800c44..a58244b2 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,16 @@ SPDX-License-Identifier: Apache-2.0 [![Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg "Apache-2.0")](https://opensource.org/license/apache-2-0) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.14082448.svg)](https://doi.org/10.5281/zenodo.14082448) -A Thai natural language processing library written in Rust, -with optional Python and Node.js bindings. Formerly known as `oxidized-thainlp`. +A Thai natural language processing library written in Rust with optional +Python and Node.js bindings. Formerly known as `oxidized-thainlp`. -To use as a library in a Rust project: +Using in a Rust project ```shell cargo add nlpo3 ``` -To use as a library in a Python project: +Using in a Python project ```shell pip install nlpo3 @@ -75,26 +75,25 @@ See more at [nlpo3-python](./nlpo3-python/). [![crates.io](https://img.shields.io/crates/v/nlpo3.svg "crates.io")](https://crates.io/crates/nlpo3/) -#### Add to dependency +#### Add as a dependency -To use as a library in a Rust project: +To add `nlpo3` to your project's dependencies: ```shell cargo add nlpo3 ``` -It will add "nlpo3" to `Cargo.toml`: +This updates `Cargo.toml` with: ```toml [dependencies] -# ... nlpo3 = "1.4.0" ``` #### Example -Create a tokenizer using a dictionary from file, -then use it to tokenize a string (safe mode = true, and parallel mode = false): +Create a tokenizer from a dictionary file and use it to tokenize a string +(safe mode = true, parallel mode = false): ```rust use nlpo3::tokenizer::newmm::NewmmTokenizer; @@ -104,7 +103,7 @@ let tokenizer = NewmmTokenizer::new("path/to/dict.file"); let tokens = tokenizer.segment("ā¸Ģāš‰ā¸­ā¸‡ā¸Ēā¸Ąā¸¸ā¸”ā¸›ā¸Ŗā¸°ā¸Šā¸˛ā¸Šā¸™", true, false).unwrap(); ``` -Create a tokenizer using a dictionary from a vector of Strings: +Create a tokenizer from a vector of strings: ```rust let words = vec!["⏛⏞ā¸Ĩā¸´āš€ā¸Ąā¸™ā¸•āšŒ".to_string(), "⏄⏭⏙ā¸Ēā¸•ā¸´ā¸•ā¸´ā¸§ā¸Šā¸ąāšˆā¸™".to_string()]; diff --git a/nlpo3-nodejs/README.md b/nlpo3-nodejs/README.md index 7d7756c4..b0879c55 100644 --- a/nlpo3-nodejs/README.md +++ b/nlpo3-nodejs/README.md @@ -57,7 +57,8 @@ For now, copy the whole `nlpo3/` directory after build to your project. ### npm (experitmental) -npm is still experimental and may not work on all platforms. Please report issues at +npm is still experimental and may not work on all platforms. +Please report issues at ```shell npm i nlpo3 diff --git a/nlpo3-python/Cargo.toml b/nlpo3-python/Cargo.toml index d602c352..aab49ce9 100644 --- a/nlpo3-python/Cargo.toml +++ b/nlpo3-python/Cargo.toml @@ -10,7 +10,15 @@ authors = [ ] description = "Python binding for nlpO3 Thai language processing library" categories = ["text-processing"] -keywords = ["thai", "tokenizer", "nlp", "word-segmentation", "python"] +keywords = [ + "thai", + "tokenizer", + "tokenization", + "text-processing", + "nlp", + "word-segmentation", + "pythainlp", +] homepage = "https://github.com/PyThaiNLP/nlpo3/tree/main/nlpo3-python" repository = "https://github.com/PyThaiNLP/nlpo3/" documentation = "https://github.com/PyThaiNLP/nlpo3/blob/main/nlpo3-python/README.md" diff --git a/nlpo3-python/MANIFEST.in b/nlpo3-python/MANIFEST.in index e7267465..0591d05a 100644 --- a/nlpo3-python/MANIFEST.in +++ b/nlpo3-python/MANIFEST.in @@ -1,2 +1,13 @@ include LICENSE README.md pyproject.toml Cargo.toml recursive-include src * + +# Ensure the Python package files are included in the sdist +graft nlpo3 + +# Include PEP 561 marker and type stubs for compiled extension +include nlpo3/py.typed +recursive-include nlpo3 *.pyi + +# Exclude test notebooks and tests from the sdist to keep it small +prune tests +prune notebooks diff --git a/nlpo3-python/README.md b/nlpo3-python/README.md index a6ba01d8..d6516739 100644 --- a/nlpo3-python/README.md +++ b/nlpo3-python/README.md @@ -1,16 +1,17 @@ --- -SPDX-FileCopyrightText: 2024 PyThaiNLP Project +SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project SPDX-License-Identifier: Apache-2.0 --- # nlpO3 Python binding [![PyPI](https://img.shields.io/pypi/v/nlpo3.svg "PyPI")](https://pypi.python.org/pypi/nlpo3) -[![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg "Python 3.7")](https://www.python.org/downloads/) +[![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg "Python 3.9")](https://www.python.org/downloads/) [![Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg "Apache-2.0")](https://opensource.org/license/apache-2-0) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.14082448.svg)](https://doi.org/10.5281/zenodo.14082448) -Python binding for nlpO3, a Thai natural language processing library in Rust. +Python binding for nlpO3, +a Thai natural language processing library written in Rust. To install: @@ -43,34 +44,33 @@ pip install nlpo3 ## Use -Load file `path/to/dict.file` to memory -and assign a name `dict_name` to it. +Load a dictionary file and assign it a name (for example, `dict_name`). -Then tokenize a text with the `dict_name` dictionary: +Then tokenize text using the named dictionary: ```python from nlpo3 import load_dict, segment -load_dict("path/to/dict.file", "custom_dict") +load_dict("path/to/dict.file", "dict_name") segment("ā¸Ēā¸§ā¸ąā¸Ē⏔ā¸ĩā¸„ā¸Ŗā¸ąā¸š", "dict_name") ``` -it will return a list of strings: +The function returns a list of strings, for example: ```python ['ā¸Ēā¸§ā¸ąā¸Ē⏔ā¸ĩ', 'ā¸„ā¸Ŗā¸ąā¸š'] ``` -(result depends on words included in the dictionary) +The result depends on the words included in the dictionary. -Use multithread mode, also use the `dict_name` dictionary: +Use multithread mode using the `dict_name` dictionary: ```python segment("ā¸Ēā¸§ā¸ąā¸Ē⏔ā¸ĩā¸„ā¸Ŗā¸ąā¸š", dict_name="dict_name", parallel=True) ``` -Use safe mode to avoid long waiting time in some edge cases -for text with lots of ambiguous word boundaries: +Use safe mode to avoid long run times for inputs with many ambiguous +word boundaries: ```python segment("ā¸Ēā¸§ā¸ąā¸Ē⏔ā¸ĩā¸„ā¸Ŗā¸ąā¸š", dict_name="dict_name", safe=True) @@ -78,10 +78,9 @@ segment("ā¸Ēā¸§ā¸ąā¸Ē⏔ā¸ĩā¸„ā¸Ŗā¸ąā¸š", dict_name="dict_name", safe=True) ### Dictionary -- For the interest of library size, nlpO3 does not assume what dictionary the - user would like to use, and it does not come with a dictionary. -- A dictionary is needed for the dictionary-based word tokenizer. -- For tokenization dictionary, try +- To keep the library small, nlpO3 does not include a dictionary. + Users must provide a dictionary when using the dictionary-based tokenizer. +- For tokenization dictionaries, try - [words_th.txt][dict-pythainlp] from [PyThaiNLP][pythainlp] - ~62,000 words - CC0-1.0 @@ -143,89 +142,116 @@ See file [LICENSE](./LICENSE) for details. ## Binary wheels -A pre-built binary package is available from [PyPI][pypi] for these platforms: +Pre-built binary packages for CPython, GraalPy, and PyPy are available +on [PyPI][pypi] for the platforms listed below. +Versions with a "t" suffix indicate CPython with free threading. [pypi]: https://pypi.org/project/nlpo3/ -|Python|OS|Architecture|Has binary wheel?| -|-|-|-|-| -|3.14|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.13|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.12|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.11|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.10|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.9|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.8 (nlpO3 1.3.1)|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|3.7 (nlpO3 1.3.1)|Windows|x86|✅| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|❌| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -||musllinux|x86_64|✅| -|PyPy 3.10|Windows|x86|❌| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -|PyPy 3.9|Windows|x86|❌| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -|PyPy 3.8 (nlpO3 1.3.1)|Windows|x86|❌| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|✅| -||manylinux|x86_64|✅| -||manylinux|i686|✅| -|PyPy 3.7 (nlpO3 1.3.1)|Windows|x86|❌| -||Windows|AMD64|✅| -||macOS|x86_64|✅| -||macOS|arm64|❌| -||manylinux|x86_64|✅| -||manylinux|i686|✅| +| Python | OS | Architecture | Binary wheel | +| ------------ | --------- | ------------ | ------------- | +| 3.14 | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.14t | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.13 | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.12 | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.11 | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.10 | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.9 | Windows | x86 | ✓ | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| | musllinux | x86_64 | ✓ | +| 3.8 | Windows | x86 | ✓ (v1.3.1) | +| | | AMD64 | ✓ (v1.3.1) | +| | macOS | x86_64 | ✓ (v1.3.1) | +| | | arm64 | ✓ (v1.3.1) | +| | manylinux | x86_64 | ✓ (v1.3.1) | +| | | i686 | ✓ (v1.3.1) | +| | musllinux | x86_64 | ✓ (v1.3.1) | +| 3.7 | Windows | x86 | ✓ (v1.3.1) | +| | | AMD64 | ✓ (v1.3.1) | +| | macOS | x86_64 | ✓ (v1.3.1) | +| | | arm64 | | +| | manylinux | x86_64 | ✓ (v1.3.1) | +| | | i686 | ✓ (v1.3.1) | +| | musllinux | x86_64 | ✓ (v1.3.1) | +| GraalPy 3.12 | Windows | x86 | | +| | | AMD64 | | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | | +| GraalPy 3.11 | Windows | x86 | | +| | | AMD64 | | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | | +| PyPy 3.11 | Windows | x86 | | +| | | AMD64 | ✓ | +| | macOS | x86_64 | ✓ | +| | | arm64 | ✓ | +| | manylinux | x86_64 | ✓ | +| | | i686 | ✓ | +| PyPy 3.10 | Windows | x86 | | +| | | AMD64 | ✓ (v1.3.1) | +| | macOS | x86_64 | ✓ (v1.3.1) | +| | | arm64 | ✓ (v1.3.1) | +| | manylinux | x86_64 | ✓ (v1.3.1) | +| | | i686 | ✓ (v1.3.1) | +| PyPy 3.9 | Windows | x86 | | +| | | AMD64 | ✓ (v1.3.1) | +| | macOS | x86_64 | ✓ (v1.3.1) | +| | | arm64 | ✓ (v1.3.1) | +| | manylinux | x86_64 | ✓ (v1.3.1) | +| | | i686 | ✓ (v1.3.1) | +| PyPy 3.8 | Windows | x86 | | +| | | AMD64 | ✓ (v1.3.1) | +| | macOS | x86_64 | ✓ (v1.3.1) | +| | | arm64 | ✓ (v1.3.1) | +| | manylinux | x86_64 | ✓ (v1.3.1) | +| | | i686 | ✓ (v1.3.1) | +| PyPy 3.7 | Windows | x86 | | +| | | AMD64 | ✓ (v1.3.1) | +| | macOS | x86_64 | ✓ (v1.3.1) | +| | | arm64 | | +| | manylinux | x86_64 | ✓ (v1.3.1) | +| | | i686 | ✓ (v1.3.1) | diff --git a/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi b/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi new file mode 100644 index 00000000..bb6536b3 --- /dev/null +++ b/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +"""Type stubs for _nlpo3_python_backend Rust extension module.""" + +from typing import List, Tuple + +def load_dict(file_path: str, dict_name: str) -> Tuple[str, bool]: + """Load a dictionary file to a tokenizer. + + Load a dictionary file into an in-memory dictionary collection, + and assign dict_name to it. This function does not override an + existing dict name. + + Args: + file_path: Path to a dictionary file (one word per line) + dict_name: A unique dictionary name, used for reference + + Returns: + A tuple of (human_readable_result_str, success_bool) + """ + ... + +def segment( + text: str, + dict_name: str, + safe: bool = False, + parallel: bool = False, +) -> List[str]: + """Break text into tokens using newmm algorithm. + + Args: + text: Input text to segment + dict_name: Dictionary name, as assigned in load_dict() + safe: Use safe mode to avoid long waiting time in a text with + lots of ambiguous word boundaries (default: False) + parallel: Use multithread mode (default: False) + + Returns: + List of tokens + + Raises: + RuntimeError: If dictionary name does not exist + """ + ... diff --git a/nlpo3-python/nlpo3/py.typed b/nlpo3-python/nlpo3/py.typed new file mode 100644 index 00000000..2dfd717f --- /dev/null +++ b/nlpo3-python/nlpo3/py.typed @@ -0,0 +1 @@ +# PEP 561 marker file for typed package diff --git a/nlpo3-python/pyproject.toml b/nlpo3-python/pyproject.toml index 60071cae..86fff470 100644 --- a/nlpo3-python/pyproject.toml +++ b/nlpo3-python/pyproject.toml @@ -39,6 +39,7 @@ classifiers = [ "Natural Language :: Thai", "Topic :: Text Processing :: Linguistic", "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", ] [project.urls] diff --git a/src/NOTE_ON_STRING.md b/src/NOTE_ON_STRING.md index d45c7a60..00686a4b 100644 --- a/src/NOTE_ON_STRING.md +++ b/src/NOTE_ON_STRING.md @@ -1,4 +1,5 @@ --- +SPDX-FileContributor: Thanathip Suntorntip Gorlph SPDX-FileCopyrightText: 2024 PyThaiNLP Project SPDX-License-Identifier: Apache-2.0 ---