test: add and refactor tests to have a full coverage (#207)

* Reorganized tests into unit/, integration/, property/, and benchmarks/ suites with collection markers.

* Added extensive new unit + property-based coverage for core modules (server, download, safe_zip, config, match, language tag, CLI).

* Added test tooling/dependencies (hypothesis, pytest-benchmark) and minor CI/dev-experience tweaks (markers, --basetemp, clean target).

* Added new makefile targets to trigger specific class tests.

* Updated python 3.15 version in CI.

Author: mdevolde

.github/workflows/test.yml

@@ -35,7 +35,7 @@ jobs:
           - os: ubuntu-26.04
             python-version: "3.14"
           - os: ubuntu-26.04
-            python-version: "3.15.0-beta.2"
+            python-version: "3.15.0-beta.3"
           - os: macos-26
             python-version: "3.14"
           - os: windows-2025

.gitignore

@@ -204,6 +204,9 @@ cython_debug/
 # Ruff stuff:
 .ruff_cache/
 
+# Pytest tmp_path base directory (project-relative to avoid Windows temp permission issues)
+.pytest_tmp/
+
 # PyPI configuration file
 .pypirc
 

Makefile

@@ -1,12 +1,12 @@
-.PHONY: default install format fix ruff-check mypy-check check test doc publish
+.PHONY: default install format fix ruff-check mypy-check check benchmark-test integration-test property-test unit-test test doc publish clean
 
 UV := $(shell command -v uv 2>/dev/null || true)
 ifeq ($(UV),)
 $(warning uv not found. Install uv (curl -LsSf https://astral.sh/uv/install.sh | sh) to use Makefile targets)
 endif
 
 default:
-	@echo "Usage: make [install|format|fix|ruff-check|mypy-check|check|test|doc|publish]"
+	@echo "Usage: make [install|format|fix|ruff-check|mypy-check|check|benchmark-test|integration-test|property-test|unit-test|test|doc|publish|clean]"
 	@exit 1
 
 install:
@@ -29,14 +29,29 @@ check:
 	make ruff-check
 	make mypy-check
 
+benchmark-test:
+	uv run --group tests --locked pytest -m perf
+
+integration-test:
+	uv run --group tests --locked pytest -m integration
+
+property-test:
+	uv run --group tests --locked pytest -m property
+
+unit-test:
+	uv run --group tests --locked pytest -m unit
+
 test:
 	uv run --group tests --locked pytest
 
 doc:
 	uv run --group docs --locked sphinx-build -M html docs/source docs/build
 
 publish:
-	rm -rf dist/
+	make clean
 	uv build
 	uvx twine check dist/*
 	uv publish
+
+clean:
+	git clean -xfd --exclude .venv

make.bat

@@ -13,11 +13,16 @@ if "%1"=="fix" goto fix
 if "%1"=="ruff-check" goto ruff-check
 if "%1"=="mypy-check" goto mypy-check
 if "%1"=="check" goto check
+if "%1"=="benchmark-test" goto benchmark-test
+if "%1"=="integration-test" goto integration-test
+if "%1"=="property-test" goto property-test
+if "%1"=="unit-test" goto unit-test
 if "%1"=="test" goto test
 if "%1"=="doc" goto doc
 if "%1"=="publish" goto publish
+if "%1"=="clean" goto clean
 
-echo Usage: make.bat [install^|format^|fix^|ruff-check^|mypy-check^|check^|test^|doc^|publish]
+echo Usage: make.bat [install^|format^|fix^|ruff-check^|mypy-check^|check^|benchmark-test^|integration-test^|property-test^|unit-test^|test^|doc^|publish^|clean]
 exit /b 1
 
 :install
@@ -49,6 +54,22 @@ if errorlevel 1 exit /b %errorlevel%
 call :mypy-check
 exit /b %errorlevel%
 
+:benchmark-test
+uv run --group tests --locked pytest -m perf
+exit /b %errorlevel%
+
+:integration-test
+uv run --group tests --locked pytest -m integration
+exit /b %errorlevel%
+
+:property-test
+uv run --group tests --locked pytest -m property
+exit /b %errorlevel%
+
+:unit-test
+uv run --group tests --locked pytest -m unit
+exit /b %errorlevel%
+
 :test
 uv run --group tests --locked pytest
 exit /b %errorlevel%
@@ -58,7 +79,7 @@ call uv run --group docs --locked sphinx-build -M html docs/source docs/build
 exit /b %errorlevel%
 
 :publish
-if exist dist\ rmdir /s /q dist\
+call .\make.bat clean
 
 uv build
 if errorlevel 1 exit /b %errorlevel%
@@ -68,3 +89,7 @@ if errorlevel 1 exit /b %errorlevel%
 
 uv publish
 exit /b %errorlevel%
+
+:clean
+git clean -xfd --exclude .venv
+exit /b %errorlevel%

pyproject.toml

@@ -42,7 +42,9 @@ changelog = "https://github.com/jxmorris12/language_tool_python/blob/master/CHAN
 [dependency-groups]
 tests = [
     "pytest",
+    "pytest-benchmark",
     "pytest-cov",
+    "hypothesis",
 ]
 
 docs = [
@@ -141,9 +143,10 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*.py" = [
-    "S101",  # Need to use assert statements in tests
-    "SLF001" # Need to use private members of the library for testing
+"tests/**/*.py" = [
+    "S101",   # Need to use assert statements in tests
+    "SLF001", # Need to use private members of the library for testing
+    "RUF001", # LanguageTool output contains typographic quotes (‘’“”)
 ]
 "src/language_tool_python/__main__.py" = ["T201"] # Allow usage of print in the CLI entry point
 
@@ -170,3 +173,9 @@ warn_return_any = true
 warn_unreachable = true
 warn_unused_configs = true
 warn_unused_ignores = true
+
+[[tool.mypy.overrides]]
+module = ["tests.property.*"]
+# hypothesis decorators contain Any expressions, so we need to disable the following checks for tests using hypothesis
+disallow_any_decorated = false
+disallow_any_expr = false

pytest.ini

@@ -1,6 +1,11 @@
 [pytest]
-addopts = -vra --cov=src --cov-report=html --cov-report=xml
+addopts = -vra --cov=src --cov-report=html --cov-report=xml --basetemp=.pytest_tmp
 testpaths = tests
+markers =
+    unit: fast, isolated tests with no external dependencies
+    integration: tests that require a live LanguageTool server or network
+    property: property-based tests using Hypothesis
+    perf: performance benchmark tests using pytest-benchmark
 
 [coverage:run]
 source = src

src/language_tool_python/__main__.py

@@ -58,7 +58,7 @@ def _read_project_version(pyproject: Path) -> str:
     __version__ = version("language_tool_python")
     # If the package is not installed in the environment,
     # read the version from pyproject.toml
-except PackageNotFoundError:
+except PackageNotFoundError:  # pragma: no cover  # package installed in test env
     project_root = Path(__file__).resolve().parent.parent
     pyproject = project_root / "pyproject.toml"
     __version__ = _read_project_version(pyproject)
@@ -258,7 +258,7 @@ def __call__(
             cli_args.disable_categories.update(rule_values)
         elif self.dest == "enable_categories":
             cli_args.enable_categories.update(rule_values)
-        else:
+        else:  # pragma: no cover  # defensive: all known dest values are handled above
             err = f"unexpected rules destination: {self.dest}"
             raise ValueError(err)
 
@@ -449,5 +449,5 @@ def main(argv: Sequence[str] | None = None) -> int:
     return status
 
 
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     raise SystemExit(main())

src/language_tool_python/_internals/compat.py

@@ -13,11 +13,13 @@
 if sys.version_info >= (3, 11):
     from tomllib import loads as toml_loads
 else:
-    from tomli import loads as toml_loads
+    # Python < 3.11 fallback, cov CI runs on 3.11+, so this branch is never executed.
+    from tomli import loads as toml_loads  # pragma: no cover
 
 if sys.version_info >= (3, 13):
     from warnings import deprecated
 else:
-    from typing_extensions import deprecated
+    # Python < 3.13 fallback, cov CI runs on 3.13+, so this branch is never executed.
+    from typing_extensions import deprecated  # pragma: no cover
 
 __all__ = ["deprecated", "toml_loads"]

src/language_tool_python/_internals/safe_zip.py

@@ -179,7 +179,9 @@ def _normalize_member_path(self, filename: str) -> PurePosixPath:
 
         member_path = PurePosixPath(*parts)
 
-        if member_path.is_absolute() or any(part == ".." for part in member_path.parts):
+        if (  # pragma: no cover  # parts validated; PurePosixPath always relative
+            member_path.is_absolute() or any(part == ".." for part in member_path.parts)
+        ):
             err = f"Unsafe ZIP member path: {filename!r}."
             raise PathError(err)
 
@@ -256,7 +258,7 @@ def _zip_target(self, destination: Path, member_path: PurePosixPath) -> Path:
 
         if destination_resolved != target_resolved and (
             destination_resolved not in target_resolved.parents
-        ):
+        ):  # pragma: no cover  # TOCTOU: escape needs concurrent modification
             err = f"Unsafe ZIP member path: {str(member_path)!r}."
             raise PathError(err)
 
@@ -457,13 +459,13 @@ def _ensure_safe_parent(self, destination: Path, target: Path) -> None:
 
         if destination_resolved != parent_resolved and (
             destination_resolved not in parent_resolved.parents
-        ):
+        ):  # pragma: no cover  # TOCTOU: escape needs concurrent modification
             err = f"Unsafe ZIP extraction parent path: {target.parent}."
             raise PathError(err)
 
         try:
             relative_parent = target.parent.relative_to(destination)
-        except ValueError as e:
+        except ValueError as e:  # pragma: no cover  # caught by check above
             err = f"Unsafe ZIP extraction parent path: {target.parent}."
             raise PathError(err) from e
 
@@ -472,19 +474,19 @@ def _ensure_safe_parent(self, destination: Path, target: Path) -> None:
         for part in relative_parent.parts:
             current = current / part
 
-            if current.is_symlink():
+            if current.is_symlink():  # pragma: no cover  # TOCTOU: mkdir'd above
                 err = f"Refusing to extract through symlinked directory: {current}."
                 raise PathError(err)
 
-            if not current.is_dir():
+            if not current.is_dir():  # pragma: no cover  # TOCTOU: mkdir'd above
                 err = f"Refusing to extract through non-directory path: {current}."
                 raise PathError(err)
 
             current_resolved = current.resolve(strict=True)
 
             if destination_resolved != current_resolved and (
                 destination_resolved not in current_resolved.parents
-            ):
+            ):  # pragma: no cover  # TOCTOU: escape needs concurrent modification
                 err = f"Unsafe ZIP extraction directory path: {current}."
                 raise PathError(err)
 
@@ -504,11 +506,11 @@ def _copy_member(
         :type target: Path
         :raises PathError: If the target is unsafe or size checks fail.
         """
-        if target.exists() or target.is_symlink():
+        if target.exists() or target.is_symlink():  # pragma: no cover  # TOCTOU
             err = f"Refusing to overwrite existing path while extracting ZIP: {target}."
             raise PathError(err)
 
-        if target.parent.is_symlink():
+        if target.parent.is_symlink():  # pragma: no cover  # TOCTOU: parent safe above
             err = (
                 f"Refusing to extract into symlinked parent directory: {target.parent}."
             )
@@ -578,21 +580,23 @@ def _extract_to_private_directory(
 
         destination.mkdir(parents=True, exist_ok=True)
 
-        if destination.is_symlink():
+        if destination.is_symlink():  # pragma: no cover  # TOCTOU: mkdtemp dir
             err = f"Refusing to extract into symlinked destination: {destination}."
             raise PathError(err)
 
         destination_resolved = destination.resolve(strict=True)
 
-        if not destination_resolved.is_dir():
+        if not destination_resolved.is_dir():  # pragma: no cover  # TOCTOU: mkdir'd
             err = f"ZIP extraction destination is not a directory: {destination}."
             raise PathError(err)
 
         for member, member_path in validated_members:
             target = self._zip_target(destination, member_path)
 
             if member.is_dir():
-                if target.exists() and not target.is_dir():
+                if (  # pragma: no cover  # dup-path check catches file-vs-dir above
+                    target.exists() and not target.is_dir()
+                ):
                     err = (
                         f"Refusing to overwrite existing path while extracting ZIP: "
                         f"{target}."
@@ -602,16 +606,17 @@ def _extract_to_private_directory(
                 target.mkdir(parents=True, exist_ok=True)
                 self._ensure_safe_parent(destination, target)
 
-                if target.is_symlink():
+                if target.is_symlink():  # pragma: no cover  # TOCTOU: mkdir'd above
                     err = (
                         f"Refusing to create or use symlinked ZIP directory: {target}."
                     )
                     raise PathError(err)
 
                 target_resolved = target.resolve(strict=True)
 
-                if destination_resolved != target_resolved and (
-                    destination_resolved not in target_resolved.parents
+                if (  # pragma: no cover  # TOCTOU: mkdir'd dir escaped
+                    destination_resolved != target_resolved
+                    and destination_resolved not in target_resolved.parents
                 ):
                     err = f"Unsafe ZIP directory path after creation: {target}."
                     raise PathError(err)
@@ -687,15 +692,15 @@ def _extractall_to_directory(
 
             final_directory_resolved = final_directory.resolve(strict=True)
 
-            if not final_directory_resolved.is_dir():
+            if not final_directory_resolved.is_dir():  # pragma: no cover  # TOCTOU
                 err = (
                     f"ZIP extraction destination is not a directory: {final_directory}."
                 )
                 raise PathError(err)
 
             destinations: list[tuple[Path, Path]] = []
             for child in extract_dir.iterdir():
-                if child.is_symlink():
+                if child.is_symlink():  # pragma: no cover  # symlinks rejected above
                     err = f"Refusing to move symlinked extracted path: {child}."
                     raise PathError(err)
 

src/language_tool_python/config_file.py

@@ -158,7 +158,9 @@ def _path_validator(v: PathLike[str] | str) -> None:
     if not p.exists():
         err = f"path does not exist: {p}"
         raise PathError(err)
-    if not p.is_file() and not p.is_dir():
+    # Defensive: a path that exists but is neither file nor directory (e.g. socket,
+    # device node, FIFO) cannot be created portably in unit tests.
+    if not p.is_file() and not p.is_dir():  # pragma: no cover
         err = f"path is not a file/directory: {p}"
         raise PathError(err)