test: split tests in categs, add bench tests

Author: mdevolde

pyproject.toml

@@ -42,7 +42,9 @@ changelog = "https://github.com/jxmorris12/language_tool_python/blob/master/CHAN
 [dependency-groups]
 tests = [
     "pytest",
+    "pytest-benchmark",
     "pytest-cov",
+    "hypothesis",
 ]
 
 docs = [
@@ -141,9 +143,10 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*.py" = [
-    "S101",  # Need to use assert statements in tests
-    "SLF001" # Need to use private members of the library for testing
+"tests/**/*.py" = [
+    "S101",   # Need to use assert statements in tests
+    "SLF001", # Need to use private members of the library for testing
+    "RUF001", # LanguageTool output contains typographic quotes (‘’“”)
 ]
 "src/language_tool_python/__main__.py" = ["T201"] # Allow usage of print in the CLI entry point
 
@@ -170,3 +173,19 @@ warn_return_any = true
 warn_unreachable = true
 warn_unused_configs = true
 warn_unused_ignores = true
+
+[[tool.mypy.overrides]]
+module = ["tests.benchmarks.*"]
+# pytest-benchmark is untyped; relax Any restrictions for benchmark files only
+disallow_any_unimported = false
+disallow_any_expr = false
+disallow_any_explicit = false
+disallow_any_decorated = false
+
+[[tool.mypy.overrides]]
+module = ["tests.property.*"]
+# hypothesis is untyped; relax Any restrictions for property test files only
+disallow_any_unimported = false
+disallow_any_expr = false
+disallow_any_explicit = false
+disallow_any_decorated = false

pytest.ini

@@ -1,6 +1,11 @@
 [pytest]
 addopts = -vra --cov=src --cov-report=html --cov-report=xml
 testpaths = tests
+markers =
+    unit: fast, isolated tests with no external dependencies
+    integration: tests that require a live LanguageTool server or network
+    property: property-based tests using Hypothesis
+    perf: performance benchmark tests using pytest-benchmark
 
 [coverage:run]
 source = src

tests/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark tests for the language_tool_python library."""

tests/benchmarks/conftest.py

@@ -0,0 +1,17 @@
+"""Configuration for the benchmark test suite."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+
+def pytest_collection_modifyitems(
+    items: list[pytest.Item],
+) -> None:
+    """Apply the 'perf' marker to all tests collected from this directory."""
+    benchmarks_dir = Path(__file__).parent
+    for item in items:
+        if item.path.is_relative_to(benchmarks_dir):
+            item.add_marker(pytest.mark.perf)

tests/benchmarks/test_bench_check.py

@@ -0,0 +1,81 @@
+"""Benchmark tests for LanguageTool grammar checking performance.
+
+Run with: pytest tests/benchmarks/ -v
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+import language_tool_python
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+_SHORT_TEXT = "This is a sentence with some erors in it. "
+_MEDIUM_TEXT = (_SHORT_TEXT * 20).strip()
+_LONG_TEXT = (_SHORT_TEXT * 100).strip()
+
+
+@pytest.fixture(scope="module")
+def tool() -> Generator[language_tool_python.LanguageTool, None, None]:
+    """Provide a LanguageTool instance shared across benchmarks in this module."""
+    with language_tool_python.LanguageTool("en-US") as t:
+        yield t
+
+
+@pytest.fixture(scope="module")
+def cached_tool() -> Generator[language_tool_python.LanguageTool, None, None]:
+    """Provide a pipeline-caching LanguageTool instance for cache benchmarks."""
+    with language_tool_python.LanguageTool(
+        "en-US",
+        config={"cacheSize": 1000, "pipelineCaching": True},
+    ) as t:
+        yield t
+
+
+def test_bench_check_short_text(
+    benchmark: BenchmarkFixture,
+    tool: language_tool_python.LanguageTool,
+) -> None:
+    """Benchmark grammar checking on a short sentence (~38 characters)."""
+    benchmark(tool.check, _SHORT_TEXT)
+
+
+def test_bench_check_medium_text(
+    benchmark: BenchmarkFixture,
+    tool: language_tool_python.LanguageTool,
+) -> None:
+    """Benchmark grammar checking on medium-length text (~840 characters)."""
+    benchmark(tool.check, _MEDIUM_TEXT)
+
+
+def test_bench_check_long_text(
+    benchmark: BenchmarkFixture,
+    tool: language_tool_python.LanguageTool,
+) -> None:
+    """Benchmark grammar checking on long text (~4200 characters)."""
+    benchmark(tool.check, _LONG_TEXT)
+
+
+def test_bench_correct_short_text(
+    benchmark: BenchmarkFixture,
+    tool: language_tool_python.LanguageTool,
+) -> None:
+    """Benchmark automatic text correction on a short sentence."""
+    benchmark(tool.correct, _SHORT_TEXT)
+
+
+def test_bench_check_with_pipeline_cache(
+    benchmark: BenchmarkFixture,
+    cached_tool: language_tool_python.LanguageTool,
+) -> None:
+    """Benchmark grammar checking with pipeline caching enabled.
+
+    Compare with test_bench_check_short_text to measure cache speedup.
+    """
+    benchmark(cached_tool.check, _SHORT_TEXT)

tests/integration/__init__.py

@@ -0,0 +1 @@
+"""Integration tests for the language_tool_python library."""

tests/integration/conftest.py

@@ -0,0 +1,17 @@
+"""Configuration for the integration test suite."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+
+def pytest_collection_modifyitems(
+    items: list[pytest.Item],
+) -> None:
+    """Apply the 'integration' marker to all tests collected from this directory."""
+    integration_dir = Path(__file__).parent
+    for item in items:
+        if item.path.is_relative_to(integration_dir):
+            item.add_marker(pytest.mark.integration)

tests/test_api_public.py tests/integration/test_api_public.pytests/test_api_public.py renamed to tests/integration/test_api_public.py

@@ -1,4 +1,4 @@
-"""Tests for the public API functionality."""
+"""Integration tests for the public API functionality."""
 
 import pytest
 
@@ -34,7 +34,7 @@ def test_remote_es() -> None:
                    'INCORRECT_EXPRESSIONS', 'rule_issue_type': 'grammar', 'sentence':
                    'LanguageTool le ayudará a afrentar algunas dificultades propias de
                    la escritura.'}), Match({'rule_id': 'PRON_HABER_PARTICIPIO',
-                   'message': 'El v. \u2018haber\u2019 se escribe con hache.',
+                   'message': 'El v. ‘haber’ se escribe con hache.',
                    'replacements': ['ha'], 'offset_in_context': 43, 'context':
                    '...ificultades propias de la escritura. Se a hecho un esfuerzo para
                    detectar errores...', 'offset': 107, 'error_length': 1, 'category':
@@ -50,8 +50,8 @@ def test_remote_es() -> None:
                    'misspelling', 'sentence': 'Se a hecho un esfuerzo para detectar
                    errores tipográficos, ortograficos y incluso gramaticales.'}),
                    Match({'rule_id': 'Y_E_O_U', 'message': 'Cuando precede a palabras
-                   que comienzan por \u2018i\u2019, la conjunción \u2018y\u2019 se
-                   transforma en \u2018e\u2019.', 'replacements': ['e'],
+                   que comienzan por ‘i’, la conjunción ‘y’ se
+                   transforma en ‘e’.', 'replacements': ['e'],
                    'offset_in_context': 43, 'context': '...ctar errores tipográficos,
                    ortograficos y incluso gramaticales. También algunos e...', 'offset':
                    176, 'error_length': 1, 'category': 'GRAMMAR', 'rule_issue_type':

tests/test_cli.py tests/integration/test_cli.pytests/test_cli.py renamed to tests/integration/test_cli.py

@@ -1,4 +1,4 @@
-"""Tests for the command-line interface (CLI) functionality."""
+"""Integration tests for the CLI using real LanguageTool server instances."""
 
 import io
 import sys
@@ -7,7 +7,39 @@
 import pytest
 
 import language_tool_python
-from language_tool_python.__main__ import main, parse_args
+from language_tool_python.__main__ import main
+
+
+def main_with_stdin(argv: list[str], stdin: str) -> int:
+    """Execute the main CLI with simulated stdin input.
+
+    :param argv: Command-line arguments to pass to the main function.
+    :param stdin: Input text to simulate as stdin.
+    :return: Exit code returned by the main function.
+    :rtype: int
+    """
+    old_stdin = sys.stdin
+    sys.stdin = io.StringIO(stdin)
+    try:
+        return main(argv)
+    finally:
+        sys.stdin = old_stdin
+
+
+@pytest.fixture(scope="module")
+def remote_server() -> Generator[tuple[str, int], None, None]:
+    """Fixture that provides a remote LanguageTool server for testing.
+
+    This fixture initializes a LanguageTool instance and yields its host and port,
+    ensuring proper cleanup after all tests in the module complete.
+
+    :return: A tuple containing the server host and port (host, port).
+    :rtype: Generator[Tuple[str, int], None, None]
+    """
+    with language_tool_python.LanguageTool("en-US") as tool:
+        host = tool._host
+        port = tool._port
+        yield host, port
 
 
 @pytest.mark.parametrize(
@@ -89,22 +121,6 @@ def test_cli_exit_codes(
         assert code != 0
 
 
-@pytest.fixture(scope="module")
-def remote_server() -> Generator[tuple[str, int], None, None]:
-    """Fixture that provides a remote LanguageTool server for testing.
-
-    This fixture initializes a LanguageTool instance and yields its host and port,
-    ensuring proper cleanup after all tests in the module complete.
-
-    :return: A tuple containing the server host and port (host, port).
-    :rtype: Generator[Tuple[str, int], None, None]
-    """
-    with language_tool_python.LanguageTool("en-US") as tool:
-        host = tool._host
-        port = tool._port
-        yield host, port
-
-
 def test_cli_remote_ok(remote_server: tuple[str, int]) -> None:
     """Test the CLI with a remote server using valid input text.
 
@@ -155,78 +171,3 @@ def test_cli_remote_error(remote_server: tuple[str, int]) -> None:
         "This is noot okay.\n",
     )
     assert code != 0
-
-
-def test_parse_args_enabled_only_with_enable_categories() -> None:
-    """Test that --enabled-only is accepted when only --enable-categories is provided.
-
-    :raises AssertionError: If parse_args raises an error for this valid combination.
-    """
-    args = parse_args(["-l", "en-US", "--enabled-only", "-E", "TYPOS", "file.txt"])
-    assert args.enabled_only is True
-    assert args.enable_categories == {"TYPOS"}
-
-
-def test_parse_args_enabled_only_rejects_disable_categories() -> None:
-    """Test that --enabled-only cannot be combined with --disable-categories.
-
-    :raises SystemExit: Expected, as argparse calls sys.exit on error.
-    """
-    with pytest.raises(SystemExit):
-        parse_args(
-            ["-l", "en-US", "--enabled-only", "-e", "RULE", "-D", "TYPOS", "file.txt"]
-        )
-
-
-def test_parse_args_enabled_only_requires_enable_or_enable_categories() -> None:
-    """Test that --enabled-only requires at least --enable or --enable-categories.
-
-    :raises SystemExit: Expected, as argparse calls sys.exit on error.
-    """
-    with pytest.raises(SystemExit):
-        parse_args(["-l", "en-US", "--enabled-only", "file.txt"])
-
-
-def test_parse_args_categories() -> None:
-    """Test that --disable-categories and --enable-categories are parsed correctly.
-
-    :raises AssertionError: If the parsed category sets do not match the expected
-    values.
-    """
-    args = parse_args(
-        ["-l", "en-US", "-D", "TYPOS,GRAMMAR", "-E", "PUNCTUATION", "file.txt"]
-    )
-    assert args.disable_categories == {"TYPOS", "GRAMMAR"}
-    assert args.enable_categories == {"PUNCTUATION"}
-
-
-def test_parse_args_categories_multiple_flags() -> None:
-    """Test that repeated -D/-E flags accumulate into the same set.
-
-    :raises AssertionError: If the category sets do not accumulate correctly.
-    """
-    args = parse_args(
-        ["-l", "en-US", "-D", "TYPOS", "-D", "GRAMMAR", "-E", "PUNCTUATION", "file.txt"]
-    )
-    assert args.disable_categories == {"TYPOS", "GRAMMAR"}
-    assert args.enable_categories == {"PUNCTUATION"}
-
-
-def main_with_stdin(argv: list[str], stdin: str) -> int:
-    """Execute the main CLI with simulated stdin input.
-
-    This utility function temporarily replaces sys.stdin with a StringIO object
-    containing the provided input, executes the main CLI function, and then restores the
-    original stdin.
-
-    :param argv: Command-line arguments to pass to the main function.
-    :param stdin: Input text to simulate as stdin.
-    :return: Exit code returned by the main function.
-    :rtype: int
-    """
-    old_stdin = sys.stdin
-    sys.stdin = io.StringIO(stdin)
-    try:
-        return main(argv)
-    finally:
-        sys.stdin = old_stdin