From c349ffc066640ae88ed6d80b01c14723ede5a2d4 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 6 Aug 2026 12:33:35 +0200 Subject: [PATCH 1/2] Add benchmarks and regenerate the parser comparison from measurements --- .github/dependabot.yml | 15 +++ .github/workflows/benchmarks.yml | 35 +++++ .github/workflows/comparison.yml | 41 ++++++ .github/workflows/publish.yml | 8 +- .github/workflows/tests-macos.yml | 11 +- .github/workflows/tests-ubuntu.yml | 11 +- .github/workflows/tests-windows.yml | 11 +- .pre-commit-config.yaml | 5 + README.rst | 94 +++++++++++--- benchmarks/__init__.py | 0 benchmarks/_parsers.py | 194 ++++++++++++++++++++++++++++ benchmarks/_run.py | 26 ++++ benchmarks/_workload.py | 36 ++++++ benchmarks/compare.py | 160 +++++++++++++++++++++++ benchmarks/requirements.txt | 2 + benchmarks/test_benchmarks.py | 20 +++ benchmarks/test_comparison.py | 67 ++++++++++ tox.ini | 16 +++ 18 files changed, 719 insertions(+), 33 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/benchmarks.yml create mode 100644 .github/workflows/comparison.yml create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/_parsers.py create mode 100644 benchmarks/_run.py create mode 100644 benchmarks/_workload.py create mode 100644 benchmarks/compare.py create mode 100644 benchmarks/requirements.txt create mode 100644 benchmarks/test_benchmarks.py create mode 100644 benchmarks/test_comparison.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..40d7f6ee --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: /benchmarks + schedule: + interval: monthly + cooldown: + default-days: 7 + +- package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + cooldown: + default-days: 7 diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..fb6fd2b5 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,35 @@ +name: Benchmarks +on: + push: + branches: [master] + pull_request: + +permissions: + contents: read + +jobs: + codspeed: + runs-on: ubuntu-latest + env: + # Hash randomization perturbs dict and set behaviour, which shows up as + # jitter in instruction counts. + PYTHONHASHSEED: "0" + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.14.7" + + - name: Install + run: | + pip install -U tox + + - uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + mode: simulation + exclude-allocations: true + run: tox -e benchmarks diff --git a/.github/workflows/comparison.yml b/.github/workflows/comparison.yml new file mode 100644 index 00000000..6358c295 --- /dev/null +++ b/.github/workflows/comparison.yml @@ -0,0 +1,41 @@ +name: Comparison table +on: + workflow_dispatch: + push: + branches: [master] + paths: + - src/** + - benchmarks/** + +permissions: + contents: write + pull-requests: write + +jobs: + comparison: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.14.7" + + - name: Install + run: | + sudo apt-get update && sudo apt-get install -y valgrind + pip install -U tox + + - name: Regenerate the table + run: tox -e comparison -- --write + + - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + branch: comparison-table + title: Update the parser comparison table + commit-message: Update the parser comparison table + body: | + Regenerated by `benchmarks/compare.py` from commit ${{ github.sha }}. + delete-branch: true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a9f10a73..a6fb082b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,12 +13,14 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: 3.14 - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 2f6bad9e..1043c918 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,6 +1,9 @@ name: macOS on: [push, pull_request] +permissions: + contents: read + jobs: tests: runs-on: macos-latest @@ -10,10 +13,12 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} @@ -23,4 +28,4 @@ jobs: tox -e py - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f28ebd3b..91c06a2f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,6 +1,9 @@ name: Ubuntu on: [push, pull_request] +permissions: + contents: read + jobs: tests: runs-on: ubuntu-latest @@ -38,10 +41,12 @@ jobs: TOXENV: typing steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} @@ -57,4 +62,4 @@ jobs: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 1f065da9..4e46c36e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,6 +1,9 @@ name: Windows on: [push, pull_request] +permissions: + contents: read + jobs: tests: runs-on: windows-latest @@ -10,10 +13,12 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} @@ -23,4 +28,4 @@ jobs: tox -e py - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d3d4f6b..8dfe708a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,8 @@ repos: rev: v1.7.10 hooks: - id: actionlint +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/README.rst b/README.rst index 3c5a6b2c..92076ad0 100644 --- a/README.rst +++ b/README.rst @@ -86,27 +86,79 @@ Comparison ========== The following table compares Protego to the most popular ``robots.txt`` parsers -implemented in Python or featuring Python bindings: - -+----------------------------+---------+-----------------+--------+---------------------------+ -| | Protego | RobotFileParser | Reppy | Robotexclusionrulesparser | -+============================+=========+=================+========+===========================+ -| Implementation language | Python | Python | C++ | Python | -+----------------------------+---------+-----------------+--------+---------------------------+ -| Reference specification | Google_ | `Martijn Koster’s 1996 draft`_ | -+----------------------------+---------+-----------------+--------+---------------------------+ -| `Wildcard support`_ | ✓ | | ✓ | ✓ | -+----------------------------+---------+-----------------+--------+---------------------------+ -| `Length-based precedence`_ | ✓ | | ✓ | | -+----------------------------+---------+-----------------+--------+---------------------------+ -| Performance_ | | +40% | +1300% | -25% | -+----------------------------+---------+-----------------+--------+---------------------------+ - -.. _Google: https://developers.google.com/search/reference/robots_txt -.. _Length-based precedence: https://developers.google.com/search/reference/robots_txt#order-of-precedence-for-group-member-lines -.. _Martijn Koster’s 1996 draft: https://www.robotstxt.org/norobots-rfc.txt -.. _Performance: https://anubhavp28.github.io/gsoc-weekly-checkin-12/ -.. _Wildcard support: https://developers.google.com/search/reference/robots_txt#url-matching-based-on-path-values +implemented in Python. Performance is the speed difference against Protego, +measured as executed instructions, so a positive value means faster than +Protego. + +.. comparison-table-start + +.. list-table:: + :header-rows: 1 + :stub-columns: 1 + + * - + - Protego + - RobotFileParser + - robotspy + - Robotexclusionrulesparser + * - Version tested + - + - Python 3.14.7 + - 0.13.0 + - 1.7.1 + * - Reference specification + - Google_ + - `Martijn Koster's 1996 draft`_ + - `RFC 9309`_ + - `Martijn Koster's 1996 draft`_ + * - `Wildcard support`_ + - ✓ + - + - ✓ + - ✓ + * - `Length-based precedence`_ + - ✓ + - + - ✓ + - + * - Crawl-delay + - ✓ + - ✓ + - + - + * - Request-rate + - ✓ + - + - + - + * - Visit-time + - ✓ + - + - + - + * - Sitemaps + - ✓ + - ✓ + - ✓ + - ✓ + * - Host + - ✓ + - + - + - + * - Performance + - + - +40% + - + - -25% + +.. comparison-table-end + +.. _Google: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec +.. _Length-based precedence: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec#order-of-precedence-for-rules +.. _Martijn Koster's 1996 draft: https://www.robotstxt.org/norobots-rfc.txt +.. _RFC 9309: https://www.rfc-editor.org/rfc/rfc9309 +.. _Wildcard support: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec#url-matching-based-on-path-values API Reference diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/_parsers.py b/benchmarks/_parsers.py new file mode 100644 index 00000000..d62b3be4 --- /dev/null +++ b/benchmarks/_parsers.py @@ -0,0 +1,194 @@ +"""Uniform adapters over the ``robots.txt`` parsers compared in the README.""" + +from __future__ import annotations + +import io +import platform +from importlib.metadata import version +from typing import TYPE_CHECKING, ClassVar +from urllib.robotparser import RobotFileParser + +from protego import Protego + +if TYPE_CHECKING: + from collections.abc import Callable + +_UA = "mybot" + + +class _Adapter: + """Common interface over one ``robots.txt`` parser.""" + + name: ClassVar[str] + distribution: ClassVar[str | None] + + #: Directives the parser exposes, as names of the probes that cover them. + #: A probe absent here is reported as unsupported without being run. + directives: ClassVar[frozenset[str]] = frozenset() + + def __init__(self, content: str): + self._parser = self._parse(content) + + @staticmethod + def _parse(content: str) -> object: + raise NotImplementedError + + def can_fetch(self, url: str) -> bool: + raise NotImplementedError + + @classmethod + def version(cls) -> str: + return version(cls.distribution) if cls.distribution else "" + + +class ProtegoAdapter(_Adapter): + name = "Protego" + # Deliberately blank in the README: the table ships with the release. + distribution = None + directives = frozenset( + {"crawl_delay", "request_rate", "visit_time", "sitemaps", "host"} + ) + + @staticmethod + def _parse(content: str) -> Protego: + return Protego.parse(content) + + def can_fetch(self, url: str) -> bool: + return self._parser.can_fetch(url, _UA) + + +class RobotFileParserAdapter(_Adapter): + name = "RobotFileParser" + distribution = None + directives = frozenset({"crawl_delay", "request_rate", "sitemaps"}) + + @staticmethod + def _parse(content: str) -> RobotFileParser: + parser = RobotFileParser() + parser.parse(io.StringIO(content).readlines()) + return parser + + def can_fetch(self, url: str) -> bool: + return self._parser.can_fetch(_UA, url) + + @classmethod + def version(cls) -> str: + return f"Python {platform.python_version()}" + + +class RobotspyAdapter(_Adapter): + name = "robotspy" + distribution = "robotspy" + directives = frozenset({"sitemaps"}) + + @staticmethod + def _parse(content: str) -> object: + # Imported here so the CodSpeed suite, which only benchmarks Protego, + # runs without the other parsers installed. + import robots # noqa: PLC0415 + + return robots.RobotsParser.from_string(content) + + def can_fetch(self, url: str) -> bool: + return self._parser.can_fetch(_UA, url) + + +class RerpAdapter(_Adapter): + name = "Robotexclusionrulesparser" + distribution = "robotexclusionrulesparser" + directives = frozenset({"crawl_delay", "sitemaps"}) + + @staticmethod + def _parse(content: str) -> object: + import robotexclusionrulesparser # noqa: PLC0415 + + parser = robotexclusionrulesparser.RobotExclusionRulesParser() + parser.parse(content) + return parser + + def can_fetch(self, url: str) -> bool: + return self._parser.is_allowed(_UA, url) + + +ADAPTERS: list[type[_Adapter]] = [ + ProtegoAdapter, + RobotFileParserAdapter, + RobotspyAdapter, + RerpAdapter, +] + + +def _probe_wildcards(adapter: type[_Adapter]) -> bool: + parser = adapter("User-agent: *\nDisallow: /a/*/b\n") + return not parser.can_fetch("/a/x/b") and parser.can_fetch("/a/x/c") + + +def _probe_precedence(adapter: type[_Adapter]) -> bool: + # The Disallow rule comes first but is shorter, so the two orderings + # disagree: length-based precedence allows the URL, first-match denies it. + parser = adapter("User-agent: *\nDisallow: /a/b/\nAllow: /a/b/c\n") + return parser.can_fetch("/a/b/c") + + +def _probe_crawl_delay(adapter: type[_Adapter]) -> bool: + parser = adapter("User-agent: *\nCrawl-delay: 4\n")._parser + getter = getattr(parser, "crawl_delay", None) or getattr( + parser, "get_crawl_delay", None + ) + return getter is not None and getter(_UA) in (4, 4.0) + + +def _probe_request_rate(adapter: type[_Adapter]) -> bool: + parser = adapter("User-agent: *\nRequest-rate: 10/1m\n")._parser + getter = getattr(parser, "request_rate", None) + if getter is None: + return False + rate = getter(_UA) + return rate is not None and (rate.requests, rate.seconds) == (10, 60) + + +def _probe_visit_time(adapter: type[_Adapter]) -> bool: + parser = adapter("User-agent: *\nVisit-time: 0100-0200\n")._parser + getter = getattr(parser, "visit_time", None) + return getter is not None and getter(_UA) is not None + + +def _probe_sitemaps(adapter: type[_Adapter]) -> bool: + content = "User-agent: *\nSitemap: http://example.com/sitemap.xml\n" + parser = adapter(content)._parser + for attribute in ("sitemaps", "site_maps"): + value = getattr(parser, attribute, None) + if value is None: + continue + found = value() if callable(value) else value + if found and list(found) == ["http://example.com/sitemap.xml"]: + return True + return False + + +def _probe_host(adapter: type[_Adapter]) -> bool: + parser = adapter("User-agent: *\nHost: http://example.co.in\n")._parser + return getattr(parser, "preferred_host", None) == "http://example.co.in" + + +#: Feature rows of the README table, in display order, each paired with the +#: probe that decides whether a parser gets a checkmark. +FEATURES: list[tuple[str, str, Callable[[type[_Adapter]], bool]]] = [ + ("`Wildcard support`_", "wildcards", _probe_wildcards), + ("`Length-based precedence`_", "precedence", _probe_precedence), + ("Crawl-delay", "crawl_delay", _probe_crawl_delay), + ("Request-rate", "request_rate", _probe_request_rate), + ("Visit-time", "visit_time", _probe_visit_time), + ("Sitemaps", "sitemaps", _probe_sitemaps), + ("Host", "host", _probe_host), +] + + +def supports(adapter: type[_Adapter], key: str) -> bool: + """Return whether *adapter* supports the feature named *key*.""" + probe = next(probe for _, name, probe in FEATURES if name == key) + # Probes for directives call getters that a parser may not define at all; + # the declared set keeps a missing attribute from reading as a failed probe. + if key in {"crawl_delay", "request_rate", "visit_time", "sitemaps", "host"}: + return key in adapter.directives and probe(adapter) + return probe(adapter) diff --git a/benchmarks/_run.py b/benchmarks/_run.py new file mode 100644 index 00000000..964bb624 --- /dev/null +++ b/benchmarks/_run.py @@ -0,0 +1,26 @@ +"""Run one benchmark a fixed number of times, for measurement under callgrind.""" + +from __future__ import annotations + +import sys + +from benchmarks._parsers import ADAPTERS +from benchmarks._workload import content, match, parse + + +def main() -> None: + name, bench, iterations = sys.argv[1], sys.argv[2], int(sys.argv[3]) + adapter = next(a for a in ADAPTERS if a.name == name) + robotstxt = content() + + if bench == "parse": + for _ in range(iterations): + parse(adapter, robotstxt) + else: + parser = parse(adapter, robotstxt) + for _ in range(iterations): + match(parser) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/_workload.py b/benchmarks/_workload.py new file mode 100644 index 00000000..e70a0e49 --- /dev/null +++ b/benchmarks/_workload.py @@ -0,0 +1,36 @@ +"""The work measured by both the CodSpeed suite and the README comparison.""" + +from __future__ import annotations + +from pathlib import Path + +CORPUS = Path(__file__).parent.parent / "tests" / "test_data" / "github.com" + +#: Paths chosen to exercise the wildcard matcher: each one shares a prefix with +#: several patterns in the corpus, so matching cannot stop at the first +#: comparison. +URLS = [ + "/scrapy/protego/tree/master/src", + "/scrapy/protego/blob/master/README.rst", + "/scrapy/protego/commits/master/setup.py", + "/scrapy/protego/issues/new", + "/scrapy/protego/wiki/Home/History", + "/scrapy/protego/pulse", + "/scrapy/protego/forks", + "/gist/someone/1234/revisions", + "/orgs/scrapy/people", + "/scrapy/protego", +] + + +def content() -> str: + return CORPUS.read_text(encoding="utf-8") + + +def parse(adapter: type, robotstxt: str) -> object: + return adapter(robotstxt) + + +def match(parser: object) -> None: + for url in URLS: + parser.can_fetch(url) diff --git a/benchmarks/compare.py b/benchmarks/compare.py new file mode 100644 index 00000000..dbf3d67d --- /dev/null +++ b/benchmarks/compare.py @@ -0,0 +1,160 @@ +"""Regenerate the parser comparison table of ``README.rst``. + +Run with ``--write`` to update the file in place, otherwise the table is +printed. Measuring requires ``valgrind``. +""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +from functools import cache +from pathlib import Path + +from benchmarks._parsers import ADAPTERS, FEATURES, ProtegoAdapter, supports + +README = Path(__file__).parent.parent / "README.rst" +START = ".. comparison-table-start" +END = ".. comparison-table-end" + +#: Iteration counts for the two-point measurement. Instruction counts are taken +#: at both, and their difference is the cost of BASE iterations alone, which +#: cancels interpreter startup, imports and corpus loading. +BASE = 20 +_REFS = re.compile(rb"^==\d+==\s+Collected\s+:\s+([\d,]+)", re.MULTILINE) + +_SPEC = { + "Protego": "Google_", + "RobotFileParser": "`Martijn Koster's 1996 draft`_", + "robotspy": "`RFC 9309`_", + "Robotexclusionrulesparser": "`Martijn Koster's 1996 draft`_", +} + + +@cache +def _valgrind() -> str: + path = shutil.which("valgrind") + if path is None: + raise SystemExit("valgrind is required to measure instruction counts.") + return path + + +def _instructions(parser: str, bench: str, iterations: int) -> int: + """Return the instructions executed by *iterations* rounds of *bench*.""" + process = subprocess.run( # noqa: S603 + [ + _valgrind(), + "--tool=callgrind", + "--callgrind-out-file=/dev/null", + sys.executable, + "-m", + "benchmarks._run", + parser, + bench, + str(iterations), + ], + capture_output=True, + check=True, + cwd=README.parent, + # Hash randomization perturbs dict and set behaviour, which shows up as + # jitter in instruction counts. + env={**os.environ, "PYTHONHASHSEED": "0"}, + ) + return _refs(process.stderr) + + +def _refs(stderr: bytes) -> int: + """Return the instruction count from callgrind's summary on *stderr*.""" + found = _REFS.search(stderr) + if not found: + raise RuntimeError( + f"No instruction count in callgrind output:\n" + f"{stderr.decode(errors='replace')}" + ) + return int(found.group(1).replace(b",", b"")) + + +def _cost(parser: str, bench: str) -> int: + return _instructions(parser, bench, 2 * BASE) - _instructions(parser, bench, BASE) + + +def _relative(baseline: int, other: int) -> str: + """Return *other* as a percentage speed difference against *baseline*.""" + if other == baseline: + return "0%" + return f"{round((baseline / other - 1) * 100):+d}%" + + +def _measure() -> dict[str, str]: + totals = {} + for adapter in ADAPTERS: + totals[adapter.name] = sum( + _cost(adapter.name, bench) for bench in ("parse", "match") + ) + print(f" {adapter.name}: {totals[adapter.name]:,} instructions", flush=True) + baseline = totals[ProtegoAdapter.name] + return { + name: "" if name == ProtegoAdapter.name else _relative(baseline, total) + for name, total in totals.items() + } + + +def _table(performance: dict[str, str]) -> str: + columns = ["", *(adapter.name for adapter in ADAPTERS)] + rows = [ + ["Version tested", *(adapter.version() for adapter in ADAPTERS)], + ["Reference specification", *(_SPEC[adapter.name] for adapter in ADAPTERS)], + ] + rows += [ + [label, *("✓" if supports(adapter, key) else "" for adapter in ADAPTERS)] + for label, key, _ in FEATURES + ] + rows.append(["Performance", *(performance[adapter.name] for adapter in ADAPTERS)]) + + out = [".. list-table::", " :header-rows: 1", " :stub-columns: 1", ""] + for row in [columns, *rows]: + for index, cell in enumerate(row): + bullet = " *" if index == 0 else " " + out.append(f"{bullet} - {cell}".rstrip()) + return "\n".join(out) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--write", action="store_true", help=f"update {README.name} in place" + ) + args = parser.parse_args() + + _valgrind() + + print("Measuring…", file=sys.stderr) + table = _table(_measure()) + + if not args.write: + print(table) + return 0 + + text = README.read_text(encoding="utf-8") + updated = re.sub( + rf"({re.escape(START)}\n\n).*?(\n\n{re.escape(END)})", + lambda m: m.group(1) + table + m.group(2), + text, + flags=re.DOTALL, + ) + if updated == text: + if START not in text: + raise SystemExit(f"Missing {START} marker in {README}") + print("Table unchanged.") + return 0 + README.write_text(updated, encoding="utf-8") + print(f"Updated {README}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 00000000..dda40ec9 --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1,2 @@ +robotexclusionrulesparser==1.7.1 +robotspy==0.13.0 diff --git a/benchmarks/test_benchmarks.py b/benchmarks/test_benchmarks.py new file mode 100644 index 00000000..fa533a12 --- /dev/null +++ b/benchmarks/test_benchmarks.py @@ -0,0 +1,20 @@ +import pytest + +from benchmarks._parsers import ProtegoAdapter +from benchmarks._workload import content, match, parse + + +@pytest.fixture(scope="module") +def robotstxt() -> str: + return content() + + +@pytest.mark.benchmark +def test_parse(benchmark, robotstxt: str) -> None: + benchmark(parse, ProtegoAdapter, robotstxt) + + +@pytest.mark.benchmark +def test_match(benchmark, robotstxt: str) -> None: + parser = parse(ProtegoAdapter, robotstxt) + benchmark(match, parser) diff --git a/benchmarks/test_comparison.py b/benchmarks/test_comparison.py new file mode 100644 index 00000000..2b84b6b4 --- /dev/null +++ b/benchmarks/test_comparison.py @@ -0,0 +1,67 @@ +import re + +import pytest +from docutils.core import publish_doctree + +from benchmarks._parsers import ADAPTERS, FEATURES, ProtegoAdapter, supports +from benchmarks.compare import README, _refs, _relative, _table + + +@pytest.mark.parametrize("adapter", ADAPTERS, ids=lambda a: a.name) +def test_probes_run(adapter: type) -> None: + for _, key, _probe in FEATURES: + assert isinstance(supports(adapter, key), bool) + + +@pytest.mark.parametrize(("_label", "key", "_probe"), FEATURES) +def test_protego_supports_everything(_label: str, key: str, _probe: object) -> None: + assert supports(ProtegoAdapter, key) + + +# Callgrind's summary, as written to stderr at the end of a run. The counts of +# the "Collected" and "I refs" lines agree; only their formatting differs. +_CALLGRIND_STDERR = b"""==1234== Callgrind, a call-graph generating cache profiler +==1234== Command: python -m benchmarks._run Protego parse 20 +==1234== +==1234== For interactive control, run 'callgrind_control -h'. +==1234== +==1234== Events : Ir +==1234== Collected : 987654321 +==1234== +==1234== I refs: 987,654,321 +""" + + +def test_refs() -> None: + assert _refs(_CALLGRIND_STDERR) == 987654321 + with pytest.raises(RuntimeError, match="No instruction count"): + _refs(b"==1234== valgrind: fatal error\n") + + +def test_relative() -> None: + assert _relative(100, 50) == "+100%" + assert _relative(100, 200) == "-50%" + assert _relative(100, 100) == "0%" + + +def test_table_is_valid_rst() -> None: + """The table parses, and every link it uses is defined in the README.""" + performance = {adapter.name: "+1%" for adapter in ADAPTERS} + performance[ProtegoAdapter.name] = "" + targets = re.findall( + r"^\.\. _.+?: \S+$", README.read_text(encoding="utf-8"), re.MULTILINE + ) + table = _table(performance) + "\n\n" + "\n".join(targets) + + problems = [] + publish_doctree( + table, + settings_overrides={ + "report_level": 2, + "halt_level": 5, + "warning_stream": type( + "_Sink", (), {"write": lambda _self, text: problems.append(text)} + )(), + }, + ) + assert not problems, "".join(problems) diff --git a/tox.ini b/tox.ini index fca57ffd..4699cf06 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,22 @@ deps = commands = pytest --cov=protego --cov-report=xml --cov-report= {posargs:tests} +[testenv:benchmarks] +deps = + pytest + pytest-codspeed==5.0.3 +commands = + pytest --codspeed {posargs:benchmarks/test_benchmarks.py} + +[testenv:comparison] +deps = + -r benchmarks/requirements.txt + docutils + pytest +commands = + pytest benchmarks/test_comparison.py + python -m benchmarks.compare {posargs} + [testenv:pre-commit] deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure From ff222ad8e4a709058ce4ca2d8b21813d9f9ac369 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 25 Aug 2026 10:10:43 +0200 Subject: [PATCH 2/2] Fix CodSpeed support --- .github/workflows/benchmarks.yml | 4 ---- .gitignore | 3 +++ testenv.junit.xml | 1 - tox.ini | 9 +++++++++ 4 files changed, 12 insertions(+), 5 deletions(-) delete mode 100644 testenv.junit.xml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f12eeee3..e408d9e4 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -22,10 +22,6 @@ jobs: permissions: contents: read id-token: write # OIDC authentication with CodSpeed - env: - # Hash randomization perturbs dict and set behaviour, which shows up as - # jitter in instruction counts. - PYTHONHASHSEED: "0" steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: diff --git a/.gitignore b/.gitignore index f095eb15..aa872efb 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ dist/ .coverage .coverage.* coverage.* + +# test reports +testenv.junit.xml diff --git a/testenv.junit.xml b/testenv.junit.xml deleted file mode 100644 index 8e9a228c..00000000 --- a/testenv.junit.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/tox.ini b/tox.ini index a572c579..bdf2874a 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,15 @@ commands = deps = pytest pytest-codspeed==5.0.3 +# The CodSpeed runner drives pytest-codspeed through CODSPEED_* variables, which +# have to reach the command tox runs for any measurement to happen. +passenv = + *codspeed* + *ci* +# Hash randomization perturbs dict and set behaviour, which shows up as +# jitter in instruction counts. +set_env = + PYTHONHASHSEED = 0 commands = pytest --codspeed {posargs:benchmarks/test_benchmarks.py}