Add benchmarks and regenerate the parser comparison from measurements

.github/dependabot.yml

@@ -1,12 +1,19 @@
 version: 2
 updates:
-  - package-ecosystem: github-actions
-    directory: "/"
-    schedule:
-      interval: monthly
-    groups:
-      github-actions:
-        patterns:
-          - "*"
-    cooldown:
-      default-days: 7
+- package-ecosystem: pip
+  directory: /benchmarks
+  schedule:
+    interval: monthly
+  cooldown:
+    default-days: 7
+
+- package-ecosystem: github-actions
+  directory: /
+  schedule:
+    interval: monthly
+  groups:
+    github-actions:
+      patterns:
+      - "*"
+  cooldown:
+    default-days: 7

.github/workflows/benchmarks.yml

@@ -0,0 +1,42 @@
+name: Benchmarks
+on:
+  push:
+    branches: [master]
+  pull_request:
+    paths:
+    - src/**
+    - benchmarks/**
+    - .github/workflows/benchmarks.yml
+    - tox.ini
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  codspeed:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write  # OIDC authentication with CodSpeed
+    steps:
+    - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+      with:
+        persist-credentials: false
+
+    - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+      with:
+        python-version: "3.14.7"
+
+    - name: Install
+      run: |
+        pip install -U tox
+
+    - uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2
+      with:
+        mode: simulation
+        exclude-allocations: true
+        run: tox -e benchmarks

.github/workflows/comparison.yml

@@ -0,0 +1,41 @@
+name: Comparison table
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+    paths:
+    - src/**
+    - benchmarks/**
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  comparison:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+      with:
+        persist-credentials: false
+
+    - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+      with:
+        python-version: "3.14.7"
+
+    - name: Install
+      run: |
+        sudo apt-get update && sudo apt-get install -y valgrind
+        pip install -U tox
+
+    - name: Regenerate the table
+      run: tox -e comparison -- --write
+
+    - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+      with:
+        branch: comparison-table
+        title: Update the parser comparison table
+        commit-message: Update the parser comparison table
+        body: |
+          Regenerated by `benchmarks/compare.py` from commit ${{ github.sha }}.
+        delete-branch: true

.gitignore

@@ -21,3 +21,6 @@ dist/
 .coverage
 .coverage.*
 coverage.*
+
+# test reports
+testenv.junit.xml

README.rst

@@ -86,27 +86,79 @@ Comparison
 ==========
 
 The following table compares Protego to the most popular ``robots.txt`` parsers
-implemented in Python or featuring Python bindings:
-
-+----------------------------+---------+-----------------+--------+---------------------------+
-|                            | Protego | RobotFileParser | Reppy  | Robotexclusionrulesparser |
-+============================+=========+=================+========+===========================+
-| Implementation language    | Python  | Python          | C++    | Python                    |
-+----------------------------+---------+-----------------+--------+---------------------------+
-| Reference specification    | Google_ | `Martijn Koster’s 1996 draft`_                       |
-+----------------------------+---------+-----------------+--------+---------------------------+
-| `Wildcard support`_        | ✓       |                 | ✓      | ✓                         |
-+----------------------------+---------+-----------------+--------+---------------------------+
-| `Length-based precedence`_ | ✓       |                 | ✓      |                           |
-+----------------------------+---------+-----------------+--------+---------------------------+
-| Performance_               |         | +40%            | +1300% | -25%                      |
-+----------------------------+---------+-----------------+--------+---------------------------+
-
-.. _Google: https://developers.google.com/search/reference/robots_txt
-.. _Length-based precedence: https://developers.google.com/search/reference/robots_txt#order-of-precedence-for-group-member-lines
-.. _Martijn Koster’s 1996 draft: https://www.robotstxt.org/norobots-rfc.txt
-.. _Performance: https://anubhavp28.github.io/gsoc-weekly-checkin-12/
-.. _Wildcard support: https://developers.google.com/search/reference/robots_txt#url-matching-based-on-path-values
+implemented in Python. Performance is the speed difference against Protego,
+measured as executed instructions, so a positive value means faster than
+Protego.
+
+.. comparison-table-start
+
+.. list-table::
+   :header-rows: 1
+   :stub-columns: 1
+
+   * -
+     - Protego
+     - RobotFileParser
+     - robotspy
+     - Robotexclusionrulesparser
+   * - Version tested
+     -
+     - Python 3.14.7
+     - 0.13.0
+     - 1.7.1
+   * - Reference specification
+     - Google_
+     - `Martijn Koster's 1996 draft`_
+     - `RFC 9309`_
+     - `Martijn Koster's 1996 draft`_
+   * - `Wildcard support`_
+     - ✓
+     -
+     - ✓
+     - ✓
+   * - `Length-based precedence`_
+     - ✓
+     -
+     - ✓
+     -
+   * - Crawl-delay
+     - ✓
+     - ✓
+     -
+     -
+   * - Request-rate
+     - ✓
+     -
+     -
+     -
+   * - Visit-time
+     - ✓
+     -
+     -
+     -
+   * - Sitemaps
+     - ✓
+     - ✓
+     - ✓
+     - ✓
+   * - Host
+     - ✓
+     -
+     -
+     -
+   * - Performance
+     -
+     - +40%
+     -
+     - -25%
+
+.. comparison-table-end
+
+.. _Google: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec
+.. _Length-based precedence: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec#order-of-precedence-for-rules
+.. _Martijn Koster's 1996 draft: https://www.robotstxt.org/norobots-rfc.txt
+.. _RFC 9309: https://www.rfc-editor.org/rfc/rfc9309
+.. _Wildcard support: https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec#url-matching-based-on-path-values
 
 
 API Reference