Name benchmark runs with the latest PyMuPDF release and branch

Author: hashiromer

.github/scripts/pymupdf_source_stack/README.md

@@ -10,6 +10,14 @@ the adapters in this directory translate GitHub environment variables and
 output files to that shared core. GitHub-only publishing and failure-summary
 behavior remains here.
 
+`launch_benchmark.py` reads the latest published PyMuPDF release tag and sends
+a `parsebench-version-lab` repository dispatch. This supplies the version
+before GitHub evaluates the benchmark's `run-name`, without adding a version
+input to the manual form. The launcher uses its built-in token with
+`contents: write` for dispatch; no extra secret is needed. The benchmark retains
+its workflow identity for indexing, checks out the launcher's exact repository
+commit, and records the launch run ID and release label alongside source provenance.
+
 MuPDF follows the version declared in the selected PyMuPDF commit's `setup.py`;
 developers do not select it separately. The workflow resolves that native
 release tag to a full SHA before checkout, then gives PyMuPDF's build system a reproducible

.github/scripts/pymupdf_source_stack/launch_benchmark.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Resolve the release label before GitHub creates the benchmark run title."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+
+from common import append_summary, env, markdown_cell
+from configure import GROUPS, RUN_SCOPES, selected
+
+
+def main() -> int:
+    branch = env("PYMUPDF_BRANCH").strip()
+    if not branch:
+        raise SystemExit("A PyMuPDF branch name is required")
+    scope = env("RUN_SCOPE_SELECTION")
+    group = env("GROUP_SELECTION")
+    selected(RUN_SCOPES, scope, "test size")
+    selected(GROUPS, group, "document category")
+    release_result = subprocess.run(
+        ["gh", "api", "repos/pymupdf/PyMuPDF/releases/latest"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    release = json.loads(release_result.stdout)
+    tag = release.get("tag_name")
+    if not isinstance(tag, str) or not re.fullmatch(r"v?\d+(?:\.\d+)+", tag):
+        raise SystemExit(f"Latest PyMuPDF release has an unsupported tag: {tag!r}")
+    repository = env("GITHUB_REPOSITORY")
+    payload = {
+        "event_type": "parsebench-version-lab",
+        "client_payload": {
+            "pymupdf_version": tag.removeprefix("v"),
+            "pymupdf_tag": tag,
+            "pymupdf_branch": branch,
+            "run_scope": scope,
+            "group": group,
+            "benchmark_ref": env("GITHUB_REF_NAME"),
+            "benchmark_sha": env("GITHUB_SHA"),
+            "launcher_run_id": env("GITHUB_RUN_ID"),
+        },
+    }
+    subprocess.run(
+        ["gh", "api", "--method", "POST", f"repos/{repository}/dispatches", "--input", "-"],
+        input=json.dumps(payload),
+        text=True,
+        check=True,
+    )
+    benchmark_url = (
+        f"{env('GITHUB_SERVER_URL')}/{repository}/actions/workflows/pymupdf-source-stack-parsebench.yml"
+    )
+    append_summary(
+        [
+            "## Benchmark launched",
+            "",
+            f"[Open benchmark runs]({benchmark_url})",
+            "",
+            f"- PyMuPDF release label: **{markdown_cell(tag)}**",
+            f"- Branch to build: **{markdown_cell(branch)}**",
+            f"- Test size: {markdown_cell(scope)}",
+            f"- Document category: {markdown_cell(group)}",
+            "",
+            "The benchmark runs separately. Its conclusion and results appear in the linked workflow.",
+        ]
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/pymupdf_source_stack/write_run_metadata.py

@@ -20,6 +20,8 @@ def main() -> int:
         "pipeline": env("PIPELINE"),
         "benchmark_ref_input": env("BENCHMARK_REF"),
         "checked_out_sha": git_sha(),
+        "pymupdf_release_tag": os.environ.get("PYMUPDF_RELEASE_TAG"),
+        "launcher_run_id": os.environ.get("LAUNCHER_RUN_ID"),
         "github_ref": env("GITHUB_REF"),
         "github_sha": env("GITHUB_SHA"),
         "github_repository": env("GITHUB_REPOSITORY"),

.github/workflows/pymupdf-source-stack-launch.yml

@@ -0,0 +1,61 @@
+name: Run ParseBench Version Lab
+
+run-name: >-
+  ${{ format('Launch PyMuPDF · {0} · {1} · {2}', inputs.pymupdf_branch,
+      inputs.run_scope == 'Quick test (15 cases)' && 'Quick 15' || 'Full ~2k',
+      inputs.group == 'All categories' && 'All' || inputs.group) }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      pymupdf_branch:
+        description: "PyMuPDF branch to benchmark (for example main or your feature branch)."
+        required: true
+        default: "main"
+        type: string
+      run_scope:
+        description: "How much data should be tested?"
+        required: true
+        default: "Quick test (15 cases)"
+        type: choice
+        options:
+          - "Quick test (15 cases)"
+          - "Full benchmark (~2,000 pages)"
+      group:
+        description: "Which document category should be tested?"
+        required: true
+        default: "All categories"
+        type: choice
+        options:
+          - "All categories"
+          - Charts
+          - Tables
+          - "Page layout"
+          - "Text content"
+          - "Text formatting"
+
+permissions:
+  contents: write # Required to send the repository_dispatch event.
+
+jobs:
+  launch:
+    name: Resolve latest PyMuPDF release and start benchmark
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - name: Checkout launcher
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+          sparse-checkout: |
+            .github/scripts/pymupdf_source_stack
+            tools/version_lab/src
+
+      - name: Launch named benchmark
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PYMUPDF_BRANCH: ${{ inputs.pymupdf_branch }}
+          RUN_SCOPE_SELECTION: ${{ inputs.run_scope }}
+          GROUP_SELECTION: ${{ inputs.group }}
+        run: python3 .github/scripts/pymupdf_source_stack/launch_benchmark.py

.github/workflows/pymupdf-source-stack-parsebench.yml

@@ -3,50 +3,28 @@ name: ParseBench Version Lab
 run-name: >-
   ${{
     format(
-      'pymupdf4llm_html_tables_rapidocr_v3 · {0} · {1} · {2}',
-      inputs.pymupdf_branch,
-      inputs.run_scope == 'Quick test (15 cases)' && 'Quick 15' || 'Full ~2k',
-      inputs.group == 'All categories' && 'All' ||
-        inputs.group == 'Page layout' && 'Layout' ||
-        inputs.group
+      'PyMuPDF {0} · {1} · {2} · {3}',
+      github.event.client_payload.pymupdf_version,
+      github.event.client_payload.pymupdf_branch,
+      github.event.client_payload.run_scope == 'Quick test (15 cases)' && 'Quick 15' || 'Full ~2k',
+      github.event.client_payload.group == 'All categories' && 'All' ||
+        github.event.client_payload.group == 'Page layout' && 'Layout' ||
+        github.event.client_payload.group
     )
   }}
 
 on:
-  workflow_dispatch:
-    inputs:
-      pymupdf_branch:
-        description: "PyMuPDF branch to benchmark (for example main or your feature branch)."
-        required: true
-        default: "main"
-        type: string
-      run_scope:
-        description: "How much data should be tested?"
-        required: true
-        default: "Quick test (15 cases)"
-        type: choice
-        options:
-          - "Quick test (15 cases)"
-          - "Full benchmark (~2,000 pages)"
-      group:
-        description: "Which document category should be tested?"
-        required: true
-        default: "All categories"
-        type: choice
-        options:
-          - "All categories"
-          - Charts
-          - Tables
-          - "Page layout"
-          - "Text content"
-          - "Text formatting"
+  repository_dispatch:
+    types: [parsebench-version-lab]
 
 permissions:
   contents: read
   packages: read
 
 env:
   PIPELINE: pymupdf4llm_html_tables_rapidocr_v3
+  PYMUPDF_RELEASE_TAG: ${{ github.event.client_payload.pymupdf_tag }}
+  LAUNCHER_RUN_ID: ${{ github.event.client_payload.launcher_run_id }}
   GCP_PROJECT_ID: pypi-download-analysis-457820
   GCS_PREFIX: parsebench/pymupdf_source_stack
   UV_NO_SYNC: "1"
@@ -72,14 +50,17 @@ jobs:
     steps:
       - name: Checkout ParseBench
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.client_payload.benchmark_sha }}
+          persist-credentials: false
 
       - name: Initialize run configuration
         id: config
         env:
-          BENCHMARK_REF: ${{ github.ref_name }}
-          GROUP_SELECTION: ${{ inputs.group }}
-          PYMUPDF_BRANCH: ${{ inputs.pymupdf_branch }}
-          RUN_SCOPE_SELECTION: ${{ inputs.run_scope }}
+          BENCHMARK_REF: ${{ github.event.client_payload.benchmark_ref }}
+          GROUP_SELECTION: ${{ github.event.client_payload.group }}
+          PYMUPDF_BRANCH: ${{ github.event.client_payload.pymupdf_branch }}
+          RUN_SCOPE_SELECTION: ${{ github.event.client_payload.run_scope }}
         run: python3 "$WORKFLOW_SCRIPTS/configure.py"
 
       - name: Resolve PyMuPDF branch and bundled sources
@@ -177,13 +158,13 @@ jobs:
 
       - name: Add selected configuration to workflow summary
         env:
-          BENCHMARK_REF: ${{ github.ref_name }}
+          BENCHMARK_REF: ${{ github.event.client_payload.benchmark_ref }}
           DATASET_REPOSITORY: ${{ steps.dataset.outputs.repository }}
           DATASET_REQUESTED_REF: ${{ steps.dataset.outputs.requested_ref }}
           DATASET_SHA: ${{ steps.dataset.outputs.sha }}
-          GROUP_SELECTION: ${{ inputs.group }}
+          GROUP_SELECTION: ${{ github.event.client_payload.group }}
           OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
-          RUN_SCOPE_SELECTION: ${{ inputs.run_scope }}
+          RUN_SCOPE_SELECTION: ${{ github.event.client_payload.run_scope }}
         run: python3 "$WORKFLOW_SCRIPTS/write_run_summary.py"
 
       - name: Install ParseBench dependencies without selected PyMuPDF packages
@@ -313,6 +294,7 @@ jobs:
       - name: Checkout publishing helper
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
+          ref: ${{ github.event.client_payload.benchmark_sha }}
           persist-credentials: false
           sparse-checkout: |
             .github/scripts/pymupdf_source_stack
@@ -357,6 +339,7 @@ jobs:
       - name: Checkout summary helpers
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
+          ref: ${{ github.event.client_payload.benchmark_sha }}
           persist-credentials: false
           sparse-checkout: |
             .github/scripts/pymupdf_source_stack

tests/test_pymupdf_source_stack_workflow.py

@@ -27,6 +27,7 @@ def _load_module(name: str):
 
 
 configure = _load_module("configure")
+launcher = _load_module("launch_benchmark")
 benchmark = importlib.import_module("parsebench_version_lab.runtime_benchmark")
 provenance = importlib.import_module("parsebench_version_lab.provenance")
 resolve_dataset = _load_module("resolve_dataset")
@@ -48,6 +49,57 @@ def test_safe_ref_produces_bounded_path_component(value: str, expected: str) ->
     assert configure.safe_ref(value) == expected
 
 
+@pytest.mark.parametrize("tag", ["2.0", "v2.1", None])
+def test_launcher_resolves_release_before_dispatch_and_preserves_selected_branch(
+    tag: str | None, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    for name, value in {
+        "PYMUPDF_BRANCH": "feature/ocr",
+        "RUN_SCOPE_SELECTION": "Full benchmark (~2,000 pages)",
+        "GROUP_SELECTION": "All categories",
+        "GITHUB_REPOSITORY": "owner/repo",
+        "GITHUB_REF_NAME": "feature/workflow",
+        "GITHUB_SHA": "a" * 40,
+        "GITHUB_RUN_ID": "123",
+        "GITHUB_SERVER_URL": "https://github.com",
+        "GITHUB_STEP_SUMMARY": str(tmp_path / "summary"),
+    }.items():
+        monkeypatch.setenv(name, value)
+
+    calls = []
+
+    def run(command, **kwargs):
+        calls.append(command)
+        if len(calls) == 1:
+            assert command == ["gh", "api", "repos/pymupdf/PyMuPDF/releases/latest"]
+            return subprocess.CompletedProcess(command, 0, stdout=json.dumps({"tag_name": tag}))
+        assert command == ["gh", "api", "--method", "POST", "repos/owner/repo/dispatches", "--input", "-"]
+        assert json.loads(kwargs["input"]) == {
+            "event_type": "parsebench-version-lab",
+            "client_payload": {
+                "pymupdf_version": tag.removeprefix("v"),
+                "pymupdf_tag": tag,
+                "pymupdf_branch": "feature/ocr",
+                "run_scope": "Full benchmark (~2,000 pages)",
+                "group": "All categories",
+                "benchmark_ref": "feature/workflow",
+                "benchmark_sha": "a" * 40,
+                "launcher_run_id": "123",
+            },
+        }
+        return subprocess.CompletedProcess(command, 0)
+
+    monkeypatch.setattr(launcher.subprocess, "run", run)
+    if tag is None:
+        with pytest.raises(SystemExit, match="unsupported tag"):
+            launcher.main()
+        assert len(calls) == 1
+    else:
+        assert launcher.main() == 0
+        assert len(calls) == 2
+        assert "pymupdf-source-stack-parsebench.yml" in (tmp_path / "summary").read_text()
+
+
 def test_configure_maps_friendly_inputs_and_records_request(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "summary"))
     github_output = tmp_path / "github-output"

tools/version_lab/README.md

@@ -123,6 +123,18 @@ Hugging Face commit for the entire run.
 
 ## Run
 
+In GitHub Actions, select **Run ParseBench Version Lab** and click **Run workflow**.
+The launcher keeps the three inputs: PyMuPDF branch, test size, and category.
+It reads PyMuPDF's latest published GitHub release tag, then starts a separate
+**ParseBench Version Lab** benchmark named, for example,
+`PyMuPDF 2.0 · main · Full ~2k · All`. The launcher summary links to benchmark runs.
+The version in the title is the latest release label; the selected branch is
+still built and its exact installed version and commit are recorded in the results.
+The benchmark workflow definition runs from the default branch; its code and
+helpers are checked out at the immutable repository commit chosen in the launcher.
+Only the benchmark run is ingested into Supabase. The launcher finishing means
+the benchmark was requested; check the benchmark run for its final outcome.
+
 Run all commands from the repository root.
 
 Check the required tools:

tools/version_lab/src/parsebench_version_lab/benchmark_index.py

@@ -17,6 +17,7 @@
 from typing import Any, Protocol
 
 from .coverage import execution_coverage
+from .model import DEFAULT_PIPELINE
 from .results import finite_number, headline_metric, report_dimension
 
 KNOWN_DIMENSIONS = ("chart", "table", "layout", "text_content", "text_formatting")
@@ -395,6 +396,8 @@ def _pipeline_from_run_name(run: Mapping[str, Any]) -> str | None:
     if not isinstance(title, str):
         return None
     first = title.split(" · ", 1)[0].strip()
+    if first == "PyMuPDF" or first.startswith("PyMuPDF "):
+        return DEFAULT_PIPELINE
     return first if first.startswith("pymupdf") else None
 
 
@@ -622,8 +625,8 @@ def index_run(
             "dataset_version_id": dataset_id,
             "gcs_bucket": gcs_bucket,
             "gcs_prefix": gcs_prefix,
-            "head_branch": github_run.get("head_branch"),
-            "head_sha": github_run.get("head_sha"),
+            "head_branch": run_metadata.get("benchmark_ref_input") or github_run.get("head_branch"),
+            "head_sha": run_metadata.get("checked_out_sha") or github_run.get("head_sha"),
             "source_created_at": github_run.get("created_at"),
             "source_updated_at": github_run.get("updated_at"),
             "started_at": github_run.get("run_started_at"),