pymupdf-source-stack-parsebench.yml

name: ParseBench Version Lab

run-name: >-
  ${{
    format(
      'PyMuPDF {0} · {1} · {2} · {3}',
      github.event.client_payload.pymupdf_version,
      github.event.client_payload.pymupdf_branch,
      github.event.client_payload.run_scope == 'Quick test (15 cases)' && 'Quick 15' || 'Full ~2k',
      github.event.client_payload.group == 'All categories' && 'All' ||
        github.event.client_payload.group == 'Page layout' && 'Layout' ||
        github.event.client_payload.group
    )
  }}

on:
  repository_dispatch:
    types: [parsebench-version-lab]

permissions:
  contents: read
  packages: read

env:
  PIPELINE: pymupdf4llm_html_tables_rapidocr_v3
  PYMUPDF_RELEASE_TAG: ${{ github.event.client_payload.pymupdf_tag }}
  LAUNCHER_RUN_ID: ${{ github.event.client_payload.launcher_run_id }}
  GCP_PROJECT_ID: pypi-download-analysis-457820
  GCS_PREFIX: parsebench/pymupdf_source_stack
  UV_NO_SYNC: "1"
  WORKFLOW_SCRIPTS: .github/scripts/pymupdf_source_stack

jobs:
  benchmark:
    name: Check compatibility and run benchmark in pinned container
    runs-on: ubuntu-24.04
    timeout-minutes: 720
    container:
      image: ghcr.io/pymupdf/parsebench-source-stack-env@sha256:e41e0c615011482a18ef950dc49f5befcd2700752eeab11670e8c64a9d9d9d07
      credentials:
        username: ${{ github.actor }}
        password: ${{ github.token }}
    defaults:
      run:
        shell: bash
    outputs:
      artifact_name: ${{ steps.config.outputs.artifact_name }}
      destination: ${{ steps.config.outputs.destination }}

    steps:
      - name: Checkout ParseBench
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.client_payload.benchmark_sha }}
          persist-credentials: false

      - name: Initialize run configuration
        id: config
        env:
          BENCHMARK_REF: ${{ github.event.client_payload.benchmark_ref }}
          GROUP_SELECTION: ${{ github.event.client_payload.group }}
          PYMUPDF_BRANCH: ${{ github.event.client_payload.pymupdf_branch }}
          RUN_SCOPE_SELECTION: ${{ github.event.client_payload.run_scope }}
        run: python3 "$WORKFLOW_SCRIPTS/configure.py"

      - name: Resolve PyMuPDF branch and bundled sources
        id: refs
        env:
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: python3 "$WORKFLOW_SCRIPTS/resolve_refs.py"

      - name: Record and verify pinned container environment
        env:
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: |
          cp /usr/local/share/parsebench/environment.txt "$OUTPUT_DIR/_container_environment.txt"
          test "$(python --version)" = "Python 3.12.13"
          test "$(uv --version | cut -d' ' -f2)" = "0.11.29"
          test "$(dpkg-query -W -f='${Version}' tesseract-ocr)" = "5.3.4-1build5"
          test "$(dpkg-query -W -f='${Version}' tesseract-ocr-eng)" = "1:4.1.0-2"
          test "$(dpkg-query -W -f='${Version}' unzip)" = "6.0-28ubuntu4.1"
          tesseract --list-langs | grep -Fx eng
          unzip -v | sed -n '1p'
          {
            echo "## Container environment"
            echo
            echo "- Image: \`ghcr.io/pymupdf/parsebench-source-stack-env\`"
            echo "- Digest: \`sha256:e41e0c615011482a18ef950dc49f5befcd2700752eeab11670e8c64a9d9d9d07\`"
            echo "- Python: \`$(python --version)\`"
            echo "- Tesseract package: \`$(dpkg-query -W -f='${Version}' tesseract-ocr)\`"
            echo
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Resolve exact ParseBench dataset commit
        id: dataset
        env:
          DATASET_REF: ${{ steps.config.outputs.dataset_ref }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
          RUN_SCOPE: ${{ steps.config.outputs.run_scope }}
        run: python3 "$WORKFLOW_SCRIPTS/resolve_dataset.py"

      - name: Restore immutable ParseBench dataset cache
        id: dataset-cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ steps.config.outputs.data_dir }}
          key: parsebench-dataset-${{ steps.dataset.outputs.sha }}

      - name: Checkout PyMuPDF
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: pymupdf/PyMuPDF
          ref: ${{ steps.refs.outputs.pymupdf_sha }}
          path: .source/pymupdf
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout MuPDF
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ArtifexSoftware/mupdf
          ref: ${{ steps.refs.outputs.mupdf_sha }}
          path: .source/mupdf
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout PyMuPDF Layout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ArtifexSoftware/pymupdf_layout
          ref: ${{ steps.refs.outputs.pymupdf_layout_sha }}
          path: .source/pymupdf-layout
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout PyMuPDF4LLM
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: pymupdf/pymupdf4llm
          ref: ${{ steps.refs.outputs.pymupdf4llm_sha }}
          path: .source/pymupdf4llm
          fetch-depth: 1
          persist-credentials: false

      - name: Trust container-mounted Git checkouts
        run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global --add safe.directory "$GITHUB_WORKSPACE/.source/mupdf"
          git config --global --add safe.directory "$GITHUB_WORKSPACE/.source/pymupdf"
          git config --global --add safe.directory "$GITHUB_WORKSPACE/.source/pymupdf-layout"
          git config --global --add safe.directory "$GITHUB_WORKSPACE/.source/pymupdf4llm"

      - name: Verify pinned source checkouts
        id: sources
        env:
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: python3 "$WORKFLOW_SCRIPTS/resolve_sources.py"

      - name: Add selected configuration to workflow summary
        env:
          BENCHMARK_REF: ${{ github.event.client_payload.benchmark_ref }}
          DATASET_REPOSITORY: ${{ steps.dataset.outputs.repository }}
          DATASET_REQUESTED_REF: ${{ steps.dataset.outputs.requested_ref }}
          DATASET_SHA: ${{ steps.dataset.outputs.sha }}
          GROUP_SELECTION: ${{ github.event.client_payload.group }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
          RUN_SCOPE_SELECTION: ${{ github.event.client_payload.run_scope }}
        run: python3 "$WORKFLOW_SCRIPTS/write_run_summary.py"

      - name: Install ParseBench dependencies without selected PyMuPDF packages
        run: |
          export UV_PROJECT_ENVIRONMENT="$GITHUB_WORKSPACE/.venv"
          uv sync --project tools/version_lab --locked \
            --no-install-package pymupdf \
            --no-install-package pymupdf-layout \
            --no-install-package pymupdf4llm
          uv pip install --python .venv "swig==4.3.1"
          .venv/bin/swig -version

      - name: Verify RapidOCR native runtime
        run: >-
          .venv/bin/python -c
          "from importlib.metadata import version; from rapidocr import RapidOCR; import onnxruntime;
          print(f'rapidocr={version(\"rapidocr\")} onnxruntime={onnxruntime.__version__} engine={RapidOCR}')"

      - name: Build and install integrated PyMuPDF
        env:
          PACKAGE_DIR: ${{ steps.sources.outputs.pymupdf_dir }}
          PYMUPDF_SETUP_MUPDF_BUILD: ${{ steps.sources.outputs.mupdf_build }}
          PYMUPDF_SETUP_SWIG: ${{ github.workspace }}/.venv/bin/swig
          PYMUPDF_SETUP_PATH_LAYOUT: ${{ steps.sources.outputs.pymupdf_layout_dir }}
          PYMUPDF_SETUP_PATH_4LLM: ${{ steps.sources.outputs.pymupdf4llm_dir }}
        run: uv pip install --python .venv --reinstall --no-deps "$PACKAGE_DIR"

      - name: Check source stack compatibility
        env:
          MUPDF_REF: ${{ steps.refs.outputs.mupdf_ref }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
          PYMUPDF4LLM_REF: ${{ steps.refs.outputs.pymupdf4llm_ref }}
          PYMUPDF_LAYOUT_REF: ${{ steps.refs.outputs.pymupdf_layout_ref }}
          PYMUPDF_LAYOUT_REPOSITORY: ArtifexSoftware/pymupdf_layout
          PYMUPDF_REF: ${{ steps.refs.outputs.pymupdf_ref }}
        run: |
          .venv/bin/python "$WORKFLOW_SCRIPTS/check_compatibility.py" \
            --output "$OUTPUT_DIR/_compatibility.json" \
            --mupdf-repository ArtifexSoftware/mupdf \
            --mupdf-ref "$MUPDF_REF" \
            --mupdf-sha "${{ steps.sources.outputs.mupdf_sha }}" \
            --pymupdf-repository pymupdf/PyMuPDF \
            --pymupdf-ref "$PYMUPDF_REF" \
            --pymupdf-sha "${{ steps.sources.outputs.pymupdf_sha }}" \
            --pymupdf-layout-repository "$PYMUPDF_LAYOUT_REPOSITORY" \
            --pymupdf-layout-ref "$PYMUPDF_LAYOUT_REF" \
            --pymupdf-layout-sha "${{ steps.sources.outputs.pymupdf_layout_sha }}" \
            --pymupdf4llm-repository pymupdf/pymupdf4llm \
            --pymupdf4llm-ref "$PYMUPDF4LLM_REF" \
            --pymupdf4llm-sha "${{ steps.sources.outputs.pymupdf4llm_sha }}"

      - name: Prepare versioned ParseBench dataset
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          # Avoid per-file Xet token API calls on shared runner IPs.
          HF_HUB_DISABLE_XET: "1"
          DATA_DIR: ${{ steps.config.outputs.data_dir }}
          DATASET_REPOSITORY: ${{ steps.dataset.outputs.repository }}
          DATASET_SHA: ${{ steps.dataset.outputs.sha }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: .venv/bin/python "$WORKFLOW_SCRIPTS/benchmark.py" download

      - name: Write run metadata
        env:
          BENCHMARK_REF: ${{ steps.config.outputs.benchmark_ref }}
          DATASET_BRANCH: ${{ steps.dataset.outputs.branch }}
          DATASET_COMMIT_URL: ${{ steps.dataset.outputs.commit_url }}
          DATASET_REPOSITORY: ${{ steps.dataset.outputs.repository }}
          DATASET_REQUESTED_REF: ${{ steps.dataset.outputs.requested_ref }}
          DATASET_SHA: ${{ steps.dataset.outputs.sha }}
          DESTINATION: ${{ steps.config.outputs.destination }}
          GROUP: ${{ steps.config.outputs.group }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
          PARSEBENCH_GCS_BUCKET: ${{ vars.PARSEBENCH_GCS_BUCKET }}
          RUN_SCOPE: ${{ steps.config.outputs.run_scope }}
        run: .venv/bin/python "$WORKFLOW_SCRIPTS/write_run_metadata.py"

      - name: Run selected PyMuPDF4LLM pipeline inference
        env:
          DATA_DIR: ${{ steps.config.outputs.data_dir }}
          GROUP: ${{ steps.config.outputs.group }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: .venv/bin/python "$WORKFLOW_SCRIPTS/benchmark.py" inference

      - name: Run ParseBench evaluation
        env:
          DATA_DIR: ${{ steps.config.outputs.data_dir }}
          GROUP: ${{ steps.config.outputs.group }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: .venv/bin/python "$WORKFLOW_SCRIPTS/benchmark.py" evaluate

      - name: Generate ParseBench reports
        env:
          DATA_DIR: ${{ steps.config.outputs.data_dir }}
          GROUP: ${{ steps.config.outputs.group }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: .venv/bin/python "$WORKFLOW_SCRIPTS/benchmark.py" report

      - name: Prepare dashboard diagnostics
        env:
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
        run: .venv/bin/python -m parsebench_version_lab.dashboard_diagnostics "$OUTPUT_DIR"

      - name: Add aggregate results to workflow summary
        env:
          GROUP: ${{ steps.config.outputs.group }}
          OUTPUT_DIR: ${{ steps.config.outputs.output_dir }}
          RUN_SCOPE: ${{ steps.config.outputs.run_scope }}
        run: .venv/bin/python "$WORKFLOW_SCRIPTS/write_results_summary.py"

      - name: Upload source-stack GitHub artifact
        if: ${{ always() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ steps.config.outputs.artifact_name }}
          path: ${{ steps.config.outputs.output_dir }}
          if-no-files-found: error

  publish:
    name: Publish results or diagnostics
    needs: benchmark
    if: ${{ always() && needs.benchmark.outputs.artifact_name != '' }}
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout publishing helper
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.client_payload.benchmark_sha }}
          persist-credentials: false
          sparse-checkout: |
            .github/scripts/pymupdf_source_stack
            tools/version_lab/src

      - name: Download benchmark artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ needs.benchmark.outputs.artifact_name }}
          path: parsebench-output

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.GCP_SA_KEY }}
          project_id: ${{ env.GCP_PROJECT_ID }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1

      - name: Publish results or failure diagnostics
        env:
          BENCHMARK_RESULT: ${{ needs.benchmark.result }}
          BUCKET: ${{ vars.PARSEBENCH_GCS_BUCKET }}
          DESTINATION: ${{ needs.benchmark.outputs.destination }}
          OUTPUT_DIR: parsebench-output
        run: python3 "$WORKFLOW_SCRIPTS/publish.py"

  report_outcome:
    name: Report final outcome
    needs:
      - benchmark
      - publish
    if: ${{ always() }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      actions: read
      contents: read

    steps:
      - name: Checkout summary helpers
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.client_payload.benchmark_sha }}
          persist-credentials: false
          sparse-checkout: |
            .github/scripts/pymupdf_source_stack
            tools/version_lab/src

      - name: Report successful completion
        if: ${{ needs.benchmark.result == 'success' && needs.publish.result == 'success' }}
        run: python3 "$WORKFLOW_SCRIPTS/write_outcome.py" success

      - name: Download diagnostic artifact
        if: >-
          ${{
            (needs.benchmark.result == 'failure' || needs.publish.result == 'failure') &&
            needs.benchmark.outputs.artifact_name != ''
          }}
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ needs.benchmark.outputs.artifact_name }}
          path: failure-diagnostics

      - name: Add error details to workflow summary
        if: ${{ needs.benchmark.result == 'failure' || needs.publish.result == 'failure' }}
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          python3 "$WORKFLOW_SCRIPTS/write_failure_summary.py" \
            --diagnostics failure-diagnostics \
            --output "$GITHUB_STEP_SUMMARY" \
            --benchmark-result "${{ needs.benchmark.result }}" \
            --publish-result "${{ needs.publish.result }}"

      - name: Report incomplete workflow
        if: >-
          ${{
            needs.benchmark.result != 'failure' &&
            needs.publish.result != 'failure' &&
            (needs.benchmark.result != 'success' || needs.publish.result != 'success')
          }}
        run: |
          python3 "$WORKFLOW_SCRIPTS/write_outcome.py" incomplete \
            --benchmark-result "${{ needs.benchmark.result }}" \
            --publish-result "${{ needs.publish.result }}"