benchmark.yml

name: Engine Benchmark

# #2125: times every extraction engine on every interpreter the engines are
# tested on, so the speed tables of docs/source/engines.rst have a weekly
# measurement to be copied from rather than waiting on a hand run (#2109).
# Natively, through examples/benchmark/native.sh rather than run.sh: a hosted
# runner is already a fresh VM per job, so containers would add a build per
# version and no isolation, and a native run makes macOS and Windows runners
# candidates too. One job measures every
# version, since the per-version table's absolute times compare only within
# one machine: by default `ubuntu-26.04-arm`, the fastest runner that
# measures every environment and engine. What a native run does not pin --
# the interpreter's patch release, tshark and libpcap -- is recorded in the
# tables; examples/benchmark/README.md's "On GitHub Actions" says more, and
# gives the runner figures.
#
# The `discussion` job posts the ratio table to this job's thread in the
# Discussions category REPORT_CATEGORY names (#2030); it alone holds a write
# scope, and a report it cannot post is a warning. Nothing here pushes, and the
# docs stay hand-updated from the report.
on:
  schedule:
    # Wednesdays 03:17 UTC: off the hour, where GitHub delays scheduled runs
    # most, and midweek, clear of the Saturday crons (#1967), Monday's Python
    # Versions and CI Timing, and Project Status at 00:37.
    - cron: '17 3 * * 3'
  workflow_dispatch:
    inputs:
      runner:
        description: Runner to measure on; dispatch once per runner to compare them.
        type: choice
        default: ubuntu-26.04-arm
        options:
          - ubuntu-26.04-arm
          - ubuntu-latest
          - macos-latest
          - windows-latest
      pythons:
        description: Comma-separated Python versions to measure (empty = every version the engines are tested on).
        type: string
        default: ''
      engines:
        description: Comma-separated engines, including default (empty = all).
        type: string
        default: ''
      rounds:
        description: Timed extractions per engine per pass (empty = benchmark.py's default).
        type: string
        default: ''
      repeats:
        description: Passes over the whole engine set (empty = benchmark.py's default).
        type: string
        default: ''

permissions:
  contents: read

# One run per runner at a time; dispatches on different runners still run side
# by side, which is how two runners are compared.
concurrency:
  group: benchmark-${{ inputs.runner || 'ubuntu-26.04-arm' }}
  cancel-in-progress: false

# #2030: one Discussions category holds every report with no pull request to
# comment on, one thread per job.
env:
  REPORT_CATEGORY: CI reports

jobs:
  # #1660: the interpreters are the engine tiers of .github/python-versions.toml,
  # as unit-tests.yml's `engine-tests` installs them.
  python-versions:
    name: Read the Python versions
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      engine-supported: ${{ steps.read.outputs.engine-supported }}
      engine-ktlo: ${{ steps.read.outputs.engine-ktlo }}

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          sparse-checkout: |
            .github/python-versions.toml
            util/_actions.py
            util/python_versions.py
          sparse-checkout-cone-mode: false
          persist-credentials: false

      - name: Read .github/python-versions.toml
        id: read
        run: python3 util/python_versions.py --github-output

  benchmark:
    name: Benchmark on ${{ matrix.os }}
    needs: [ python-versions ]
    # One leg, on the dispatch's runner, or `ubuntu-26.04-arm` on the schedule,
    # which has no inputs; spelt as the other matrix jobs spell their runner.
    runs-on: ${{ matrix.os || 'ubuntu-26.04-arm' }}
    # Run 40062653380 measured this job at about 38 minutes on `ubuntu-26.04-arm`
    # with every version and engine, 04:36:11Z to 05:14:34Z, nearly all of it a
    # tshark per pyshark extraction. `ubuntu-latest` took about 30% longer per
    # packet in run 40047570595, so roughly 50 minutes by that ratio; 90 leaves
    # room for that or a busy host, and stops a hung extraction in an hour and
    # a half rather than at the six-hour job limit.
    timeout-minutes: 90
    strategy:
      matrix:
        os: [ "${{ inputs.runner || 'ubuntu-26.04-arm' }}" ]
    defaults:
      run:
        shell: bash
    outputs:
      status: ${{ steps.measure.outputs.status }}
    env:
      # Read by util/apt_packages.py install: libpcap's headers for pypcap's
      # build, and tshark for pyshark.
      APT_PACKAGES: libpcap-dev tshark
      APT_SET: benchmark

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # As in `engine-tests`: each interpreter is then reachable, and the KTLO
      # one gets a step of its own so that losing it cannot fail the job.
      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        if: ${{ needs.python-versions.outputs.engine-ktlo != '' }}
        continue-on-error: true
        with:
          python-version: ${{ needs.python-versions.outputs.engine-ktlo }}

      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        with:
          python-version: ${{ needs.python-versions.outputs.engine-supported }}

      # Without its system package an engine is a row with its reason, so a
      # failed install warns rather than failing the job.
      - name: Install libpcap headers and tshark
        if: runner.os == 'Linux'
        timeout-minutes: 10
        run: |
          /usr/bin/python3 util/apt_packages.py install \
            || echo '::warning title=System packages not installed::pypcap and pyshark may be reported unmeasured.'

      # #2019: tshark's AppArmor profile on Ubuntu 25.04 and later; see
      # unit-tests.yml's `engine-tests` step of the same name, whose body this is.
      - name: Let tshark read the checkout past its AppArmor profile
        if: runner.os == 'Linux'
        run: |
          set -eu
          if [ -f /etc/apparmor.d/tshark ]; then
            sudo mkdir -p /etc/apparmor.d/local
            echo "file r ${GITHUB_WORKSPACE}/**," | sudo tee -a /etc/apparmor.d/local/tshark >/dev/null
            sudo apparmor_parser -r /etc/apparmor.d/tshark
          fi

      # macOS ships libpcap; Homebrew's `wireshark` formula is the tshark CLI.
      - name: Install tshark with Homebrew
        if: runner.os == 'macOS'
        timeout-minutes: 20
        env:
          HOMEBREW_NO_AUTO_UPDATE: '1'
          HOMEBREW_NO_INSTALL_CLEANUP: '1'
        run: |
          brew install wireshark \
            || echo '::warning title=tshark not installed::pyshark will be reported unmeasured.'

      # Reading a file needs no capture driver. Npcap's free edition cannot
      # install unattended, so the pcap engines report the missing libpcap.
      - name: Install tshark with Chocolatey
        if: runner.os == 'Windows'
        timeout-minutes: 15
        run: |
          if choco install wireshark --yes --no-progress; then
            echo 'C:\Program Files\Wireshark' >> "$GITHUB_PATH"
          else
            echo '::warning title=tshark not installed::pyshark will be reported unmeasured.'
          fi

      # native.sh's exit status 3 is a table with gaps, each explained in it,
      # so it warns; anything else but 0 is no table and fails.
      - name: Run examples/benchmark/native.sh
        id: measure
        env:
          INSTALLED: ${{ needs.python-versions.outputs.engine-ktlo }} ${{ needs.python-versions.outputs.engine-supported }}
          PYTHONS: ${{ inputs.pythons }}
          ENGINES: ${{ inputs.engines }}
          ROUNDS: ${{ inputs.rounds }}
          REPEATS: ${{ inputs.repeats }}
          RUNNER_LABEL: ${{ matrix.os }}
        run: |
          set -uo pipefail
          # `engine-supported` is a multi-line output, one version a line, and an
          # input may say `3.11, 3.12`, so both are split on commas and whitespace.
          read -ra installed <<< "$(echo "$INSTALLED" | tr -s ',[:space:]' ' ')"
          read -ra wanted <<< "$(echo "${PYTHONS:-${installed[*]}}" | tr -s ',[:space:]' ' ')"
          for version in "${wanted[@]}"; do
            case " ${installed[*]} " in
              *" $version "*) ;;
              *) echo "::error title=Unknown Python version::'$version' is not among ${installed[*]}."; exit 2 ;;
            esac
          done
          args=(--pythons "$(IFS=,; echo "${wanted[*]}")" --out out
                --image "GitHub-hosted $RUNNER_LABEL, image ${ImageOS:-unknown} ${ImageVersion:-unknown}")
          if [ -n "$ENGINES" ]; then args+=(--engines "$ENGINES"); fi
          if [ -n "$ROUNDS" ]; then args+=(--rounds "$ROUNDS"); fi
          if [ -n "$REPEATS" ]; then args+=(--repeats "$REPEATS"); fi
          code=0
          examples/benchmark/native.sh "${args[@]}" || code=$?
          case "$code" in
            0) echo 'status=success' >> "$GITHUB_OUTPUT" ;;
            3) echo 'status=warning' >> "$GITHUB_OUTPUT"
               echo '::warning title=Not every Python version measured in full::The tables explain each gap.' ;;
            *) echo 'status=failure' >> "$GITHUB_OUTPUT"
               exit "$code" ;;
          esac

      # The report names the runner, as the tables' Image row does, since a
      # native run has no image digest to name.
      - name: Write the report
        if: ${{ always() && hashFiles('out/table.rst') != '' }}
        env:
          RUNNER_LABEL: ${{ matrix.os }}
        run: |
          set -eu
          {
            echo "## Engine benchmark on $RUNNER_LABEL"
            echo
            echo "Measured natively on $RUNNER_OS $RUNNER_ARCH, image ${ImageOS:-unknown} ${ImageVersion:-unknown},"
            echo "so the absolute times compare only within this run. The docs' tables are updated by hand."
            echo
            echo '```rst'
            cat out/table.rst
            echo '```'
          } > out/report.md
          {
            cat out/report.md
            echo
            echo '```rst'
            cat out/table-versions.rst
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload the results
        if: ${{ always() }}
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: benchmark-out
          path: out/
          if-no-files-found: ignore
          retention-days: 90

  # #2030: posts the ratio table, with its absolute-time footnote, on
  # `always()` like ci-timing.yml's, so a run with gaps is posted too.
  # util/discussion_report.py writes a warning rather than failing, including
  # for a run that wrote no report; `continue-on-error` covers the rest.
  discussion:
    name: Post the engine benchmark report
    needs: [ benchmark ]
    if: ${{ always() && needs.benchmark.result != 'skipped' }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    continue-on-error: true
    permissions:
      contents: read
      discussions: write

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Download the report
        continue-on-error: true
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: benchmark-out
          path: report

      - name: Post the report to Discussions
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
          STATUS: ${{ needs.benchmark.outputs.status || needs.benchmark.result }}
        # `scheduled` on dispatch too: a manual run is a deliberate report.
        run: |
          python3 util/discussion_report.py --category "$REPORT_CATEGORY" \
            --title 'benchmark: weekly engine benchmark report' --report report/report.md \
            --mode scheduled --status "$STATUS"