macro_benchmarks.yml

name: macro-benchmarks
permissions:
  contents: read

# The `macro` suite of reflex-bench on the CodSpeed macro runner: timing and
# latency, the metrics that need a quiet machine. Everything else runs on
# standard runners (benchmarks-daily). Results are not uploaded to CodSpeed: its
# walltime instrument times its own loop, and our metrics are not the duration
# of one call. Advisory: this workflow never blocks a merge.
#
# Budget: 500 billed macro minutes a month, and the whole job is billed.
#   30 daily runs * J + 10 label runs * J <= 400 min, so J <= 10 min,
#   which leaves >= 100 min (20 %) for re-runs, release backfills and gate checks.
# If a warm job takes longer than 10 min, shrink the `macro` suite first, then
# move the schedule to every second day ("17 3 */2 * *").
# Measured 2026-09-29 on the macro runner (Cortex-A72, 16 cores, profile
# linux-arm64-cortex-a72-py3.12), billed time after 1:10-1:20 in the queue
# (not billed):
#   warm (Playwright and uv caches hit): 7:24 = checkout 0:11, uv + Python +
#   Node 0:19, uv sync 0:25, cache restore + chromium 1:24 (apt deps run even
#   on a cache hit),
#   the suite 4:48 (browser.dev.ready 71 s, hmr.handler 47 s, hmr.render.leaf
#   54 s, compile.incremental 18 s, compile.warm 15 s, import 11 s,
#   run.prod.ready 69 s), summary, upload and cleanup 0:10.
#   cold (no cache): 5:35 with an older, shorter suite, chromium 1:28.
# So 30 * 7.4 + 10 * 7.4 = 296 min a month, 204 min of headroom.
# events.simple.capacity failed on this runner (its 2 s setup probe got no
# answer), so the event benchmarks run on the standard runner only.
#
# Policy: --min-runs 6 --max-runs 10 --min-time 10: at least 6 timed runs per
# instance (the fewest with a median CI), up to 10 for samples under 1 s.

on:
  schedule:
    - cron: "17 3 * * *"
  workflow_dispatch:
  # Remove and re-add the label to run again; a push does not re-run it.
  pull_request:
    types: [labeled]

env:
  REFLEX_TELEMETRY_ENABLED: false
  REFLEX_CHECK_LATEST_VERSION: false

defaults:
  run:
    shell: bash

jobs:
  macro:
    name: macro
    # Self-hosted runners must never run a fork's code: pull requests run only
    # when labeled run-benchmarks from a branch of this repository.
    if: >-
      github.event_name != 'pull_request'
      || (github.event.label.name == 'run-benchmarks'
      && github.event.pull_request.head.repo.full_name == github.repository)
    # One group for every workflow on the macro runner, held by the job only,
    # so labels other than run-benchmarks never enter it. A cancelled run still
    # bills the minutes it used, so the running job is never cancelled: at most
    # one job runs and GitHub keeps at most one pending (a newer pending job
    # replaces an older one). macro_watchdog.yml cancels jobs no runner picks up.
    concurrency:
      group: codspeed-macro
      cancel-in-progress: false
    timeout-minutes: 12
    runs-on: codspeed-macro-arm64-graviton-ubuntu-22-04
    steps:
      # fetch-tags/fetch-depth: the workspace packages derive their versions from git.
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-tags: true
          fetch-depth: 0
          persist-credentials: false
      # Also installs Node, which reflex 0.9 uses to write the build's .gz files.
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: "3.12"
      # Not the dev group: libsass has no arm64 wheel and builds for minutes.
      - name: Install the harness
        run: uv sync --no-dev --group bench --extra db
      # The harness owns every cache a measured phase reads; none is restored.
      - name: Point the bench home at the job's temporary directory
        run: echo "REFLEX_BENCH_HOME=$RUNNER_TEMP/reflex-bench" >> "$GITHUB_ENV"
      - name: Read the Playwright version
        id: playwright
        run: echo "version=$(uv run --no-sync python -c 'from importlib.metadata import version; print(version("playwright"))')" >> "$GITHUB_OUTPUT"
      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: ~/.cache/ms-playwright
          key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright.outputs.version }}
      - name: Install chromium
        run: uv run --no-sync playwright install --with-deps --only-shell chromium
      - name: Check the machine
        run: uv run --no-sync reflex-bench doctor
      - name: Run the macro suite
        env:
          RUN_KIND: ${{ github.event_name == 'schedule' && 'daily' || github.event_name == 'pull_request' && 'pr' || 'ci' }}
        run: >-
          uv run --no-sync reflex-bench run --suite macro --kind "$RUN_KIND"
          --json macro.json --no-save
          --min-runs 6 --max-runs 10 --min-time 10 --fail-on never
      # Warn-only: a regression against the latest daily result never fails the job.
      - name: Summarize
        if: always() && hashFiles('macro.json') != ''
        env:
          LABELED: ${{ github.event_name == 'pull_request' }}
        run: |
          set -euo pipefail
          if [ "$LABELED" = "true" ]; then
            profile=$(jq -r .machine.profile_id macro.json)
            base=""
            if git fetch --quiet --depth 1 origin benchmark-data; then
              base=$(git ls-tree --name-only FETCH_HEAD "runs/$profile/" | grep '_daily_' | sort | tail -n 1 || true)
            fi
            if [ -n "$base" ]; then
              git show "FETCH_HEAD:$base" > base.json
              echo "Compared with \`$base\` of the benchmark-data branch." >> "$GITHUB_STEP_SUMMARY"
              # compare exits 4 when no metric is comparable (another benchmark
              # version, fixture or profile); the table then lists the reasons.
              # Any other failure (an unreadable result, every benchmark
              # failing) fails the step.
              status=0
              uv run --no-sync reflex-bench compare base.json macro.json --format md --fail-on never >> "$GITHUB_STEP_SUMMARY" || status=$?
              case "$status" in
                0) exit 0 ;;
                4) echo "Nothing in this run is comparable with the daily result; this run's own values:" >> "$GITHUB_STEP_SUMMARY" ;;
                *) exit "$status" ;;
              esac
            else
              echo "No daily result of profile \`$profile\` on the benchmark-data branch yet: nothing to compare with." >> "$GITHUB_STEP_SUMMARY"
            fi
          fi
          uv run --no-sync reflex-bench export macro.json --to md >> "$GITHUB_STEP_SUMMARY"
      - name: Upload the result
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: macro
          path: |
            macro.json
            base.json
          if-no-files-found: ignore

  # Scheduled and manual runs of main append their result to the history. PR
  # results stay in the artifact and the summary.
  history:
    needs: macro
    if: >-
      needs.macro.result == 'success'
      && github.event_name != 'pull_request'
      && github.ref == 'refs/heads/main'
    permissions:
      contents: write
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: macro
          path: results
      - uses: ./.github/actions/bench_history
        with:
          results: results/macro.json