benchmark.yml

name: Benchmarks

# Manual + weekly only — NOT per-PR. Benchmarks are noisy and slow; we run them
# on demand and on a schedule, compare against the like-for-like ubuntu baseline
# (informational-first), and upload the results + generated docs as artifacts.
on:
  workflow_dispatch:
    inputs:
      full_size:
        description: 'Run full-size sweeps (slower; the representative capture, NOT the like-for-like gate baseline)'
        type: boolean
        default: false
  schedule:
    - cron: '0 6 * * 1' # Mondays 06:00 UTC

jobs:
  benchmark:
    name: Run benchmark suite
    runs-on: ubuntu-latest

    # Postgres is started as a step (below), not a service: the connection-budget
    # and churn tiers deliberately push past 50 tenant connections, so the stock
    # service cap of 100 throws "too many clients". Service containers can't set
    # the postgres command or mount a post-checkout config, so we run it manually
    # with benchmarks/postgres.bench.conf (max_connections=300), mirroring local.
    services:
      redis:
        image: redis:7-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10

    env:
      TZ: UTC
      # development (not test) so the rate-limit middleware runs its real Redis
      # pipeline instead of the test-env bypass.
      NODE_ENV: development
      BENCH_CI: '1'
      HOST: 127.0.0.1
      APP_KEY: a-32-character-long-secret-key!!
      LOG_LEVEL: error
      TENANT_HEADER_KEY: x-tenant-id
      DB_HOST: 127.0.0.1
      DB_PORT: '5432'
      DB_USER: postgres
      DB_PASSWORD: postgres
      DB_DATABASE: lasagna_bench
      REDIS_HOST: 127.0.0.1
      REDIS_PORT: '6379'
      QUEUE_REDIS_HOST: 127.0.0.1
      QUEUE_REDIS_PORT: '6379'
      QUEUE_REDIS_DB: '1'
      CACHE_REDIS_HOST: 127.0.0.1
      CACHE_REDIS_PORT: '6379'
      CACHE_REDIS_DB: '2'

    steps:
      - uses: actions/checkout@v4

      # Raised max_connections (300) so the cap=100 churn sweep × tenant poolMax
      # plus central/backoffice connections stays under the limit. Mounts the same
      # benchmarks/postgres.bench.conf the local docker-compose uses.
      - name: Start Postgres (bench tuning, max_connections=300)
        run: |
          docker run -d --name bench-postgres \
            -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=lasagna_bench \
            -p 5432:5432 \
            -v "$GITHUB_WORKSPACE/benchmarks/postgres.bench.conf:/etc/postgresql/postgresql.conf:ro" \
            postgres:16-alpine -c config_file=/etc/postgresql/postgresql.conf
          echo "Waiting for Postgres to accept connections…"
          for i in $(seq 1 30); do
            if docker exec bench-postgres pg_isready -U postgres -d lasagna_bench >/dev/null 2>&1; then
              echo "Postgres ready (max_connections=$(docker exec bench-postgres psql -U postgres -tAc 'show max_connections'))"
              exit 0
            fi
            sleep 2
          done
          echo "Postgres did not become ready in time"; docker logs bench-postgres; exit 1

      # Default run is CI-sized (fast, stable, like-for-like with the committed
      # ci-ubuntu baseline). A `full_size=true` dispatch overrides each BENCH_*
      # knob with the production-ish sweep (budget to 2000 tenants, catalog to
      # 5000 schemas, full churn). Explicit BENCH_* vars win over the CI defaults
      # in src/harness/config.ts, so BENCH_CI stays 1 either way.
      - name: Configure bench sizes
        run: |
          if [ "${{ github.event.inputs.full_size }}" = "true" ]; then
            echo "Running FULL-SIZE sweeps (representative capture)"
            {
              echo "BENCH_DB_TENANTS=50"
              echo "BENCH_DB_ROWS=1000"
              echo "BENCH_CHURN_CAPS=25,50,100"
              echo "BENCH_CHURN_CONCURRENCY=16"
              echo "BENCH_CHURN_OPS=2000"
              echo "BENCH_BUDGET_COUNTS=100,500,2000"
              echo "BENCH_CATALOG_COUNTS=100,1000,5000"
              echo "BENCH_HTTP_TENANTS=50"
              echo "BENCH_HTTP_ROWS=200"
              echo "BENCH_HTTP_CONNECTIONS=25"
              echo "BENCH_HTTP_DURATION=10"
            } >> "$GITHUB_ENV"
          else
            echo "Running CI-size sweeps (gate default)"
          fi

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '24'

      - name: Install dependencies
        env:
          npm_config_ignore_scripts: 'false'
        run: npm install --legacy-peer-deps --include=optional

      # The bench fixture imports the built core via the workspace symlink, so a
      # fresh build of the core is mandatory. The satellites are not used.
      - name: Build core
        run: npm run build

      - name: Tier 1 — micro
        run: npm run bench:micro

      - name: Tiers 2-3 — DB + HTTP, per driver
        run: |
          for driver in schema-pg database-pg rowscope-pg; do
            echo "::group::$driver"
            BENCH_DRIVER=$driver npm run bench:db
            BENCH_DRIVER=$driver npm run bench:http
            echo "::endgroup::"
          done

      - name: Tier 4 — memory + budget (schema-pg)
        run: BENCH_DRIVER=schema-pg npm run bench:mem

      - name: Regenerate report
        run: npm run bench:report

      # Informational by default. Once the ci-ubuntu baseline has stabilized,
      # set BENCH_GATE_ENFORCE=1 here to make a regression fail the job.
      - name: Regression check vs ci-ubuntu baseline
        run: npm run bench:check -- --baseline=ci-ubuntu

      - name: Upload results + generated report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: |
            benchmarks/results/
            docs/docs/performance.md
          if-no-files-found: warn