Benchmarks #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| # Manual + weekly only — NOT per-PR. Benchmarks are noisy and slow; we run them | |
| # on demand and on a schedule, compare against the like-for-like ubuntu baseline | |
| # (informational-first), and upload the results + generated docs as artifacts. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| full_size: | |
| description: 'Run full-size sweeps (slower; the representative capture, NOT the like-for-like gate baseline)' | |
| type: boolean | |
| default: false | |
| schedule: | |
| - cron: '0 6 * * 1' # Mondays 06:00 UTC | |
| jobs: | |
| benchmark: | |
| name: Run benchmark suite | |
| runs-on: ubuntu-latest | |
| # Postgres is started as a step (below), not a service: the connection-budget | |
| # and churn tiers deliberately push past 50 tenant connections, so the stock | |
| # service cap of 100 throws "too many clients". Service containers can't set | |
| # the postgres command or mount a post-checkout config, so we run it manually | |
| # with benchmarks/postgres.bench.conf (max_connections=300), mirroring local. | |
| services: | |
| redis: | |
| image: redis:7-alpine | |
| ports: | |
| - 6379:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| TZ: UTC | |
| # development (not test) so the rate-limit middleware runs its real Redis | |
| # pipeline instead of the test-env bypass. | |
| NODE_ENV: development | |
| BENCH_CI: '1' | |
| HOST: 127.0.0.1 | |
| APP_KEY: a-32-character-long-secret-key!! | |
| LOG_LEVEL: error | |
| TENANT_HEADER_KEY: x-tenant-id | |
| DB_HOST: 127.0.0.1 | |
| DB_PORT: '5432' | |
| DB_USER: postgres | |
| DB_PASSWORD: postgres | |
| DB_DATABASE: lasagna_bench | |
| REDIS_HOST: 127.0.0.1 | |
| REDIS_PORT: '6379' | |
| QUEUE_REDIS_HOST: 127.0.0.1 | |
| QUEUE_REDIS_PORT: '6379' | |
| QUEUE_REDIS_DB: '1' | |
| CACHE_REDIS_HOST: 127.0.0.1 | |
| CACHE_REDIS_PORT: '6379' | |
| CACHE_REDIS_DB: '2' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Raised max_connections (300) so the cap=100 churn sweep × tenant poolMax | |
| # plus central/backoffice connections stays under the limit. Mounts the same | |
| # benchmarks/postgres.bench.conf the local docker-compose uses. | |
| - name: Start Postgres (bench tuning, max_connections=300) | |
| run: | | |
| docker run -d --name bench-postgres \ | |
| -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=lasagna_bench \ | |
| -p 5432:5432 \ | |
| -v "$GITHUB_WORKSPACE/benchmarks/postgres.bench.conf:/etc/postgresql/postgresql.conf:ro" \ | |
| postgres:16-alpine -c config_file=/etc/postgresql/postgresql.conf | |
| echo "Waiting for Postgres to accept connections…" | |
| for i in $(seq 1 30); do | |
| if docker exec bench-postgres pg_isready -U postgres -d lasagna_bench >/dev/null 2>&1; then | |
| echo "Postgres ready (max_connections=$(docker exec bench-postgres psql -U postgres -tAc 'show max_connections'))" | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Postgres did not become ready in time"; docker logs bench-postgres; exit 1 | |
| # Default run is CI-sized (fast, stable, like-for-like with the committed | |
| # ci-ubuntu baseline). A `full_size=true` dispatch overrides each BENCH_* | |
| # knob with the production-ish sweep (budget to 2000 tenants, catalog to | |
| # 5000 schemas, full churn). Explicit BENCH_* vars win over the CI defaults | |
| # in src/harness/config.ts, so BENCH_CI stays 1 either way. | |
| - name: Configure bench sizes | |
| run: | | |
| if [ "${{ github.event.inputs.full_size }}" = "true" ]; then | |
| echo "Running FULL-SIZE sweeps (representative capture)" | |
| { | |
| echo "BENCH_DB_TENANTS=50" | |
| echo "BENCH_DB_ROWS=1000" | |
| echo "BENCH_CHURN_CAPS=25,50,100" | |
| echo "BENCH_CHURN_CONCURRENCY=16" | |
| echo "BENCH_CHURN_OPS=2000" | |
| echo "BENCH_BUDGET_COUNTS=100,500,2000" | |
| echo "BENCH_CATALOG_COUNTS=100,1000,5000" | |
| echo "BENCH_HTTP_TENANTS=50" | |
| echo "BENCH_HTTP_ROWS=200" | |
| echo "BENCH_HTTP_CONNECTIONS=25" | |
| echo "BENCH_HTTP_DURATION=10" | |
| } >> "$GITHUB_ENV" | |
| else | |
| echo "Running CI-size sweeps (gate default)" | |
| fi | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '24' | |
| - name: Install dependencies | |
| env: | |
| npm_config_ignore_scripts: 'false' | |
| run: npm install --legacy-peer-deps --include=optional | |
| # The bench fixture imports the built core via the workspace symlink, so a | |
| # fresh build of the core is mandatory. The satellites are not used. | |
| - name: Build core | |
| run: npm run build | |
| - name: Tier 1 — micro | |
| run: npm run bench:micro | |
| - name: Tiers 2-3 — DB + HTTP, per driver | |
| run: | | |
| for driver in schema-pg database-pg rowscope-pg; do | |
| echo "::group::$driver" | |
| BENCH_DRIVER=$driver npm run bench:db | |
| BENCH_DRIVER=$driver npm run bench:http | |
| echo "::endgroup::" | |
| done | |
| - name: Tier 4 — memory + budget (schema-pg) | |
| run: BENCH_DRIVER=schema-pg npm run bench:mem | |
| - name: Regenerate report | |
| run: npm run bench:report | |
| # Informational by default. Once the ci-ubuntu baseline has stabilized, | |
| # set BENCH_GATE_ENFORCE=1 here to make a regression fail the job. | |
| - name: Regression check vs ci-ubuntu baseline | |
| run: npm run bench:check -- --baseline=ci-ubuntu | |
| - name: Upload results + generated report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| benchmarks/results/ | |
| docs/docs/performance.md | |
| if-no-files-found: warn |