benchmark.yml

name: benchmark

on:
  pull_request:

permissions:
  contents: read
  pull-requests: write

jobs:
  benchmark:
    name: Benchmark
    runs-on: ubuntu-latest
    steps:
      - name: Checkout head
        uses: actions/checkout@v7
        with:
          path: head
          persist-credentials: false
      - name: Checkout base
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: base
          persist-credentials: false
      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: lts/*
      - name: Run base and head alternately
        env:
          ROUNDS: 5
        run: |
          set -euo pipefail
          mkdir -p results
          run_side () {
            if [ "$1" = base ] && [ ! -f base/benchmark/index.js ]; then return 0; fi
            if (cd "$1" && node benchmark/index.js) && [ -f "$1/benchmark-output.json" ]; then
              mv "$1/benchmark-output.json" "results/$1-$2.json"
            elif [ "$1" = base ]; then
              echo "::warning::base benchmark failed in round $2, comparing without it"
            else
              exit 1
            fi
          }
          for round in $(seq "$ROUNDS"); do
            if [ $((round % 2)) -eq 1 ]; then run_side base "$round"; run_side head "$round"
            else run_side head "$round"; run_side base "$round"; fi
          done
      - name: Compare
        env:
          NAME: https-tls
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
          node - <<'EOF'
          const { readdirSync, readFileSync, writeFileSync, mkdirSync, appendFileSync } = require('fs')
          const median = values => {
            const sorted = [...values].sort((a, b) => a - b)
            const middle = sorted.length >> 1
            return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
          }
          const round = value => Number(value.toFixed(2))
          const readSide = side =>
            readdirSync('results')
              .filter(file => file.startsWith(`${side}-`))
              .map(file => JSON.parse(readFileSync(`results/${file}`, 'utf8')))
          const merge = runs => {
            const names = [...new Set(runs.flatMap(run => run.map(result => result.name)))]
            return names.map(name => {
              const matches = runs.flatMap(run => run.filter(result => result.name === name))
              const values = matches.map(result => result.value)
              return {
                name,
                unit: matches[0].unit,
                value: round(median(values)),
                range: `${round(Math.min(...values))}-${round(Math.max(...values))}`,
                extra: `median of ${values.length} rounds: ${values.join(', ')}`
              }
            })
          }
          const head = merge(readSide('head'))
          const baseRuns = readSide('base')
          const base = baseRuns.length ? merge(baseRuns) : []
          writeFileSync('head-output.json', JSON.stringify(head, null, 2))
          mkdirSync('cache', { recursive: true })
          const seed = base.length
            ? { [process.env.NAME]: [{ commit: { id: process.env.BASE_SHA }, date: Date.now(), tool: 'customSmallerIsBetter', benches: base }] }
            : {}
          writeFileSync('cache/benchmark-data.json', JSON.stringify({ lastUpdate: Date.now(), repoUrl: '', entries: seed }))
          const rows = head.map(current => {
            const previous = base.find(result => result.name === current.name)
            const ratio = previous ? (current.value / previous.value).toFixed(2) : ''
            const format = result => (result ? `${result.value} ${result.unit}` : '')
            return `| \`${current.name}\` | ${format(previous)} | ${format(current)} | ${ratio} |`
          })
          const summary = ['| case | base | head | head/base |', '|-|-|-|-|', ...rows].join('\n')
          console.log(summary)
          appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n')
          EOF
      - name: Comment
        if: github.event.pull_request.head.repo.full_name == github.repository
        uses: benchmark-action/github-action-benchmark@v1.22.2
        with:
          name: https-tls
          tool: customSmallerIsBetter
          output-file-path: head-output.json
          external-data-json-path: cache/benchmark-data.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: false
          comment-always: true
          comment-on-alert: false
          fail-on-alert: false
          summary-always: false