phpstan-src/.github/workflows/benchmarks.yml at 2.3.x · phpstan/phpstan-src · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
name: "Benchmarks"

on:
  pull_request:
    paths:
      - 'src/**'
      - 'conf/**'
      - 'stubs/**'
      - 'composer.lock'
      - 'tests/bench/**'
      - 'tally.php'
      - '.github/workflows/benchmarks.yml'
  push:
    branches:
      - "2.3.x"
    paths:
      - 'src/**'
      - 'conf/**'
      - 'stubs/**'
      - 'composer.lock'
      - 'tests/bench/**'
      - 'tally.php'
      - '.github/workflows/benchmarks.yml'

concurrency:
  group: benchmarks-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  run:
    name: "Benchmarks shard ${{ matrix.shard }}"
    # Pinned because the instruction counts depend on the PHP build and the system libraries
    runs-on: ubuntu-24.04
    # The slowest shard takes about 20 minutes, and tally.php stops a benchmark after 25
    timeout-minutes: 40

    strategy:
      fail-fast: false
      matrix:
        # Each shard is one runner, tally run --shard balances them by the expected instructions
        shard: [1, 2, 3, 4]

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
        with:
          egress-policy: audit

      - name: "Checkout"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: "Install PHP"
        uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0
        with:
          coverage: "none"
          php-version: "8.5"
          extensions: ds,mbstring,posix
          ini-file: production
          ini-values: memory_limit=-1

      - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0

      # Tally brings CodSpeed's Valgrind and its tally_hooks extension, nothing else to set up
      - name: "Run benchmarks"
        # Valgrind runs each benchmark on one CPU, so the shard runs as many at once as the runner has
        run: "php vendor/bin/tally run --shard=${{ matrix.shard }}/${{ strategy.job-total }} --output=tmp/tally-results"

      - name: "Upload results"
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: tally-results-${{ matrix.shard }}
          path: tmp/tally-results
          if-no-files-found: ignore

  compare:
    name: "Compare all shards"
    needs: run
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    timeout-minutes: 10

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
        with:
          egress-policy: audit

      - name: "Checkout"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: "Install PHP"
        uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0
        with:
          coverage: "none"
          php-version: "8.5"

      - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0

      - name: "Download results"
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          pattern: tally-results-*
          merge-multiple: true
          path: tmp/tally-results

      - name: "Compare"
        run: "php vendor/bin/tally compare --table tmp/tally-results"

      # vendor/bin/tally accept <this artifact> writes the numbers into the files in tests/bench/data
      - name: "Upload results of all shards"
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: tally-results
          path: tmp/tally-results
          if-no-files-found: ignore