Skip to content

Commit a82af18

Browse files
committed
Build the pie extensions from the core's profile instead of training again
The self-contained extensions of the pie legs ran a full `make pgo` each: an instrumented build, a training run and the optimised build, 6-18 minutes per leg. The core jobs now pack the core's share of their profile (`make SPLIT=1 pgo-pack`, uploaded as turbo-pgo-profile-<platform> and carried over with a reused core), and the pie legs build from it with `make pgo-from-profile`, which falls back to `make pgo` when another compiler recorded the profile. The core compiles to the same code against every version's headers, but GCC instruments before optimising: with the 8.5 core's profile, 85 of 33136 functions mismatch on 8.2-zts (none on 8.4, 8.4-zts, 8.5), and those build without a profile.
1 parent de22776 commit a82af18

4 files changed

Lines changed: 127 additions & 11 deletions

File tree

‎.github/workflows/phar.yml‎

Lines changed: 73 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,7 @@ jobs:
497497
PHP_MINOR: ${{ matrix.target.php-version }}
498498
PHP_ZTS: "0"
499499
TURBO_ARTIFACT: "phpstan_turbo_core-${{ matrix.target.name }}"
500+
PGO_PROFILE_ARTIFACT: "turbo-pgo-profile-${{ matrix.target.name }}"
500501
ORIGIN_RUN_ID: ${{ fromJSON(needs.turbo-origins.outputs.origins || '{}')[format('phpstan_turbo_core-{0}', matrix.target.name)] }}
501502

502503
strategy:
@@ -544,6 +545,17 @@ jobs:
544545
run-id: ${{ env.ORIGIN_RUN_ID }}
545546
github-token: ${{ github.token }}
546547

548+
# the profile that core was compiled with, uploaded before it (see
549+
# "Upload the core's profile")
550+
- name: "Download the profile of the core compiled from identical build inputs"
551+
if: env.ORIGIN_RUN_ID != ''
552+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
553+
with:
554+
name: ${{ env.PGO_PROFILE_ARTIFACT }}
555+
path: "turbo-ext"
556+
run-id: ${{ env.ORIGIN_RUN_ID }}
557+
github-token: ${{ github.token }}
558+
547559
- name: "Record the run that compiled the core"
548560
if: env.ORIGIN_RUN_ID != ''
549561
run: echo "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$ORIGIN_RUN_ID" | tee turbo-reused.txt
@@ -614,8 +626,11 @@ jobs:
614626
# locally, so a profile-guided core can be reproduced outside CI.
615627
# The strict set itself lives in turbo-ext/Makefile
616628
# (STRICT_WARN_FLAGS), which documents why each exemption is there.
629+
# `make pgo-pack` keeps the core's share of the profile for the pie
630+
# legs of turbo-compile, which build with it instead of training again.
617631
run: |
618632
make SPLIT=1 pgo WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
633+
make SPLIT=1 pgo-pack
619634
620635
# Before signing (stripping invalidates a signature) and before the
621636
# upload. The symbol tables are a quarter of the binary; see `strip`
@@ -651,6 +666,16 @@ jobs:
651666
echo "core + extension report: $REPORTED, enabler expects: $EXPECTED"
652667
[ "$REPORTED" = "$EXPECTED" ]
653668
669+
# Before the core, so that a run carrying the core (all that
670+
# find-turbo-origins.sh checks) carries its profile too. A reused one
671+
# is uploaded again, like the core.
672+
- name: "Upload the core's profile"
673+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
674+
with:
675+
name: ${{ env.PGO_PROFILE_ARTIFACT }}
676+
path: "turbo-ext/pgo-profile.tar"
677+
if-no-files-found: "error"
678+
654679
# A reused core is uploaded again under the same name, so the
655680
# turbo-compile legs find it in this run.
656681
- name: "Upload core artifact"
@@ -688,6 +713,7 @@ jobs:
688713
PHP_MINOR: "8.5"
689714
PHP_ZTS: "0"
690715
TURBO_ARTIFACT: "phpstan_turbo_core-linux-musl-arm64"
716+
PGO_PROFILE_ARTIFACT: "turbo-pgo-profile-linux-musl-arm64"
691717
ORIGIN_RUN_ID: ${{ fromJSON(needs.turbo-origins.outputs.origins || '{}')['phpstan_turbo_core-linux-musl-arm64'] }}
692718

693719
steps:
@@ -696,7 +722,7 @@ jobs:
696722
with:
697723
egress-policy: audit
698724

699-
# See the same two steps in turbo-compile-core.
725+
# See the same three steps in turbo-compile-core.
700726
- name: "Download the core compiled from identical build inputs"
701727
if: env.ORIGIN_RUN_ID != ''
702728
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
@@ -706,6 +732,15 @@ jobs:
706732
run-id: ${{ env.ORIGIN_RUN_ID }}
707733
github-token: ${{ github.token }}
708734

735+
- name: "Download the profile of the core compiled from identical build inputs"
736+
if: env.ORIGIN_RUN_ID != ''
737+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
738+
with:
739+
name: ${{ env.PGO_PROFILE_ARTIFACT }}
740+
path: "turbo-ext"
741+
run-id: ${{ env.ORIGIN_RUN_ID }}
742+
github-token: ${{ github.token }}
743+
709744
- name: "Record the run that compiled the core"
710745
if: env.ORIGIN_RUN_ID != ''
711746
run: echo "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$ORIGIN_RUN_ID" | tee turbo-reused.txt
@@ -751,6 +786,7 @@ jobs:
751786
run: |
752787
docker exec -i -w /work/turbo-ext alpine-build sh -e <<'EOF'
753788
make SPLIT=1 pgo WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
789+
make SPLIT=1 pgo-pack
754790
make SPLIT=1 strip
755791
ls -l phpstan_turbo_core.so phpstan_turbo.so
756792
EOF
@@ -765,6 +801,13 @@ jobs:
765801
[ "$REPORTED" = "$EXPECTED" ]
766802
EOF
767803
804+
- name: "Upload the core's profile"
805+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
806+
with:
807+
name: ${{ env.PGO_PROFILE_ARTIFACT }}
808+
path: "turbo-ext/pgo-profile.tar"
809+
if-no-files-found: "error"
810+
768811
- name: "Upload core artifact"
769812
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
770813
with:
@@ -791,7 +834,8 @@ jobs:
791834
# there, and the commit job checks that every leg of a platform carries
792835
# the same core.
793836
# The `pie` variant legs build the same version self-contained instead,
794-
# profile-guided like the core — PIE installs a pre-packaged binary by
837+
# profile-guided with the profile the platform's core was built with
838+
# (`make pgo-from-profile`) — PIE installs a pre-packaged binary by
795839
# copying the one extension file, so the PIE release assets cannot be
796840
# the thin extension (see .github/scripts/turbo-pie-assets.sh). Nothing
797841
# else ships them, so they only run for release tags (turbo-pie-release
@@ -910,8 +954,21 @@ jobs:
910954
run: |
911955
make SPLIT=1 extension WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
912956
957+
# The profile the platform's core was compiled with (turbo-compile-core,
958+
# `make pgo-pack`): the core is all but the four version-specific
959+
# sources and compiles to the same code against every version's
960+
# headers, so the self-contained build of this version uses its profile
961+
# instead of training one of its own — which took most of the leg.
962+
- name: "Download the profile of the platform's core"
963+
if: matrix.variant == 'pie'
964+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
965+
with:
966+
name: "turbo-pgo-profile-${{ matrix.target.name }}"
967+
path: "turbo-ext"
968+
913969
# see the same steps in turbo-compile-core: the training run of
914-
# `make pgo` runs bin/phpstan on the checkout
970+
# `make pgo` runs bin/phpstan on the checkout (pgo-from-profile falls
971+
# back to it when another compiler recorded the profile)
915972
- uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
916973
if: matrix.variant == 'pie' && matrix.target.family == 'macos'
917974
with:
@@ -924,12 +981,12 @@ jobs:
924981
php composer.phar install --no-interaction --no-progress
925982
rm composer.phar
926983
927-
# A plain `make pgo` builds one self-contained phpstan_turbo.so.
984+
# A plain `make pgo-use` builds one self-contained phpstan_turbo.so.
928985
- name: "Compile the self-contained extension with strict warnings (profile-guided)"
929986
if: matrix.variant == 'pie'
930987
working-directory: "turbo-ext"
931988
run: |
932-
make pgo WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
989+
make pgo-from-profile WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
933990
934991
# Before signing and the upload; a plain `make strip` leaves the core
935992
# (already stripped and signed by its own job) untouched.
@@ -1027,6 +1084,14 @@ jobs:
10271084
name: "phpstan_turbo_core-linux-musl-arm64"
10281085
path: "turbo-ext"
10291086

1087+
# see turbo-compile
1088+
- name: "Download the profile of the platform's core"
1089+
if: matrix.variant == 'pie'
1090+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
1091+
with:
1092+
name: "turbo-pgo-profile-linux-musl-arm64"
1093+
path: "turbo-ext"
1094+
10301095
- name: "Start the Alpine build container"
10311096
env:
10321097
ALPINE_IMAGE: ${{ matrix.php-version == '8.6' && matrix.ts == 'nts' && 'alpine:edge' || 'alpine:3.24' }}
@@ -1052,8 +1117,8 @@ jobs:
10521117
ls -l phpstan_turbo.so phpstan_turbo_core.so
10531118
EOF
10541119
1055-
# see turbo-compile: the training run of `make pgo` runs bin/phpstan
1056-
# on the checkout
1120+
# see turbo-compile: the training run of `make pgo`, which
1121+
# pgo-from-profile falls back to, runs bin/phpstan on the checkout
10571122
- name: "Compile the self-contained extension with strict warnings (profile-guided)"
10581123
if: matrix.variant == 'pie'
10591124
run: |
@@ -1062,7 +1127,7 @@ jobs:
10621127
php composer.phar install --no-interaction --no-progress
10631128
rm composer.phar
10641129
cd turbo-ext
1065-
make pgo WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
1130+
make pgo-from-profile WARN_FLAGS="$(make -s print-warn-flags)" -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc)"
10661131
make strip
10671132
ls -l phpstan_turbo.so
10681133
EOF

‎turbo-ext/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
/pgo/
4646
/src/*.gcda
4747
/src/parser/*.gcda
48+
/pgo-profile.tar
4849

4950
# the compilation database clang-tidy reads (make lint-turbo), regenerated
5051
# from the Makefile's own flags by bin/generate-compile-commands.php

‎turbo-ext/CLAUDE.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,8 @@ make SPLIT=1 # phpstan_turbo_core.so + phpstan_turbo.so (loads t
834834
make SPLIT=1 pgo # the same, profile-guided (what turbo-compile-core runs)
835835
make SPLIT=1 extension # only the extension, against a phpstan_turbo_core.so already here
836836
make SPLIT=1 strip # both; a plain `make strip` leaves the core untouched
837+
make SPLIT=1 pgo-pack # after `pgo`: the core's profile, into pgo-profile.tar
838+
make pgo-from-profile # self-contained, from that profile (CI's pie legs); trains if it does not fit
837839
```
838840

839841
A plain `make`, the phpize build (`config.m4`, what PIE drives) and

‎turbo-ext/Makefile‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -332,18 +332,66 @@ pgo:
332332
$(MAKE) pgo-clean
333333
$(MAKE) PGO_FLAGS="$(PGO_GEN_FLAGS)" phpstan_turbo.so
334334
PGO_DIR="$(PGO_DIR)" bin/pgo-train.sh
335-
rm -f $(OBJECTS) phpstan_turbo.so $(CORE_LIB)
336335
ifeq ($(CXX_IS_CLANG),1)
337336
$(LLVM_PROFDATA) merge -output=$(PGO_DIR)/turbo.profdata $(PGO_DIR)/*.profraw
338337
endif
338+
$(MAKE) pgo-use
339+
340+
# the optimised stage of `make pgo` alone, from a profile already in place:
341+
# the one `make pgo` just recorded, or one unpacked from $(PGO_PROFILE)
342+
pgo-use:
343+
rm -f $(OBJECTS) phpstan_turbo.so $(CORE_LIB)
339344
$(MAKE) PGO_FLAGS="$(PGO_USE_FLAGS)" phpstan_turbo.so
340345

346+
# The core's share of the profile `make SPLIT=1 pgo` recorded, with the
347+
# compiler that recorded it (a profile only fits that one: GCC rejects
348+
# another version's .gcda files), so that `make pgo-use` builds the
349+
# self-contained extension of every other PHP version from it without a
350+
# training run of its own — the CI pie legs (phar.yml). The core compiles to
351+
# the same code against every supported version's headers
352+
# (bin/shared-core/gate-*.sh), so its profile fits nearly all of them (see
353+
# pgo-from-profile); the shim's
354+
# records stay out, since those objects do not (GCC rejects them as a
355+
# coverage mismatch), and the shim builds without a profile, like the shim
356+
# of the split extension. Clang's one merged file is taken whole: it drops
357+
# the records of functions that changed on its own.
358+
PGO_PROFILE := pgo-profile.tar
359+
CXX_VERSION = $(shell $(CXX) --version | head -n 1)
360+
pgo-pack:
361+
mkdir -p $(PGO_DIR)
362+
echo '$(CXX_VERSION)' > $(PGO_DIR)/compiler.txt
363+
ifeq ($(CXX_IS_CLANG),1)
364+
tar -cf $(PGO_PROFILE) $(PGO_DIR)/compiler.txt $(PGO_DIR)/turbo.profdata
365+
else
366+
tar -cf $(PGO_PROFILE) $(PGO_DIR)/compiler.txt $(wildcard $(CORE_OBJECTS:.o=.gcda))
367+
endif
368+
369+
# `make pgo-use` from the $(PGO_PROFILE) pgo-pack wrote, when the compiler
370+
# that recorded it is the one in use; `make pgo` otherwise (or without one).
371+
# The same code is not the same control flow when GCC instruments it, before
372+
# its optimisations: against other headers a few functions count other arcs
373+
# (with the GCC 11.4 profile of an 8.5 core, 85 of 33136 functions for
374+
# 8.2-zts, none for 8.4, 8.4-zts or 8.5). GCC rejects the profile of such a
375+
# function, which then builds without one, like the shim; the warning would
376+
# fail the strict build, so it is off.
377+
ifneq ($(CXX_IS_CLANG),1)
378+
PGO_FOREIGN_FLAGS := -Wno-coverage-mismatch
379+
endif
380+
pgo-from-profile:
381+
@if [ -f $(PGO_PROFILE) ] && [ "$$(tar -xOf $(PGO_PROFILE) $(PGO_DIR)/compiler.txt)" = '$(CXX_VERSION)' ]; then \
382+
echo "building with the profile in $(PGO_PROFILE)"; \
383+
tar -xf $(PGO_PROFILE) && $(MAKE) pgo-use PGO_USE_FLAGS="$(PGO_USE_FLAGS) $(PGO_FOREIGN_FLAGS)"; \
384+
else \
385+
echo "no profile recorded by $(CXX_VERSION) in $(PGO_PROFILE); training one"; \
386+
$(MAKE) pgo; \
387+
fi
388+
341389
# llvm-profdata must match the clang in use: Xcode ships it behind xcrun,
342390
# Linux distributions suffix it with the LLVM major version
343391
LLVM_PROFDATA ?= $(shell command -v llvm-profdata 2>/dev/null || (command -v xcrun > /dev/null 2>&1 && echo "xcrun llvm-profdata") || ls /usr/bin/llvm-profdata-* 2>/dev/null | sort -V | tail -1)
344392

345393
pgo-clean:
346-
rm -f $(OBJECTS) phpstan_turbo.so $(CORE_LIB) version.stamp
394+
rm -f $(OBJECTS) phpstan_turbo.so $(CORE_LIB) version.stamp $(PGO_PROFILE)
347395
rm -rf $(PGO_DIR)
348396
find src -name '*.gcda' -delete
349397

@@ -432,4 +480,4 @@ sanitize:
432480
done
433481
$(MAKE) clean
434482

435-
.PHONY: clean lint pgo pgo-clean print-clang-tidy-version print-warn-flags sanitize strip FORCE
483+
.PHONY: clean lint pgo pgo-clean pgo-from-profile pgo-pack pgo-use print-clang-tidy-version print-warn-flags sanitize strip FORCE

0 commit comments

Comments
 (0)