Skip to content

Commit 424eae8

Browse files
wchilders-nvidiawchilders-nvidia
committed
Optimize workflows for merge_queue and introduce caching of build + test CI runs []
1 parent 78685b6 commit 424eae8

11 files changed

Lines changed: 712 additions & 380 deletions

File tree

‎.github/workflows/build-and-test-ci.yml‎

Lines changed: 71 additions & 331 deletions
Large diffs are not rendered by default.

‎.github/workflows/ci-toolchain.yml‎

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
2+
# Exceptions.
3+
# See https://edgcpp.org/LICENSE.txt for license information.
4+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5+
#
6+
# Builds one toolchain's preset and tests it with each of its configurations.
7+
# Called once per toolchain by build-and-test-ci.yml, which describes the
8+
# overall job graph, the test cache, and coverage; each test leg waits only on
9+
# this workflow's build, so the toolchains are independent of one another.
10+
#
11+
# There is deliberately no concurrency group here: in a called workflow,
12+
# github.workflow is the caller's name, so a group would make the toolchains
13+
# cancel each other. The caller's group covers this workflow.
14+
15+
name: CI Toolchain
16+
17+
on:
18+
workflow_call:
19+
inputs:
20+
preset:
21+
description: The CMake preset to build (also names the build artifact)
22+
required: true
23+
type: string
24+
build_dir:
25+
description: The preset's build directory (e.g., build/gcc)
26+
required: true
27+
type: string
28+
base:
29+
description: The directory under bases/docker/dev-env/ for EDG_BASE
30+
required: true
31+
type: string
32+
configs:
33+
description: A JSON list of the edgy configurations to test
34+
required: true
35+
type: string
36+
run_timestamp:
37+
description: The edgy run timestamp shared by all test legs
38+
required: true
39+
type: string
40+
test_inputs:
41+
description: The hash of the test inputs (part of the test cache key)
42+
required: true
43+
type: string
44+
45+
permissions:
46+
contents: read
47+
48+
# The steps refer to the inputs through these variables (in run scripts, as
49+
# shell variables rather than expressions).
50+
env:
51+
GCOVR_VERSION: '8.7'
52+
CI_PRESET: ${{ inputs.preset }}
53+
CI_BUILD_DIR: ${{ inputs.build_dir }}
54+
CI_BASE: ${{ inputs.base }}
55+
56+
jobs:
57+
build:
58+
name: Build ${{ inputs.preset }}
59+
runs-on: ubuntu-latest
60+
timeout-minutes: 120
61+
container:
62+
image: &ci-runner-image ghcr.io/edgcpp/ci-runner@sha256:da140ca90c420977f187279d7939fe0c7cb3c4b656e5ab2dc0f6b9ac2a43a255
63+
defaults: &container-defaults
64+
run:
65+
shell: edg-container-exec "bash -eo pipefail {0}"
66+
67+
steps:
68+
- name: Check out repository
69+
uses: actions/checkout@v7
70+
with:
71+
persist-credentials: false
72+
73+
- name: Configure ${{ env.CI_PRESET }}
74+
env:
75+
EDG_BASE: ${{ github.workspace }}/bases/docker/dev-env/${{ env.CI_BASE }}
76+
run: >-
77+
cmake --preset "${CI_PRESET}" -DEDG_CPFE_USE_GCOV=ON
78+
-DEDG_BUILD_IDENTIFIER=ci
79+
80+
- name: Build ${{ env.CI_PRESET }}
81+
env:
82+
EDG_BASE: ${{ github.workspace }}/bases/docker/dev-env/${{ env.CI_BASE }}
83+
run: cmake --build "${CI_BUILD_DIR}"
84+
85+
# Runtime layout only: environment.sh / eccp need bin/, lib/, and the
86+
# per-target lib_linux_* / lib_win* dirs (eccp rewrites ECCP_LIBDIR to
87+
# lib_<target>). lib_src is build-only and omitted. The .gcno files
88+
# let the test legs produce coverage reports.
89+
- name: Upload build products
90+
uses: actions/upload-artifact@v7
91+
with:
92+
name: build-${{ env.CI_PRESET }}
93+
path: |
94+
${{ env.CI_BUILD_DIR }}/bin
95+
${{ env.CI_BUILD_DIR }}/lib
96+
${{ env.CI_BUILD_DIR }}/lib_linux_*
97+
${{ env.CI_BUILD_DIR }}/lib_win*
98+
${{ env.CI_BUILD_DIR }}/environment.sh
99+
${{ env.CI_BUILD_DIR }}/src/**/*.gcno
100+
if-no-files-found: error
101+
retention-days: 1
102+
compression-level: 6
103+
104+
test:
105+
name: Test ${{ matrix.config }}
106+
needs: build
107+
runs-on: ubuntu-latest
108+
timeout-minutes: 360
109+
strategy:
110+
fail-fast: false
111+
matrix:
112+
config: ${{ fromJSON(inputs.configs) }}
113+
container:
114+
image: *ci-runner-image
115+
# Required for setarch ... -R (disable ASLR) used by the test harness.
116+
options: --privileged
117+
defaults: *container-defaults
118+
119+
steps:
120+
- name: Check out repository
121+
uses: actions/checkout@v7
122+
with:
123+
persist-credentials: false
124+
125+
- name: Download ${{ env.CI_PRESET }} build
126+
uses: actions/download-artifact@v7
127+
with:
128+
name: build-${{ env.CI_PRESET }}
129+
# upload-artifact strips the shared build directory prefix, leaving
130+
# bin/, lib/, and environment.sh at the artifact root.
131+
path: ${{ env.CI_BUILD_DIR }}
132+
133+
- name: Restore execute bits on build binaries
134+
run: |
135+
# Artifact round-trip does not reliably preserve +x.
136+
chmod -R a+x "${CI_BUILD_DIR}/bin"
137+
138+
- name: Compute test cache key
139+
id: key
140+
env:
141+
CONFIG: ${{ matrix.config }}
142+
TEST_INPUTS: ${{ inputs.test_inputs }}
143+
run: |
144+
# Hash the test files this configuration can read, using the
145+
# image's tools (like edgy itself, so the selection rules match).
146+
# The helper reads git trees, and git refuses a repository owned
147+
# by another user (the workspace belongs to the runner's user, not
148+
# the container's) unless it is marked safe.
149+
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
150+
tests="$(EDG_TEST_HOME="${GITHUB_WORKSPACE}/tests" \
151+
edg-test-inputs-hash --config "${CONFIG}")"
152+
echo "Test files: ${tests}"
153+
# Hash the names and contents of the downloaded build products.
154+
build="$(cd "${CI_BUILD_DIR}" &&
155+
find . -type f -print0 | LC_ALL=C sort -z |
156+
xargs -0 sha256sum | sha256sum | cut -c1-64)"
157+
echo "Build products: ${build}"
158+
echo "key=tested-${CONFIG}-${TEST_INPUTS}-${tests}-${build}" \
159+
>> "${GITHUB_OUTPUT}"
160+
161+
- name: Look for an earlier passing run of this build
162+
id: tested
163+
uses: actions/cache/restore@v6
164+
with:
165+
key: ${{ steps.key.outputs.key }}
166+
path: .tmp/tested
167+
168+
- name: Reuse earlier results
169+
if: ${{ steps.tested.outputs.cache-hit == 'true' }}
170+
run: |
171+
echo "::notice::This build already passed with the same test" \
172+
"inputs: $(cat .tmp/tested/ci-meta.json)"
173+
mkdir -p coverage
174+
cp .tmp/tested/coverage/*.json coverage/ || true
175+
176+
- name: Run edgy (${{ matrix.config }})
177+
id: edgy
178+
if: ${{ steps.tested.outputs.cache-hit != 'true' }}
179+
env:
180+
RUN_TIMESTAMP: ${{ inputs.run_timestamp }}
181+
CONFIG: ${{ matrix.config }}
182+
run: |
183+
mkdir -p .tmp
184+
cat > .tmp/extensive-ci-edg-base.sh <<EOF
185+
export EDG_BASE="${GITHUB_WORKSPACE}/bases/docker/dev-env/${CI_BASE}"
186+
EOF
187+
188+
cat > .tmp/extensive-ci.edgivk <<EOF
189+
prefix_source TEST_${CONFIG} ${CI_BUILD_DIR}/environment.sh
190+
prefix_source TEST_${CONFIG} .tmp/extensive-ci-edg-base.sh
191+
exec edgy --config="${CONFIG}" --run-timestamp="${RUN_TIMESTAMP}" --max-failure-count=10 --timeout=600 -A --meticulous all
192+
EOF
193+
194+
edg-prefix-env-loader .tmp/extensive-ci.edgivk
195+
196+
- name: Upload partial elist, elog, and .rt.tar.gz results
197+
if: ${{ failure() }}
198+
uses: actions/upload-artifact@v7
199+
with:
200+
name: test-results-${{ matrix.config }}
201+
path: |
202+
tests/runs/${{ inputs.run_timestamp }}/**/*.elist
203+
tests/runs/${{ inputs.run_timestamp }}/**/*.elog
204+
tests/runs/${{ inputs.run_timestamp }}/**/*.rt.tar.gz
205+
if-no-files-found: warn
206+
retention-days: 1
207+
208+
# Coverage: the instrumented cpfe / cpfe-cp wrote .gcda files next to
209+
# the downloaded .gcno files under the build directory's src/. Each leg
210+
# saves a gcovr JSON report; the coverage job merges them and uploads
211+
# to Codecov. Coverage problems must never fail the test leg, hence
212+
# continue-on-error.
213+
- name: Install gcovr
214+
if: ${{ !cancelled() && steps.tested.outputs.cache-hit != 'true' }}
215+
continue-on-error: true
216+
run: |
217+
python3 -m venv "${RUNNER_TEMP}/gcovr"
218+
"${RUNNER_TEMP}/gcovr/bin/pip" install --quiet \
219+
"gcovr==${GCOVR_VERSION}"
220+
221+
# cpfe and cpfe-cp compile the same sources; separate function merging
222+
# keeps gcovr from rejecting functions seen at different lines. A few
223+
# hot loops exceed gcovr's "suspicious hits" threshold; warn about them
224+
# instead of failing.
225+
- name: Generate coverage report
226+
if: ${{ !cancelled() && steps.tested.outputs.cache-hit != 'true' }}
227+
continue-on-error: true
228+
env:
229+
CONFIG: ${{ matrix.config }}
230+
run: |
231+
mkdir -p coverage
232+
"${RUNNER_TEMP}/gcovr/bin/gcovr" \
233+
--root . \
234+
--filter 'src/' \
235+
--merge-mode-functions=separate \
236+
--gcov-ignore-parse-errors=negative_hits.warn_once_per_file \
237+
--gcov-ignore-parse-errors=suspicious_hits.warn_once_per_file \
238+
-j "$(nproc)" \
239+
--json "coverage/${CONFIG}.json" \
240+
--print-summary \
241+
"${CI_BUILD_DIR}"
242+
243+
- name: Upload coverage report
244+
if: ${{ !cancelled() && hashFiles('coverage/*.json') != '' }}
245+
continue-on-error: true
246+
uses: actions/upload-artifact@v7
247+
with:
248+
name: coverage-${{ matrix.config }}
249+
path: coverage/${{ matrix.config }}.json
250+
if-no-files-found: warn
251+
retention-days: 1
252+
253+
# Record the pass (with its coverage report, so a run that skips edgy
254+
# still contributes coverage). Without a report, don't record it.
255+
- name: Prepare record of this passing run
256+
id: record
257+
if: >-
258+
${{ steps.edgy.outcome == 'success' &&
259+
hashFiles(format('coverage/{0}.json', matrix.config)) != '' }}
260+
env:
261+
CONFIG: ${{ matrix.config }}
262+
RUN_ID: ${{ github.run_id }}
263+
RUN_ATTEMPT: ${{ github.run_attempt }}
264+
TESTED_SHA: ${{ github.sha }}
265+
run: |
266+
mkdir -p .tmp/tested/coverage
267+
cp "coverage/${CONFIG}.json" .tmp/tested/coverage/
268+
python3 - > .tmp/tested/ci-meta.json <<'EOF'
269+
import json, os
270+
print(json.dumps({
271+
'tested_sha': os.environ['TESTED_SHA'],
272+
'run_id': int(os.environ['RUN_ID']),
273+
'run_attempt': int(os.environ['RUN_ATTEMPT']),
274+
}))
275+
EOF
276+
277+
- name: Save record of this passing run
278+
if: ${{ steps.record.outcome == 'success' }}
279+
uses: actions/cache/save@v6
280+
with:
281+
key: ${{ steps.key.outputs.key }}
282+
path: .tmp/tested

‎.github/workflows/commit-messages.yml‎

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,14 @@
33
# See https://edgcpp.org/LICENSE.txt for license information.
44
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
55
#
6-
# Check the messages of the new commits pushed to main and of every pull
7-
# request's commits with edg-check-commit-messages (the same policy the
8-
# pre-push hook applies, from edgchanges). Only commits with a violation are
9-
# reported.
6+
# Check the messages of every pull request's commits and of every merge queue
7+
# entry with edg-check-commit-messages (the same policy the pre-push hook
8+
# applies, from edgchanges). Only commits with a violation are reported.
109
#
1110
# A pull request is checked from its base commit to its head commit, and a
1211
# merge queue entry from the branch it is merged into to its tip (so the
13-
# commits the queue creates are checked too). A push is checked from the
14-
# previous tip of main. If that tip is unavailable (e.g. after a force-push),
15-
# the job fails rather than check nothing.
12+
# commits the queue creates are checked too). Commits land on main only
13+
# through the merge queue, so pushes to main are not checked again.
1614
#
1715
# The checker and the commits come from one checkout of github.sha, the same
1816
# commit GitHub loads this workflow from. For a pull request that is the merge
@@ -26,9 +24,6 @@
2624
name: Commit Messages
2725

2826
on:
29-
push:
30-
branches:
31-
- main
3227
pull_request:
3328
merge_group:
3429

@@ -63,21 +58,12 @@ jobs:
6358
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
6459
QUEUE_BASE_SHA: ${{ github.event.merge_group.base_sha }}
6560
QUEUE_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
66-
PUSH_BEFORE_SHA: ${{ github.event.before }}
6761
ALLOW_MERGE_COMMITS: ${{ vars.ALLOW_MERGE_COMMITS }}
6862
run: |
6963
if [[ "${EVENT_NAME}" == pull_request ]]; then
7064
range=("${PR_BASE_SHA}..${PR_HEAD_SHA}")
71-
elif [[ "${EVENT_NAME}" == merge_group ]]; then
72-
range=("${QUEUE_BASE_SHA}..${QUEUE_HEAD_SHA}")
73-
elif [[ "${PUSH_BEFORE_SHA}" != 0000000000000000000000000000000000000000 ]] &&
74-
git cat-file -e "${PUSH_BEFORE_SHA}^{commit}" 2> /dev/null; then
75-
range=("${PUSH_BEFORE_SHA}..HEAD")
7665
else
77-
echo "::error::No previous tip of the branch to compare against" \
78-
"(push before: ${PUSH_BEFORE_SHA}); the pushed commits" \
79-
"were not checked."
80-
exit 1
66+
range=("${QUEUE_BASE_SHA}..${QUEUE_HEAD_SHA}")
8167
fi
8268
options=()
8369
if [[ "${ALLOW_MERGE_COMMITS}" == true ]]; then

‎.github/workflows/dev-tools-tests.yml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,6 @@
99
name: Dev Tools Tests
1010

1111
on:
12-
push:
13-
branches:
14-
- main
15-
paths:
16-
- .github/workflows/dev-tools-tests.yml
17-
- dev_tools/bin/edg-check-commit-messages
18-
- dev_tools/bin/edg-check-file-changes
19-
- dev_tools/bin/edg-collect-changes
20-
- dev_tools/pylibs/edgchanges/**
21-
- dev_tools/pylibs/edgutil/**
22-
- dev_tools/tests/**
2312
pull_request:
2413
paths:
2514
- .github/workflows/dev-tools-tests.yml

0 commit comments

Comments
 (0)