Skip to content

Commit 11b0a2f

Browse files
Marve10sMarve10s
committed
ci: shard smoke jobs so no CI job runs longer than ~10 minutes
Strict Smoke (Broad) ran 31 presets sequentially in one job (38.9m of a 43.8m job) and Core ran 12 (15.5m); the Smoke Test workflow ran its whole combo batch in one job (12-23m). Agents waiting on CI were blocked on the single longest sequential job. - Add --partition support to the preset path: getPresetCombos now takes an optional {index, total} stride partition and smoke-test.ts forwards --partition to it, with a test proving shards are disjoint and cover the full group. - Shard Broad across 6 matrix jobs and Core across 3 in e2e-test.yaml (fail-fast off, per-shard artifacts, 20m timeouts as guardrails). - Shard the Smoke Test workflow across 3 matrix jobs using the harness's existing generated-combo partitioning (per-shard count = ceil(count/3) so total coverage is unchanged) and add a summary aggregator job so callers keep receiving one combined workflow summary output. - Unify every Bun cache key to the shared deps- key (same as test.yaml) so jobs restore each other's cache, and add the missing dependency cache to the generated-project-proof workflow. No Rust/Vite+ rewrite is needed: setup and installs are under 3 minutes per job; the wall time was sequential preset execution, which the matrix removes.
1 parent 57a534d commit 11b0a2f

6 files changed

Lines changed: 141 additions & 62 deletions

File tree

‎.github/workflows/e2e-test.yaml‎

Lines changed: 45 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,8 @@ jobs:
9191
path: |
9292
node_modules
9393
~/.bun/install/cache
94-
key: e2e-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
95-
restore-keys: e2e-${{ runner.os }}-bun-
94+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
95+
restore-keys: deps-${{ runner.os }}-bun-
9696

9797
- name: Install Dependencies
9898
run: bun install --frozen-lockfile
@@ -119,10 +119,14 @@ jobs:
119119
retention-days: 7
120120

121121
smoke-strict-core:
122-
name: Strict Smoke Tests (Core)
122+
name: Strict Smoke Tests (Core ${{ matrix.shard }}/3)
123123
if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_smoke_strict }}
124124
runs-on: ubuntu-latest
125-
timeout-minutes: 60
125+
timeout-minutes: 20
126+
strategy:
127+
fail-fast: false
128+
matrix:
129+
shard: [0, 1, 2]
126130
services:
127131
postgres:
128132
image: postgres:16
@@ -183,8 +187,8 @@ jobs:
183187
path: |
184188
node_modules
185189
~/.bun/install/cache
186-
key: smoke-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
187-
restore-keys: smoke-${{ runner.os }}-bun-
190+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
191+
restore-keys: deps-${{ runner.os }}-bun-
188192

189193
- name: Install Dependencies
190194
run: bun install --frozen-lockfile
@@ -200,32 +204,37 @@ jobs:
200204
run: |
201205
bun run testing/smoke-test.ts \
202206
--preset pr-core \
207+
--partition ${{ matrix.shard }}/3 \
203208
--dev-check \
204209
--strict \
205-
--output testing/.smoke-output/core
210+
--output testing/.smoke-output/core-${{ matrix.shard }}
206211
207212
- name: Post Summary
208213
if: always()
209214
run: |
210-
if [ -f testing/.smoke-output/core/summary.md ]; then
211-
cat testing/.smoke-output/core/summary.md >> "$GITHUB_STEP_SUMMARY"
215+
if [ -f testing/.smoke-output/core-${{ matrix.shard }}/summary.md ]; then
216+
cat testing/.smoke-output/core-${{ matrix.shard }}/summary.md >> "$GITHUB_STEP_SUMMARY"
212217
fi
213218
214219
- name: Upload Results
215220
if: always()
216221
uses: actions/upload-artifact@v4
217222
with:
218-
name: smoke-strict-core-results
219-
path: testing/.smoke-output/core/
223+
name: smoke-strict-core-results-${{ matrix.shard }}
224+
path: testing/.smoke-output/core-${{ matrix.shard }}/
220225
include-hidden-files: true
221226
if-no-files-found: warn
222227
retention-days: 7
223228

224229
smoke-strict-broad:
225-
name: Strict Smoke Tests (Broad)
230+
name: Strict Smoke Tests (Broad ${{ matrix.shard }}/6)
226231
if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_smoke_strict }}
227232
runs-on: ubuntu-latest
228-
timeout-minutes: 60
233+
timeout-minutes: 20
234+
strategy:
235+
fail-fast: false
236+
matrix:
237+
shard: [0, 1, 2, 3, 4, 5]
229238
services:
230239
postgres:
231240
image: postgres:16
@@ -253,10 +262,11 @@ jobs:
253262
--health-timeout 5s
254263
--health-retries 10
255264
steps:
256-
# The broad matrix scaffolds and installs many projects across every
257-
# ecosystem and has exhausted the runner disk (run 29582988254 died with
258-
# "No space left on device"). Drop the big preinstalled toolchains this
259-
# job never uses (~25GB: Android SDK, GHC, CodeQL, Swift, boost).
265+
# The broad matrix once exhausted the runner disk when all presets ran
266+
# in one job (run 29582988254 died with "No space left on device"). Each
267+
# shard now runs ~1/6 of the presets and cleans up per project, but the
268+
# big preinstalled toolchains this job never uses (~25GB: Android SDK,
269+
# GHC, CodeQL, Swift, boost) are still dropped as a safety margin.
260270
- name: Free runner disk space
261271
run: |
262272
sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup \
@@ -293,8 +303,8 @@ jobs:
293303
path: |
294304
node_modules
295305
~/.bun/install/cache
296-
key: smoke-broad-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
297-
restore-keys: smoke-broad-${{ runner.os }}-bun-
306+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
307+
restore-keys: deps-${{ runner.os }}-bun-
298308

299309
- name: Install Dependencies
300310
run: bun install --frozen-lockfile
@@ -310,23 +320,24 @@ jobs:
310320
run: |
311321
bun run testing/smoke-test.ts \
312322
--preset pr-broad \
323+
--partition ${{ matrix.shard }}/6 \
313324
--dev-check \
314325
--strict \
315-
--output testing/.smoke-output/broad
326+
--output testing/.smoke-output/broad-${{ matrix.shard }}
316327
317328
- name: Post Summary
318329
if: always()
319330
run: |
320-
if [ -f testing/.smoke-output/broad/summary.md ]; then
321-
cat testing/.smoke-output/broad/summary.md >> "$GITHUB_STEP_SUMMARY"
331+
if [ -f testing/.smoke-output/broad-${{ matrix.shard }}/summary.md ]; then
332+
cat testing/.smoke-output/broad-${{ matrix.shard }}/summary.md >> "$GITHUB_STEP_SUMMARY"
322333
fi
323334
324335
- name: Upload Results
325336
if: always()
326337
uses: actions/upload-artifact@v4
327338
with:
328-
name: smoke-strict-broad-results
329-
path: testing/.smoke-output/broad/
339+
name: smoke-strict-broad-results-${{ matrix.shard }}
340+
path: testing/.smoke-output/broad-${{ matrix.shard }}/
330341
include-hidden-files: true
331342
if-no-files-found: warn
332343
retention-days: 7
@@ -366,8 +377,8 @@ jobs:
366377
path: |
367378
node_modules
368379
~/.bun/install/cache
369-
key: benchmark-deep-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
370-
restore-keys: benchmark-deep-${{ runner.os }}-bun-
380+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
381+
restore-keys: deps-${{ runner.os }}-bun-
371382

372383
- name: Install Dependencies
373384
run: bun install --frozen-lockfile
@@ -423,8 +434,8 @@ jobs:
423434
path: |
424435
node_modules
425436
~/.bun/install/cache
426-
key: pw-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
427-
restore-keys: pw-${{ runner.os }}-bun-
437+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
438+
restore-keys: deps-${{ runner.os }}-bun-
428439

429440
- name: Install Dependencies
430441
run: bun install --frozen-lockfile
@@ -474,8 +485,8 @@ jobs:
474485
path: |
475486
node_modules
476487
~/.bun/install/cache
477-
key: web-roundtrip-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
478-
restore-keys: web-roundtrip-${{ runner.os }}-bun-
488+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
489+
restore-keys: deps-${{ runner.os }}-bun-
479490

480491
- name: Install Dependencies
481492
run: bun install --frozen-lockfile
@@ -519,8 +530,8 @@ jobs:
519530
with:
520531
path: |
521532
node_modules
522-
key: benchmark-install-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
523-
restore-keys: benchmark-install-${{ runner.os }}-bun-
533+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
534+
restore-keys: deps-${{ runner.os }}-bun-
524535

525536
- name: Install Dependencies
526537
run: bun install --frozen-lockfile
@@ -564,8 +575,8 @@ jobs:
564575
with:
565576
path: |
566577
node_modules
567-
key: package-managers-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
568-
restore-keys: package-managers-${{ runner.os }}-bun-
578+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
579+
restore-keys: deps-${{ runner.os }}-bun-
569580

570581
- name: Install Dependencies
571582
run: bun install --frozen-lockfile

‎.github/workflows/generated-project-proof.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,16 @@ jobs:
7272
otp-version: 28.0.2
7373
elixir-version: 1.19.0
7474

75+
- name: Cache Dependencies
76+
uses: actions/cache@v4
77+
with:
78+
path: |
79+
node_modules
80+
~/.bun/install/cache
81+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
82+
restore-keys: |
83+
deps-${{ runner.os }}-bun-
84+
7585
- name: Install repository dependencies
7686
run: bun install --frozen-lockfile
7787

‎.github/workflows/smoke-test.yaml‎

Lines changed: 55 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ on:
2727
outputs:
2828
summary:
2929
description: "Markdown smoke-test summary"
30-
value: ${{ jobs.smoke.outputs.summary }}
30+
value: ${{ jobs.smoke-summary.outputs.summary }}
3131
push:
3232
branches: [main]
3333
paths:
@@ -77,11 +77,13 @@ concurrency:
7777

7878
jobs:
7979
smoke:
80-
name: Smoke Test
80+
name: Smoke Test (${{ matrix.shard }}/3)
8181
runs-on: ubuntu-latest
82-
timeout-minutes: 60
83-
outputs:
84-
summary: ${{ steps.smoke-summary.outputs.summary }}
82+
timeout-minutes: 20
83+
strategy:
84+
fail-fast: false
85+
matrix:
86+
shard: [0, 1, 2]
8587
services:
8688
postgres:
8789
image: postgres:16
@@ -170,9 +172,9 @@ jobs:
170172
path: |
171173
node_modules
172174
~/.bun/install/cache
173-
key: smoke-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
175+
key: deps-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }}
174176
restore-keys: |
175-
smoke-${{ runner.os }}-bun-
177+
deps-${{ runner.os }}-bun-
176178
177179
- name: Install Dependencies
178180
run: bun install --frozen-lockfile
@@ -202,9 +204,15 @@ jobs:
202204
ECOSYSTEM_FLAG="--ecosystem $SMOKE_ECOSYSTEM"
203205
fi
204206
207+
# Split the requested combo count across the 3 matrix shards; the
208+
# harness generates count*shards combos from the same seed and each
209+
# shard runs its own partition, so coverage is unchanged.
210+
SHARD_COUNT=$(( (SMOKE_COUNT + 2) / 3 ))
211+
205212
bun run testing/smoke-test.ts \
206213
--seed "$SEED" \
207-
--count "$SMOKE_COUNT" \
214+
--count "$SHARD_COUNT" \
215+
--partition ${{ matrix.shard }}/3 \
208216
$ECOSYSTEM_FLAG \
209217
$FORCE_FLAGS \
210218
--output testing/.smoke-output
@@ -216,28 +224,12 @@ jobs:
216224
cat testing/.smoke-output/summary.md >> "$GITHUB_STEP_SUMMARY"
217225
fi
218226
219-
- name: Capture Smoke Summary
220-
id: smoke-summary
221-
if: always()
222-
run: |
223-
{
224-
echo 'summary<<EOF'
225-
if [ -f testing/.smoke-output/summary.md ]; then
226-
cat testing/.smoke-output/summary.md
227-
else
228-
echo '## Smoke Test Results'
229-
echo
230-
echo 'The smoke harness did not produce a summary.'
231-
fi
232-
echo EOF
233-
} >> "$GITHUB_OUTPUT"
234-
235227
- name: Upload Results
236228
if: always()
237229
continue-on-error: true
238230
uses: actions/upload-artifact@v4
239231
with:
240-
name: smoke-results
232+
name: smoke-results-${{ matrix.shard }}
241233
path: |
242234
testing/.smoke-output/
243235
!testing/.smoke-output/**/node_modules/**
@@ -253,3 +245,41 @@ jobs:
253245
include-hidden-files: true
254246
if-no-files-found: warn
255247
retention-days: 7
248+
249+
# The sharded smoke jobs each upload their own summary; combine them so
250+
# callers keep receiving one workflow-level summary output.
251+
smoke-summary:
252+
name: Combine Smoke Summaries
253+
needs: smoke
254+
if: always()
255+
runs-on: ubuntu-latest
256+
timeout-minutes: 5
257+
outputs:
258+
summary: ${{ steps.combine.outputs.summary }}
259+
steps:
260+
- name: Download shard results
261+
uses: actions/download-artifact@v4
262+
with:
263+
pattern: smoke-results-*
264+
path: smoke-shards
265+
266+
- name: Combine summaries
267+
id: combine
268+
run: |
269+
{
270+
echo 'summary<<EOF'
271+
found=0
272+
for file in smoke-shards/smoke-results-*/summary.md; do
273+
if [ -f "$file" ]; then
274+
cat "$file"
275+
echo
276+
found=1
277+
fi
278+
done
279+
if [ "$found" = "0" ]; then
280+
echo '## Smoke Test Results'
281+
echo
282+
echo 'The smoke harness did not produce a summary.'
283+
fi
284+
echo EOF
285+
} >> "$GITHUB_OUTPUT"

‎testing/lib/presets.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,18 @@ describe("preset groups", () => {
7171
]);
7272
});
7373

74+
it("partitions a preset group into disjoint shards covering the whole group", () => {
75+
const shardNames = [0, 1, 2].map((index) =>
76+
getPresetCombos("pr-broad", { index, total: 3 }).map((combo) => combo.name),
77+
);
78+
79+
expect(shardNames.flat().sort()).toEqual([...PR_BROAD_PRESET_NAMES].sort());
80+
expect(new Set(shardNames.flat()).size).toBe(PR_BROAD_PRESET_NAMES.length);
81+
for (const names of shardNames) {
82+
expect(Math.abs(names.length - PR_BROAD_PRESET_NAMES.length / 3)).toBeLessThan(1);
83+
}
84+
});
85+
7486
it("renders complete CLI commands for all presets", () => {
7587
for (const combo of getPresetCombos("all")) {
7688
expect(combo.command).not.toContain(" undefined");

‎testing/lib/presets.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1126,10 +1126,21 @@ function resolvePresetIds(presetId: string): string[] | null {
11261126
return null;
11271127
}
11281128

1129-
export function getPresetCombos(presetId: string): ComboCandidate[] {
1129+
export interface PresetPartition {
1130+
index: number;
1131+
total: number;
1132+
}
1133+
1134+
export function getPresetCombos(presetId: string, partition?: PresetPartition): ComboCandidate[] {
11301135
const resolvedPresetIds = resolvePresetIds(presetId);
11311136
if (resolvedPresetIds) {
1132-
return resolvedPresetIds.map((id) => buildSinglePresetCombo(id));
1137+
// Stride by index so each shard mixes ecosystems and heavy/light presets
1138+
// instead of inheriting the grouped list's ordering.
1139+
const ids =
1140+
partition && partition.total > 1
1141+
? resolvedPresetIds.filter((_, i) => i % partition.total === partition.index)
1142+
: resolvedPresetIds;
1143+
return ids.map((id) => buildSinglePresetCombo(id));
11331144
}
11341145

11351146
const def = SMOKE_TEST_PRESETS[presetId];

‎testing/smoke-test.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,12 @@ if (args.majorDeps) {
446446
console.log();
447447
}
448448
} else if (args.preset) {
449-
combos = getPresetCombos(args.preset);
449+
combos = getPresetCombos(
450+
args.preset,
451+
args.partitionIndex !== undefined && args.partitionTotal
452+
? { index: args.partitionIndex, total: args.partitionTotal }
453+
: undefined,
454+
);
450455
const flags = [
451456
args.devCheck && "dev-check",
452457
args.strict && "strict",

0 commit comments

Comments
 (0)