Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
204 commits
Select commit Hold shift + click to select a range
bb2ee1b
fix(agentic): publish the Dynamo frontend cache hit rate as the overa…
adibarra Oct 23, 2026
ca30549
fix(agentic): exclude warmup and failed requests from the workload su…
adibarra Oct 23, 2026
89d46f8
fix(agentic): report non-object profile JSON as invalid power artifacts
adibarra Oct 23, 2026
25b0eb8
fix(power): record a non-object aggregate as unwritable instead of cr…
adibarra Oct 23, 2026
91ec41c
fix(results): ignore raw agentic dirs of superseded run attempts
adibarra Oct 23, 2026
d0ca47e
perf(results): parse job event times once when reconciling records
adibarra Oct 23, 2026
56473ce
fix(results): compare eval framework and precision case-insensitively
adibarra Oct 23, 2026
48bbbe9
fix(results): include spec decoding in single-node agentic identities
adibarra Oct 23, 2026
e6051b3
fix(evals): fail non-finite and boolean eval scores
adibarra Oct 23, 2026
9427c85
fix(evals): fail validation when the thresholds file cannot be loaded
adibarra Oct 23, 2026
dd9179a
fix(evals): treat empty threshold sections as empty and reject non-ma…
adibarra Oct 23, 2026
4a0ac68
fix(evals): report malformed result files as failures and keep checking
adibarra Oct 23, 2026
ae58bfb
fix(evals): fail each result file that contributes no checked metric
adibarra Oct 23, 2026
2d81f57
fix(evals): split MiniMax native results on newlines only
adibarra Oct 23, 2026
0df936e
fix(evals): retry a MiniMax source download whose SHA256 does not match
adibarra Oct 23, 2026
9d9f13f
fix(evals): retry a Kimi verifier archive download that was cut short
adibarra Oct 23, 2026
f9c90e5
fix(evals): compare the Kimi archive SHA256 case-insensitively
adibarra Oct 23, 2026
b1ac38c
fix(evals): reject non-numeric score thresholds
adibarra Oct 23, 2026
41db6cd
fix(workflows): keep same-named checks from different workflows apart…
adibarra Oct 23, 2026
c4f4db1
fix(workflows): drop only main's new entries when resolving changelog…
adibarra Oct 23, 2026
8765196
fix(github): read the HTTP status when gh prints a hint after it
adibarra Oct 23, 2026
1500bfb
fix(workflows): stop retrying permission-denied 403s while waiting fo…
adibarra Oct 23, 2026
65159a1
fix(workflows): treat a queued check re-run as newer than the run it …
adibarra Oct 23, 2026
7b7828b
perf(yaml): parse repository YAML with libyaml
adibarra Oct 23, 2026
b30f588
fix(datasets): clear stale stage outputs before rebuilding a weka dat…
adibarra Oct 23, 2026
e0e5ac2
fix(datasets): stop echoing the proxy DB URL in the build log
adibarra Oct 23, 2026
7502bda
fix(datasets): let weka dataset builds use random sampling
adibarra Oct 23, 2026
b6677de
fix(datasets): make --limit pick the same sessions on every run
adibarra Oct 23, 2026
9e272ab
fix(datasets): reject a non-positive sampler --limit
adibarra Oct 23, 2026
1645c9f
fix(datasets): keep log histograms valid for tiny or single-valued me…
adibarra Oct 23, 2026
1e7c0dd
fix(srt-slurm): record no golden AL when vLLM verification is adaptive
adibarra Oct 23, 2026
a85bb6a
fix(srt-slurm): report a variant with a missing field instead of abor…
adibarra Oct 23, 2026
68377ac
fix(srt-slurm): refuse a prepare output that would overwrite the boun…
adibarra Oct 23, 2026
ace948f
fix(golden-al): reject curve names that leave the golden directory
adibarra Oct 23, 2026
abddeb6
fix(matrix): compare rows without a recipe fingerprint one by one
adibarra Oct 23, 2026
0107f37
fix(matrix): stop --model-prefix from selecting longer model prefixes
adibarra Oct 23, 2026
0177242
fix(matrix): reject repeated concurrencies in conc-list
adibarra Oct 23, 2026
2c45acb
fix(matrix): reject a lone conc-start or conc-end next to conc-list
adibarra Oct 23, 2026
f77186f
fix(matrix): require a positive tensor parallel size in master configs
adibarra Oct 23, 2026
6a37a91
fix(matrix): count recipe nodes when recipe sections are null
adibarra Oct 23, 2026
999adf0
fix(matrix): report a failing git diff when reading changelog additions
adibarra Oct 23, 2026
0696d5b
fix(matrix): parse changelog diff headers only before the first hunk
adibarra Oct 23, 2026
8fd3dfd
fix(klaud): skip unreadable ownership records instead of aborting rec…
adibarra Oct 23, 2026
7231c6d
fix(klaud): name baseline request-error notes by their concurrency row
adibarra Oct 23, 2026
2caf65d
fix(klaud): fail source reads of a directory with a verification error
adibarra Oct 23, 2026
2c25ca6
fix(klaud): report a missing candidate PR instead of crashing
adibarra Oct 23, 2026
9f47d99
fix(klaud): let finish continue when a run ends before its cancel
adibarra Oct 23, 2026
aca70bc
perf(klaud): reuse session listings until the next write or ownership…
adibarra Oct 23, 2026
79acb47
perf(klaud): rule out failed jobs before reading timeout annotations
adibarra Oct 23, 2026
d41172c
perf(klaud): generate each candidate's base matrix once during selection
adibarra Oct 23, 2026
29e0973
perf(klaud): project each producer's points once while resolving a ba…
adibarra Oct 23, 2026
a710ccf
perf(klaud): resolve the frozen baseline once per attempt report
adibarra Oct 23, 2026
ffb4686
fix(klaud): keep sweeping abandoned work past a claim whose planner r…
adibarra Oct 23, 2026
31c445f
fix(launch): retry a failing squeue before treating a Slurm job as gone
adibarra Oct 23, 2026
e6f74dc
fix(launch): require INFERENCEX_RESULTS_PYTHON before an srt power po…
adibarra Oct 23, 2026
732603e
fix(launch): keep a shared squash when this host cannot validate it
adibarra Oct 23, 2026
6d51253
fix(launch): fail the uv install when its download fails
adibarra Oct 23, 2026
29da3d3
fix(launch): mark image staging configuration failures as not retriable
adibarra Oct 23, 2026
2e60878
fix(launch): stop waiting on Slurm jobs that never leave the queue in…
adibarra Oct 23, 2026
dbf6ffb
fix(launch): cancel a queued salloc request when the launch is interr…
adibarra Oct 23, 2026
afc4add
fix(launch): accept a batch log created just before its job left the …
adibarra Oct 23, 2026
94bc579
fix(launch): reject an OUT_YAML that climbs out of the container work…
adibarra Oct 23, 2026
f11a192
fix(launch): report a runner config failure in run as one error line
adibarra Oct 23, 2026
e3a3f2b
fix(launch): ignore additional settings that are not a JSON list
adibarra Oct 23, 2026
a37c1d8
fix(launch): report a non-object srtctl manifest as a launch error
adibarra Oct 23, 2026
2d26100
fix(launch): mark a container step on a lost allocation as retriable
adibarra Oct 23, 2026
c7d4786
fix(launch): mask secrets from the container env in the echoed srun step
adibarra Oct 23, 2026
bced3b0
fix(matrix): reject duplicate keys in master configs and changelog ad…
adibarra Oct 23, 2026
78a9632
fix(bench): fall back to retokenizing when a stream reports no usage
adibarra Oct 23, 2026
4725453
fix(bench): fail OpenAI stream requests that carry an error event
adibarra Oct 23, 2026
59bf01f
fix(bench): fail chat stream requests that never yield a token chunk
adibarra Oct 23, 2026
e17d1f2
fix(bench): mask credential variables, not token-count settings, in e…
adibarra Oct 23, 2026
29d10fe
fix(bench): skip unselected metrics when building PyTorch benchmark r…
adibarra Oct 23, 2026
953c987
fix(bench): keep equals signs inside --metadata values
adibarra Oct 23, 2026
bafc230
fix(bench): reject a non-positive --request-rate before loading anything
adibarra Oct 23, 2026
7d77974
fix(bench): reject a missing --result-dir before running the benchmark
adibarra Oct 23, 2026
cf04a4c
fix(bench): reject zero or negative --num-prompts with a clear error
adibarra Oct 23, 2026
732dc48
fix(bench): write the goodput result under request_goodput without a …
adibarra Oct 23, 2026
e3ecb2d
fix(bench): stop the benchmark clock before the profiler shutdown req…
adibarra Oct 23, 2026
aeec59d
fix(bench): keep waiting when /v1/models returns a null model list
adibarra Oct 23, 2026
9ff356b
fix(bench): measure the chat-route deadline after the readiness probes
adibarra Oct 23, 2026
b7944b6
fix(bench): keep streaming a truncated server log and whole UTF-8 cha…
adibarra Oct 23, 2026
2418788
fix(bench): return 127 when tee cannot start its command
adibarra Oct 23, 2026
bd6922a
perf(bench): build serial random prompts from plain int offsets
adibarra Oct 23, 2026
641567e
perf(bench): convert latency lists to arrays once in calculate_metrics
adibarra Oct 23, 2026
96e0db2
docs(agentic): show the module invocation for the workload summary
adibarra Oct 23, 2026
2364470
test(bench): drop the completions case of the empty-stream test
adibarra Oct 23, 2026
d68d0e4
test(bench): check the zero-prompt rejection with one case
adibarra Oct 23, 2026
08d7736
fix(bench): restart log streaming when the server log is replaced
adibarra Oct 23, 2026
affe5af
fix(datasets): rebuild the weka upload payload from scratch
adibarra Oct 23, 2026
55499a9
fix(srt-slurm): record no golden AL when the generation role verifies…
adibarra Oct 23, 2026
f3744be
fix(klaud): count an owner whose planner run recovery cannot read as …
adibarra Oct 23, 2026
6851185
fix(klaud): decide claim liveness before the sweep closes abandoned work
adibarra Oct 23, 2026
6de49ab
test(klaud): check that a session lists fresh state after a GitHub write
adibarra Oct 23, 2026
1480b68
fix(evals): reject falsy non-mapping threshold sections
adibarra Oct 23, 2026
d55e319
fix(evals): fail malformed task entries and undecodable result files
adibarra Oct 23, 2026
7ff2ec1
fix(launch): read squeue's invalid job id answer as a job no longer l…
adibarra Oct 23, 2026
593a233
test(launch): keep host binaries out of the missing-unsquashfs squash…
adibarra Oct 23, 2026
cae260a
fix(launch): reject additional settings that are not a JSON list by name
adibarra Oct 23, 2026
6737bc8
fix(srt-slurm): reject a submission manifest that is not a JSON object
adibarra Oct 23, 2026
eb9f788
fix(launch): cancel an interrupted salloc request without waiting on …
adibarra Oct 23, 2026
1393c87
perf(srt-slurm): copy recipe trees without copy.deepcopy dispatch
adibarra Oct 23, 2026
c00584b
perf(yaml): parse each repeated YAML text once per process
adibarra Oct 23, 2026
8442367
fix(changelog): reject NEL, LS and PS line breaks in perf-changelog.yaml
adibarra Oct 23, 2026
f7b7d96
fix(changelog): reject a whitespace-only last line after appended ent…
adibarra Oct 23, 2026
3df0f95
fix(datasets): only reuse a completed proxy sample with --skip-sample
adibarra Oct 23, 2026
14d4043
fix(datasets): only reuse converted traces that match the proxy sample
adibarra Oct 23, 2026
48072c7
fix(datasets): recompute model lists after the 256k and ISL filters
adibarra Oct 23, 2026
68c2ea3
fix(datasets): stop logging key=value proxy DB conninfo with its pass…
adibarra Oct 23, 2026
2031be8
fix(evals): register BFCL models under the name evaluation looks up
adibarra Oct 23, 2026
99b1d01
fix(bench): skip SSE comment and event lines in OpenAI streams
adibarra Oct 23, 2026
4fde8b6
fix(workflows): retry dropped connections while waiting for merge checks
adibarra Oct 23, 2026
1049f55
fix(workflows): keep the check-changelog verdict when a later run ski…
adibarra Oct 23, 2026
7fb5980
perf(matrix): fork base-revision generators from a preloaded process
adibarra Oct 23, 2026
56c085b
perf(matrix): build the head matrix while base generators run
adibarra Oct 23, 2026
6cddf97
fix(evals): keep pipeline and context parallel sizes in collected eva…
adibarra Oct 23, 2026
fbb7334
fix(launch): read null additional settings as a role that sets none
adibarra Oct 23, 2026
5f3651b
fix(matrix): keep GSM8K evals when trimming all-evals vendor rows
adibarra Oct 23, 2026
d9e552d
fix(matrix): keep standalone evals out of smoke throughput rows
adibarra Oct 23, 2026
fb383eb
fix(matrix): keep one eval per app identity after trimming concurrency
adibarra Oct 23, 2026
844c593
fix(matrix): read node counts from zip_override recipe variants
adibarra Oct 23, 2026
743f29d
fix(golden-al): reject a thinking mode that is not a mapping of draft…
adibarra Oct 23, 2026
ac89518
fix(golden-al): show a curve whose thinking mode has no measurements
adibarra Oct 23, 2026
9c5a273
fix(klaud): point attempt translations to a table only when one is shown
adibarra Oct 23, 2026
332627d
fix(klaud): translate where a chunked attempt's results continue
adibarra Oct 23, 2026
45969e4
fix(klaud): translate an unknown repair count in closing reports
adibarra Oct 23, 2026
c5035c1
fix(changelog): report a tagged non-mapping entry as invalid YAML
adibarra Oct 23, 2026
3da2f26
fix(bench): name every physical GPU in multi-node sweep result files
adibarra Oct 23, 2026
94314a7
fix(matrix): reject plan rows for sequence lengths no sweep job runs
adibarra Oct 23, 2026
780acc8
fix(matrix): trim single-node bundle points to one row per deployment
adibarra Oct 23, 2026
3e6c2ca
docs(configs): list the fixed-sequence lengths PR sweeps run
adibarra Oct 23, 2026
000a10b
fix(yaml): match PyYAML where adversarial testing found libyaml diffe…
adibarra Oct 23, 2026
d7f7906
fix(launch): record no Slurm end time while the allocation still runs
adibarra Oct 23, 2026
5813782
fix(results): publish null for a non-numeric eval standard error
adibarra Oct 23, 2026
03ecb1f
fix(workflows): keep polling merge checks after PyGithub gives up on …
adibarra Oct 23, 2026
41bae00
chore(workflows): authenticate the merge helper with PyGithub's Auth.…
adibarra Oct 23, 2026
665673f
fix(launch): exit 128 plus the signal when a launch step dies by signal
adibarra Oct 23, 2026
aa4f3fc
fix(srt-slurm): exit 128 plus the signal when srtctl apply dies by si…
adibarra Oct 23, 2026
44a5977
fix(launch): exit with the wrapped launch's code on the srt-batch path
adibarra Oct 23, 2026
294be8c
fix(matrix): exit 128 plus the signal when a revision's tool dies by …
adibarra Oct 23, 2026
17555db
fix(bench): stream a dying server's last log lines before failing the…
adibarra Oct 23, 2026
d8d5b73
docs(evals): describe single-node eval groups and BFCL full-suite tas…
adibarra Oct 23, 2026
eed9ed1
docs(matrix): say what --all-evals selects and that it emits eval job…
adibarra Oct 23, 2026
5b74664
docs(golden-al): describe the SGLang collector, dflash lookup and srt…
adibarra Oct 23, 2026
c1db578
docs(golden-al): list dflash among the lookup help's methods
adibarra Oct 23, 2026
696c8b0
docs(datasets): stop saying the weka build's --limit forces top sampling
adibarra Oct 23, 2026
71b1fbf
docs(datasets): say the sampler hashes random seeds locally
adibarra Oct 23, 2026
0211522
docs(datasets): say the weka converter writes traces into --out-dir
adibarra Oct 23, 2026
9561540
docs(bench-serving): show a client command that uses the random dataset
adibarra Oct 23, 2026
1cc7ea6
docs(matrix): describe how the concurrency and parallelism caps rewri…
adibarra Oct 23, 2026
5094159
fix(launch): skip PyTorch-format sidecars when staging srt results
adibarra Oct 23, 2026
8969334
fix(matrix): reject test-config concurrencies no selected config gene…
adibarra Oct 23, 2026
4dd3b62
fix(matrix): reject full-sweep filter values that no config has
adibarra Oct 23, 2026
574a525
fix(workflows): reject misspelled keys in the CI priority policy
adibarra Oct 23, 2026
8be2d08
fix(evals): reject unknown sections in the eval thresholds file
adibarra Oct 23, 2026
ddbdf5d
fix(datasets): reject --dwbug-min-peak without the dynamic-workflow f…
adibarra Oct 23, 2026
b87ed87
fix(results): compare a sweep result only with its own model's baseline
adibarra Oct 23, 2026
b5e5044
fix(results): look up compare baselines under the GPU a runner label …
adibarra Oct 23, 2026
02ea3c1
fix(results): look up sglang-disagg compare baselines as mori-sglang
adibarra Oct 23, 2026
1db2542
fix(klaud): name sglang-disagg and atom-disagg families by their publ…
adibarra Oct 23, 2026
5d52eeb
fix(matrix): list changed points in generation order when names and c…
adibarra Oct 23, 2026
0641f4f
fix(results): aggregate benchmark results in artifact path order
adibarra Oct 23, 2026
0bfa066
fix(results): collect eval artifacts in directory name order
adibarra Oct 23, 2026
98d00d6
fix(klaud): report fixed-sequence request errors from the benchmark o…
adibarra Oct 23, 2026
f07f9b3
fix(launch): stop waiting on NFS locks after the last cleanup attempt
adibarra Oct 23, 2026
e36c8cf
fix(klaud): match legacy baseline producers on the config keys ingest…
adibarra Oct 23, 2026
67520a1
fix(launch): read a Slurm job killed by signal N as exit 128 plus N
adibarra Oct 23, 2026
7333afd
fix(recipes): match qwen3.5 h100 agentic DRAM to runner memory
adibarra Oct 23, 2026
bc7fb40
fix(recipes): match minimaxm3 h100 agentic DRAM to runner memory
adibarra Oct 23, 2026
95e1bb4
fix(recipes): match qwen3.8next gb300 agentic DRAM to runner memory
adibarra Oct 23, 2026
c078b4d
fix(ci-priority): give draft-model speculative decoding its bonus
adibarra Oct 23, 2026
284a55d
fix(recipes): drop binder-owned fields from llm-d AgentX recipes
adibarra Oct 23, 2026
6f9a6c2
fix(configs): drop dsr1 MI355X FP4 MTP disagg rows duplicated in the …
adibarra Oct 23, 2026
7449574
fix(configs): drop Qwen3.5 FP8 GB300 AgentX midcurve points duplicate…
adibarra Oct 23, 2026
a31ce39
docs(configs): match CONFIGS.md master example to the validator
adibarra Oct 23, 2026
3cf53bc
docs(recipes): list the atom engine and drop the fixed-seq-len workload
adibarra Oct 23, 2026
bb3008f
docs(models): state which A/B baselines active configs still carry
adibarra Oct 23, 2026
c003ce3
docs(runner-setup): point at the scripts under inferencex-e2e
adibarra Oct 23, 2026
024182e
fix(agentx): drop the unread trace idle gap cap from Kimi-K3 recipes
adibarra Oct 23, 2026
3eb50e9
fix(agentx): drop the unread live-assistant switch from GLM-5.2 and M…
adibarra Oct 23, 2026
5600108
fix(agentx): drop the unread warmup override from the DSv4.1-Flash AT…
adibarra Oct 23, 2026
c733445
fix(bench): remove the unread INFERENCEX_RUNTIME_ENV_VARS forwarding …
adibarra Oct 23, 2026
f819333
chore(recipes): delete 1k1k recipes no master config references
adibarra Oct 23, 2026
d8dc468
revert: restore the unreferenced 1k1k recipes
adibarra Oct 23, 2026
90fbf7d
fix(ci): offer draft_model instead of eagle in PR priority classifier
adibarra Oct 23, 2026
1d6ef0b
docs(ci): fix full-sweep examples and all-evals/conc wording
adibarra Oct 23, 2026
d668598
ci: drop workflow env vars that no reader consumes
adibarra Oct 23, 2026
e7e8f49
fix(speedbench-al): let dispatch select variant collectors
adibarra Oct 23, 2026
e7c52c9
fix(speedbench-al): open the PR against the real golden curve file
adibarra Oct 23, 2026
b9f2ad1
docs(speedbench): point the DSv4 collector at its golden curve file
adibarra Oct 23, 2026
d0c9dbc
fix(ci): launch profile runs with the inputs single-node srt needs
adibarra Oct 23, 2026
7a8a279
docs(infx): replace comments with function docstrings
adibarra Oct 23, 2026
54c54d8
docs(matrix): replace comments with function docstrings
adibarra Oct 23, 2026
f6600f4
docs(results): replace comments with function docstrings
adibarra Oct 23, 2026
0f88fc6
docs(launch): replace comments with function docstrings
adibarra Oct 23, 2026
e02dbf0
docs(workflows): replace comments with function docstrings
adibarra Oct 23, 2026
6c6f3cb
docs(bench): replace comments with function docstrings
adibarra Oct 23, 2026
378f462
docs(klaud): replace comments with function docstrings
adibarra Oct 23, 2026
e8b986a
chore(lint): drop ruff suppressions the code can satisfy
adibarra Oct 23, 2026
cad5f17
chore(lint): drop ruff suppressions the code can satisfy
adibarra Oct 23, 2026
f0796a5
refactor(yaml): cache PyYAML parses instead of routing through libyaml
adibarra Oct 23, 2026
9388098
chore(ci): remove comments from workflow files
adibarra Oct 23, 2026
28fc4ba
Merge remote-tracking branch 'origin/main' into perf-lab/2026-10-23
adibarra Oct 24, 2026
107eeab
revert(ci): leave profile.yml as main has it
adibarra Oct 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ usage: python -m infx.matrix.generate full-sweep

If neither `--single-node` nor `--multi-node` is specified, both types are generated.

By default, throughput runs for every generated config and eval-only jobs run for the selected 8k1k subset and the AgentX GSM8K subset. `--no-evals` disables eval jobs, `--evals-only` emits only that selected subset, and adding `--all-evals` expands it to every fixed-sequence config. `--all-evals` alone is an equivalent eval-only shorthand, but it cannot be combined with `--no-evals`.
By default, throughput runs for every generated config and eval-only jobs run for the selected 8k1k subset and the AgentX GSM8K subset. `--no-evals` disables eval jobs, `--evals-only` emits only that selected subset, and adding `--all-evals` expands it to every eligible 8k1k fixed-sequence and agentic config. `--all-evals` always emits only those eval-only jobs, with or without `--evals-only`, but it cannot be combined with `--no-evals`.

`--step-size` must be greater than 1 and applies to concurrency ranges. Explicit `conc-list` values are emitted directly and are filtered by `--min-conc` / `--max-conc` when provided. When both bounds are set, `--min-conc` must not exceed `--max-conc`.
`--step-size` must be greater than 1 and applies to concurrency ranges. `--min-conc` / `--max-conc` drop concurrencies outside the bounds, including explicit `conc-list` values. A single-node fixed-sequence `conc-start`/`conc-end` range that crosses a bound is clamped to it, and a fixed-sequence config with no concurrency left under `--max-conc` runs at the cap. When both bounds are set, `--min-conc` must not exceed `--max-conc`.

`--trim-conc` (the `trim-conc` input of `e2e-tests.yml`) keeps only the minimum concurrency of every generated single- and multi-node deployment shape, after eval selection, for a lowest-concurrency smoke run; in changelog-ref mode only throughput rows are trimmed, and evals keep their selected concurrency. No PR label enables trimming.

Expand All @@ -65,7 +65,7 @@ full-sweep --config-files configs/nvidia-master.yaml

**Test all single-node dsr1 configurations on B200 with 8k1k sequence lengths:**
```
full-sweep --single-node --model-prefix dsr1 --runner-type b200 --seq-lens 8k1k --config-files configs/nvidia-master.yaml
full-sweep --single-node --model-prefix dsr1 --runner-type cluster:b200-nscale --seq-lens 8k1k --config-files configs/nvidia-master.yaml
```

**Test all single-node fp8 precision configs for 8k1k workloads:**
Expand All @@ -75,12 +75,12 @@ full-sweep --single-node --precision fp8 --seq-lens 8k1k --config-files configs/

**Test all single-node TRT configs on H200 runners:**
```
full-sweep --single-node --framework trt --runner-type h200 b200-trt --config-files configs/nvidia-master.yaml
full-sweep --single-node --framework trt --runner-type h200 cluster:h200-dgxc --config-files configs/nvidia-master.yaml
```

**Test specific single-node model on specific hardware with specific sequence lengths:**
```
full-sweep --single-node --model-prefix dsr1 --runner-type b200 --precision fp4 --framework sglang --seq-lens 8k1k --config-files configs/nvidia-master.yaml
full-sweep --single-node --model-prefix dsr1 --runner-type cluster:b200-nscale --precision fp4 --framework sglang --seq-lens 8k1k --config-files configs/nvidia-master.yaml
```

**Limit concurrency and parallelism for faster testing:**
Expand Down
26 changes: 0 additions & 26 deletions .github/workflows/benchmark-multinode-tmpl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ on:
type: string
required: false
default: ''
# Agentic-coding inputs. Fixed-seq-len jobs leave these empty.
scenario-type:
description: "Scenario type (fixed-seq-len or agentic-coding)"
type: string
Expand Down Expand Up @@ -114,17 +113,10 @@ env:
INFERENCEX_E2E_ROOT: ${{ github.workspace }}
PORT: '8888'
INFMAX_CONTAINER_WORKSPACE: '/workspace'
DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1'
MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600'
ENROOT_IMPORT_TIME_LIMIT: '120'
SALLOC_TIME_LIMIT: '480'
KEEP_LOGS: '0'
IS_MULTINODE: 'true'
RANDOM_RANGE_RATIO: 0.8
# Day-zero models resolved via hf: ids download from the Hub inside the
# slurm job (srtctl pre-download + dynamo hub fetch). Anonymous requests
# get 429-rate-limited when several workers pull a 444 GB snapshot at
# once; sbatch/srun inherit this env so the token reaches the workers.
HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
EXP_NAME: ${{ fromJSON(inputs.config).exp-name }}
RECIPE_FINGERPRINT: ${{ fromJSON(inputs.config)['recipe-fingerprint'] || '' }}
Expand Down Expand Up @@ -153,9 +145,7 @@ env:
POWER: ${{ fromJSON(inputs.config).power && '1' || '0' }}
REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }}
POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }}
# Agentic-coding env. Fixed-seq-len jobs leave these empty.
SCENARIO_TYPE: ${{ inputs.scenario-type }}
SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }}
CONC: ${{ inputs.conc }}
DURATION: ${{ inputs.duration }}
Expand Down Expand Up @@ -218,9 +208,6 @@ jobs:
) ||
format('[{0}]', toJSON(inputs.runner))
) }}
# Full-context AgentX warmup can legitimately exceed the fixed-sequence
# eight-hour envelope at high concurrency. Keep one hour beyond the
# recipe's 12-hour Slurm limit for checkout, staging, and artifact upload.
timeout-minutes: ${{ inputs.scenario-type == 'agentic-coding' && 780 || 480 }}
name: >-
${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }}
Expand All @@ -245,8 +232,6 @@ jobs:
done
fi

# A cancelled or crashed job can leave git state in this persistent
# workspace that breaks checkout; re-cloning fixes it and is idempotent.
- id: checkout
continue-on-error: true
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -271,7 +256,6 @@ jobs:
with:
ref: ${{ github.workflow_sha }}
path: .result-tooling
# Only inferencex-e2e/ is read from this checkout.
sparse-checkout: inferencex-e2e
persist-credentials: false

Expand Down Expand Up @@ -320,15 +304,13 @@ jobs:
set -x
RESULT_FILENAME=$(python3 -m infx.results.result_filename)
export RESULT_FILENAME
# Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV"
echo "EVAL_ARTIFACT_RECIPE=${RECIPE_FINGERPRINT:0:16}" >> "$GITHUB_ENV"
eval_artifact_conc="$(python3 -c 'import hashlib,os; print(hashlib.sha256(os.environ["CONC_LIST"].encode()).hexdigest()[:12])')"
echo "EVAL_ARTIFACT_CONC=${eval_artifact_conc}" >> "$GITHUB_ENV"

source benchmarks/runtime_settings.sh
source benchmarks/multi_node/runtime_settings.sh
# Assign data literally; recipe values must never become shell code.
settings_json=$(jq -cen \
--argjson prefill "$PREFILL_ADDITIONAL_SETTINGS" \
--argjson decode "$DECODE_ADDITIONAL_SETTINGS" \
Expand All @@ -338,7 +320,6 @@ jobs:
while IFS= read -r -d '' setting; do
export "$setting"
done < <(jq -j '.[] + "\u0000"' <<< "$settings_json")
# Resolve the workflow's documented automatic eval-concurrency selection.
if [[ -z "$EVAL_CONC" ]]; then
EVAL_CONC=$(python3 -c 'import os; print(max(map(int, os.environ["CONC_LIST"].split())))')
fi
Expand All @@ -347,7 +328,6 @@ jobs:
"$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run
if [ "${EVAL_ONLY}" = "true" ]; then
echo "Eval-only mode: skipping benchmark result file check"
# Verify eval produced results
if ! ls results*.json 1>/dev/null 2>&1; then
echo "Eval-only run failed: no results*.json files found." >&2
exit 1
Expand All @@ -361,9 +341,6 @@ jobs:
echo "Run failed: expected $expected_count agentic results, found ${#agentic_results[@]}." >&2
exit 1
fi
# Existence is not enough: the agentic aggregation step writes the
# aggregate even when aiperf recorded zero valid requests. Require
# successful requests in every concurrency result.
for result_file in "${agentic_results[@]}"; do
echo "Found agentic result file: $result_file"
ok=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(int(bool(d.get('num_requests_successful'))))" "$result_file" 2>/dev/null || echo 0)
Expand All @@ -390,8 +367,6 @@ jobs:
if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
source "$PYTHONPATH/benchmarks/check_env.sh"
check_env_vars INFERENCEX_RESULTS_PYTHON
# Launcher stamp supplies the producer SHA unless the
# power-producer-sha input is set as a manual override
if [ -z "$POWER_PRODUCER_SHA" ] && [ -f power-producer-sha.txt ]; then
export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)"
echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA"
Expand Down Expand Up @@ -455,7 +430,6 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: multinode_server_logs_${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }}
# multinode launchers package server logs into this tarball.
path: |
${{ env.INFERENCEX_E2E_ROOT }}/multinode_server_logs.tar.gz
${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log
Expand Down
16 changes: 0 additions & 16 deletions .github/workflows/benchmark-tmpl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ on:
description: "Git ref (branch/sha) to checkout"
required: false
type: string
# Agentic-coding inputs. Fixed-seq-len jobs leave these empty.
scenario-type:
description: "Scenario type (fixed-seq-len or agentic-coding)"
required: false
Expand Down Expand Up @@ -95,11 +94,8 @@ env:
INFERENCEX_E2E_ROOT: ${{ github.workspace }}
PORT: '8888'
INFMAX_CONTAINER_WORKSPACE: '/workspace'
DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1'
MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600'
ENROOT_IMPORT_TIME_LIMIT: '120'
SALLOC_TIME_LIMIT: '480'
KEEP_LOGS: '0'
IS_MULTINODE: 'false'
REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }}
RANDOM_RANGE_RATIO: 0.8
Expand Down Expand Up @@ -130,9 +126,7 @@ env:
EVAL_ONLY: ${{ inputs.eval-only }}
EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
EVAL_SUITE: ${{ inputs.eval-suite }}
# Agentic-coding env. Fixed-seq-len jobs leave these empty.
SCENARIO_TYPE: ${{ inputs.scenario-type }}
SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }}
KV_OFFLOADING: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offloading) || '' }}
KV_OFFLOAD_BACKEND: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offload-backend.name) || '' }}
Expand Down Expand Up @@ -206,8 +200,6 @@ jobs:
done
fi

# A cancelled or crashed job can leave git state in this persistent
# workspace that breaks checkout; re-cloning fixes it and is idempotent.
- id: checkout
continue-on-error: true
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -232,7 +224,6 @@ jobs:
with:
ref: ${{ github.workflow_sha }}
path: .result-tooling
# Only inferencex-e2e/ is read from this checkout.
sparse-checkout: inferencex-e2e
persist-credentials: false

Expand Down Expand Up @@ -272,8 +263,6 @@ jobs:
RUNNER_TYPE: ${{ inputs.runner }}
JOB_CHECK_RUN_ID: ${{ job.check_run_id }}
RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_a${{ github.run_attempt }}_${{ runner.name }}
# Suppress per-job eval markdown from being appended to the step summary.
# We'll publish a single combined eval table in the collection job instead.
GITHUB_STEP_SUMMARY: ''
run: |
if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
Expand All @@ -284,15 +273,13 @@ jobs:
export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE))
echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV"

# Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV"

source benchmarks/runtime_settings.sh
"$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run

if [ "${EVAL_ONLY}" = "true" ]; then
echo "Eval-only mode: skipping benchmark result file check"
# Verify eval produced results
if ! ls results*.json 1>/dev/null 2>&1; then
echo "Eval-only run failed: no results*.json files found." >&2
exit 1
Expand Down Expand Up @@ -339,7 +326,6 @@ jobs:
echo "no raw result to process: $RESULT_FILENAME.json" >&2
exit 1
fi
# A native power job's launcher stamp names the telemetry producer
if [ -f power-producer-sha.txt ]; then
export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)"
fi
Expand Down Expand Up @@ -467,7 +453,6 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ inputs.eval-only && 'eval_server_logs_' || 'server_logs_' }}${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }}
# fixed-seq writes server.log at root; agentic writes logs under results/.
path: |
${{ env.INFERENCEX_E2E_ROOT }}/server.log
${{ env.INFERENCEX_E2E_ROOT }}/results/*.log
Expand Down Expand Up @@ -584,7 +569,6 @@ jobs:
run: |
if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
rm -f meta_env.json || true
# Remove any eval results JSONs that were moved into workspace
rm -f results*.json || true
rm -f -- ./*_report.json || true
rm -f -- ./*_results.jsonl || true
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,4 @@ jobs:
if: matrix.check == 'Tests'
run: |
python -c "import torch; assert torch.version.cuda is None and torch.version.hip is None"
# SRT is initialized for our connector tests; its upstream suite runs in its own CI.
python -m pytest -c inferencex-e2e/pyproject.toml inferencex-e2e/infx/tests/ inferencex-e2e/utils/ inferencex-e2e/runners/ collectivex/tests/ operatorx/tests/ --ignore=inferencex-e2e/utils/srt-slurm -n 4
2 changes: 0 additions & 2 deletions .github/workflows/claude.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ jobs:
concurrency:
group: pr-review-${{ github.event.pull_request.number }}
cancel-in-progress: false
# Only review PRs opened by someone with write access.
if: contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)
permissions:
contents: read
Expand Down Expand Up @@ -42,7 +41,6 @@ jobs:
uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234
with:
path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }}
# Repository review credential; the job gates eligible review requests.
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env]
track_progress: true
allowed_bots: ''
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/codeowner-signoff-verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ jobs:
concurrency:
group: codeowner-signoff-pr-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr-number }}
cancel-in-progress: false
# The verifier agent reads untrusted PR content, so this job's token is read-only.
# The separate publish job holds the write permissions.
permissions:
contents: read
pull-requests: read # Read PR metadata, diff, and review comments.
Expand Down Expand Up @@ -103,7 +101,6 @@ jobs:
SIGNOFF_KIND: ${{ steps.resolve.outputs.signoff-kind }}
SIGNOFF_FETCH_CMD: ${{ steps.resolve.outputs.signoff-fetch-cmd }}
run: |
# These placeholders must remain literal for envsubst.
# shellcheck disable=SC2016
envsubst '${REPO} ${PR_NUMBER} ${HEAD_SHA} ${SIGNOFF_AUTHOR} ${SIGNOFF_KIND} ${SIGNOFF_FETCH_CMD}' \
< .github/codeowner-signoff-verify-prompt.md \
Expand All @@ -124,13 +121,11 @@ jobs:
if: steps.resolve.outputs.proceed == 'true'
uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234
env:
# Read-only job token: the agent reads untrusted PR content and never publishes.
GH_TOKEN: ${{ github.token }}
GITHUB_TOKEN: ${{ github.token }}
BASH_DEFAULT_TIMEOUT_MS: "1800000"
BASH_MAX_TIMEOUT_MS: "3600000"
with:
# Repository verifier credential; trusted code checks writer authorization before this step.
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env]
github_token: ${{ github.token }}
path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }}
Expand All @@ -141,8 +136,6 @@ jobs:
settings: |
{"fastMode": true}

# Bash is limited to the read-only commands the verifier prompt uses.
# /proc is denied so file tools cannot read credentials from process environments.
claude_args: |
--model 'claude-opus-5-5'
--allowedTools "Read,Write,Glob,Grep,WebFetch,mcp__github_ci__*,Bash(gh pr view:*),Bash(gh pr diff:*),Bash(gh api:*),Bash(gh run view:*),Bash(gh run list:*),Bash(gh run download:*),Bash(git log:*),Bash(git show:*),Bash(uv run --project inferencex-e2e --locked python -m infx.workflows.pareto_coverage:*),Bash(uv run --project inferencex-e2e --locked python -m infx.workflows.signoff_validate:*)"
Expand All @@ -153,8 +146,6 @@ jobs:
already rendered with this run's PR number, head SHA, and sign-off
metadata.

# The publisher reads the verdict only when verification succeeded, so success
# here guarantees an uploaded verdict artifact for the publish job.
- name: Record verification outcome
id: outcome
if: always() && steps.resolve.outputs.proceed == 'true'
Expand Down Expand Up @@ -214,7 +205,6 @@ jobs:
inferencex-e2e/pyproject.toml
inferencex-e2e/uv.lock

# Any download failure fails this job rather than posting a false rejection.
- name: Download verdict
if: needs.verify.outputs.verification-succeeded == 'true'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/collect-job-events.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ jobs:
ref: ${{ inputs.tooling-ref }}
persist-credentials: false

# Job records are diagnostics: they never fail the run.
- name: Download job events
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
Loading
Loading