diff --git a/skills/nemo-mbridge-mlm-bridge-training/evals/evals.json b/skills/nemo-mbridge-mlm-bridge-training/evals/evals.json index fe51488c70..4720d6c45b 100644 --- a/skills/nemo-mbridge-mlm-bridge-training/evals/evals.json +++ b/skills/nemo-mbridge-mlm-bridge-training/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "mlm-bridge-training-positive-recipe-smoke", + "question": "Use the nemo-mbridge-mlm-bridge-training skill. I need a concrete MLM-vs-Bridge correlation smoke plan. Name the Bridge recipe, the Bridge entry point, the MLM entry point, the launch wrapper, and the stale-checkpoint cleanup step.", + "expected_skill": "nemo-mbridge-mlm-bridge-training", + "expected_script": null, + "ground_truth": "The answer should use the MLM-vs-Bridge training skill and recommend vanilla_gpt_pretrain_config for loss-correlation testing. It should name scripts/training/run_recipe.py as the Bridge entry point and 3rdparty/Megatron-LM/pretrain_gpt.py as the Megatron-LM entry point, launched via uv run python -m torch.distributed.run. It should mention MLM needs PYTHONPATH=3rdparty/Megatron-LM:$PYTHONPATH, Bridge should remove stale nemo_experiments before a fresh run, and matched losses should agree within BF16 rounding. It should not tell the user to edit files under 3rdparty/Megatron-LM.", + "expected_behavior": [ + "Read the nemo-mbridge-mlm-bridge-training skill before answering.", + "Identify that the task is about running Megatron Bridge or Megatron-LM training, not model conversion or performance tuning alone.", + "Recommend vanilla_gpt_pretrain_config for correlation testing.", + "Name scripts/training/run_recipe.py and 3rdparty/Megatron-LM/pretrain_gpt.py as the Bridge and MLM entry points.", + "Mention uv run python -m torch.distributed.run, MLM PYTHONPATH, and rm -rf nemo_experiments.", + "Avoid instructing the user to modify files under 3rdparty/Megatron-LM directly." + ] + } +] diff --git a/skills/nemo-mbridge-multi-node-slurm/SKILL.md b/skills/nemo-mbridge-multi-node-slurm/SKILL.md index 73d8c6610b..4d7b4073d8 100644 --- a/skills/nemo-mbridge-multi-node-slurm/SKILL.md +++ b/skills/nemo-mbridge-multi-node-slurm/SKILL.md @@ -20,34 +20,9 @@ Convert single-node `uv run python -m torch.distributed.run` commands into multi ## Cluster Environment -### Container +Use a shared filesystem for the repository, data, logs, `HF_HOME`, `UV_CACHE_DIR`, and `NEMO_HOME`. `NEMO_HOME` must not use the container-local default (`/root/.cache/nemo`) for multi-node SFT/PEFT jobs, because packed-sequence data prepared on node 0 must be visible to the other nodes. -```bash -CONTAINER_IMAGE=".sqsh" -CONTAINER_MOUNTS=":,:/opt/Megatron-Bridge,:/opt/data" -``` - -### Standard Paths - -```bash -WORKDIR="/opt/Megatron-Bridge" -DATA_PATH="/dclm_01_01_text_document" -``` - -### Tokens / Caches - -```bash -export GH_TOKEN= -export HF_TOKEN= -export HF_HOME=/HF_HOME -export UV_CACHE_DIR="/uv_cache" -export NEMO_HOME="/cache/nemo" -``` - -**Important**: `NEMO_HOME` must point to a shared filesystem (e.g. Lustre) for multi-node SFT/PEFT jobs. -The default (`/root/.cache/nemo`) is container-local and not shared across nodes. -Without this, packed-sequence data files prepared on node 0 are invisible to other -nodes, causing `TypeError: 'NoneType' object is not an iterator`. +Keep credentials out of sbatch templates and logs. Provide `HF_TOKEN`, `GH_TOKEN`, and `WANDB_API_KEY` through the scheduler environment or a restricted secrets file, and never hardcode token values in the script body. For copy-paste environment and sbatch templates, read `references/templates.md`. ### Log Directory @@ -75,27 +50,7 @@ Slurm spawns all processes directly. No `torch.distributed.run`, no TRAIN_CMD es ### Build and Launch -Two-phase srun: first a single-process srun to populate the uv cache, then the full multi-node srun. - -```bash -# Env exports at sbatch level (before srun) -export TORCH_NCCL_AVOID_RECORD_STREAMS=1 -export NCCL_NVLS_ENABLE=0 - -# Phase 1: Single-process uv sync to build/populate the shared cache -srun --mpi=pmix -N 1 --ntasks=1 \ - --container-image="$CONTAINER_IMAGE" \ - --container-mounts="$CONTAINER_MOUNTS" \ - --no-container-mount-home \ - bash -c "cd $WORKDIR && uv sync" - -# Phase 2: Full multi-node run (uv sync is a fast no-op since cache is warm) -srun --mpi=pmix \ - --container-image="$CONTAINER_IMAGE" \ - --container-mounts="$CONTAINER_MOUNTS" \ - --no-container-mount-home \ - bash -c "cd $WORKDIR && uv sync && uv run --no-sync python " -``` +Use a two-phase `srun` pattern: first run a single-process `uv sync` to populate the shared cache, then launch the full multi-node job. The full copy-paste version lives in `references/templates.md`. ### srun-native Key Points @@ -152,43 +107,11 @@ uv run python -m torch.distributed.run \ Use the same two-phase pattern: first a single-process srun to warm the uv cache, then the full run. -**Environment exports go inside TRAIN_CMD** (they must be set inside the container): - -```bash -TRAIN_CMD=" -export CUDA_DEVICE_MAX_CONNECTIONS=1 && \ -export NVTE_ALLOW_NONDETERMINISTIC_ALGO=1 && \ -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True && \ -export NCCL_NVLS_ENABLE=0 && \ -export GH_TOKEN=$GH_TOKEN && \ -export HF_TOKEN=$HF_TOKEN && \ -export HF_HOME=$HF_HOME && \ -export UV_CACHE_DIR=$UV_CACHE_DIR && \ -wandb login \$WANDB_API_KEY && \ -mkdir -p $LOGDIR && \ -cd $WORKDIR && \ -uv sync && \ - -" -``` +Set runtime variables inside the container, but do not inject token values into a long `bash -c` string. Export credentials through the scheduler or source a restricted secrets file before the job starts. Keep `HF_HOME`, `UV_CACHE_DIR`, and `NEMO_HOME` on shared storage. ### 4. Launch (two-phase) -```bash -# Phase 1: Single-process uv sync to build/populate the shared cache -srun --mpi=pmix -N 1 --ntasks=1 \ - --container-image="$CONTAINER_IMAGE" \ - --container-mounts="$CONTAINER_MOUNTS" \ - --no-container-mount-home \ - bash -c "cd $WORKDIR && uv sync" - -# Phase 2: Full multi-node run (uv sync in TRAIN_CMD is a fast no-op) -srun --mpi=pmix --no-kill \ - --container-image="$CONTAINER_IMAGE" \ - --container-mounts="$CONTAINER_MOUNTS" \ - --no-container-mount-home \ - bash -c "$TRAIN_CMD" 2>&1 | tee "$LOGDIR/_${SLURM_JOB_ID}.log" -``` +Use the two-phase launch template in `references/templates.md`, keeping `#SBATCH --ntasks-per-node=1` for this legacy approach. ### 5. (Optional) Add Loss Extraction Footer diff --git a/skills/nemo-mbridge-multi-node-slurm/evals/evals.json b/skills/nemo-mbridge-multi-node-slurm/evals/evals.json index fe51488c70..b2671f0d81 100644 --- a/skills/nemo-mbridge-multi-node-slurm/evals/evals.json +++ b/skills/nemo-mbridge-multi-node-slurm/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "multi-node-slurm-positive-sbatch-smoke", + "question": "Use the nemo-mbridge-multi-node-slurm skill. For a Megatron Bridge recipe that goes through initialize.py, convert my single-node launch to a two-node Slurm sbatch plan. Include the preferred srun-native shape, shared cache/mount requirements, and the exact first log checks for NCCL timeout debugging.", + "expected_skill": "nemo-mbridge-multi-node-slurm", + "expected_script": null, + "ground_truth": "The answer should use the multi-node Slurm skill and recommend the Bridge srun-native pattern: Slurm launches 8 tasks per node, not torch.distributed.run spawning inside one Slurm task. It should state that Bridge derives RANK, WORLD_SIZE, LOCAL_RANK, MASTER_ADDR, and MASTER_PORT from SLURM env vars, require shared filesystem paths for repo/data/logs/HF_HOME/UV_CACHE_DIR/NEMO_HOME plus container mounts, and give the first timeout-debugging checks: grep for real errors while filtering noise, inspect the first failed rank/node, and check NCCL ncclUniqueId/timeout or rank-0 crash lines.", + "expected_behavior": [ + "Read the nemo-mbridge-multi-node-slurm skill before answering.", + "Identify the task as multi-node Slurm launch conversion.", + "Recommend the srun-native Bridge approach with Slurm spawning 8 tasks per node.", + "Mention that Bridge derives distributed rank and rendezvous variables from SLURM env vars.", + "Require shared cache/storage paths and container mounts for multi-node jobs.", + "List the first NCCL timeout debugging checks: filtered error grep, first failed rank/node, and ncclUniqueId or rank-0 crash lines." + ] + } +] diff --git a/skills/nemo-mbridge-multi-node-slurm/references/templates.md b/skills/nemo-mbridge-multi-node-slurm/references/templates.md index 5d32b66584..67c6227b4d 100644 --- a/skills/nemo-mbridge-multi-node-slurm/references/templates.md +++ b/skills/nemo-mbridge-multi-node-slurm/references/templates.md @@ -42,8 +42,9 @@ SEED=1234 TRAIN_ITERS=20 # ── Tokens / Caches ────────────────────────────────────────────────────── -export GH_TOKEN= -export HF_TOKEN= +# Provide tokens through the scheduler environment or a chmod 600 secrets file. +# Never hardcode token values in this script or write them to logs. +: "${HF_TOKEN:?Set HF_TOKEN in the secure job environment before submitting}" export HF_HOME=/HF_HOME export UV_CACHE_DIR="/uv_cache" export NEMO_HOME="/cache/nemo" @@ -54,8 +55,6 @@ export CUDA_DEVICE_MAX_CONNECTIONS=1 && \ export NVTE_ALLOW_NONDETERMINISTIC_ALGO=1 && \ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True && \ export NCCL_NVLS_ENABLE=0 && \ -export GH_TOKEN=$GH_TOKEN && \ -export HF_TOKEN=$HF_TOKEN && \ export HF_HOME=$HF_HOME && \ export UV_CACHE_DIR=$UV_CACHE_DIR && \ export NEMO_HOME=$NEMO_HOME && \ diff --git a/skills/nemo-mbridge-perf-activation-recompute/SKILL.md b/skills/nemo-mbridge-perf-activation-recompute/SKILL.md index c375a86704..8622dcfe10 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/SKILL.md +++ b/skills/nemo-mbridge-perf-activation-recompute/SKILL.md @@ -44,18 +44,11 @@ cost entirely, but it is **incompatible with PP > 1**. ## Enablement -### Selective recompute (default for most recipes) +### Selective recompute ```python cfg.model.recompute_granularity = "selective" -cfg.model.recompute_modules = ["core_attn"] -``` - -### Selective recompute with additional modules - -```python -cfg.model.recompute_granularity = "selective" -cfg.model.recompute_modules = ["core_attn", "layernorm"] # or ["mlp"] or ["mlp", "core_attn"] +cfg.model.recompute_modules = ["core_attn"] # add "layernorm", "mlp", or other valid modules as needed ``` ### Full-layer recompute diff --git a/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json b/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json index fe51488c70..69040b9e32 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json +++ b/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "activation-recompute-positive-memory-smoke", + "question": "Use the nemo-mbridge-perf-activation-recompute skill. My Megatron Bridge model is close to OOM and an FP8 config already uses TE-scoped CUDA graphs. Give the exact recompute order, the first environment fix, and the full-recompute CUDA-graph assertion workaround.", + "expected_skill": "nemo-mbridge-perf-activation-recompute", + "expected_script": null, + "ground_truth": "The answer should use the activation recompute skill. It should say to try PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True first, then start with recompute_granularity=\"selective\" and recompute_modules=[\"core_attn\"], optionally add layernorm, and use full recompute only if selective still does not fit. It should state full recompute requires recompute_method and recompute_num_layers, and that full/layer-level recompute is incompatible with TE-scoped CUDA graph scopes such as attn, mlp, or moe_router. It should give valid workarounds: use selective recompute, disable CUDA graphs with cuda_graph_impl=\"none\", or switch to cuda_graph_impl=\"local\" with cuda_graph_scope=\"full_iteration\".", + "expected_behavior": [ + "Read the nemo-mbridge-perf-activation-recompute skill before answering.", + "Recommend PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True before recompute changes.", + "Prefer selective recompute with core_attn and optionally layernorm before full recompute.", + "State that full recompute requires recompute_method and recompute_num_layers.", + "Explain the TE-scoped CUDA graph incompatibility and list a valid workaround." + ] + } +] diff --git a/skills/nemo-mbridge-perf-cpu-offloading/SKILL.md b/skills/nemo-mbridge-perf-cpu-offloading/SKILL.md index 1ea0d1ce9a..c8f52df0d1 100644 --- a/skills/nemo-mbridge-perf-cpu-offloading/SKILL.md +++ b/skills/nemo-mbridge-perf-cpu-offloading/SKILL.md @@ -106,61 +106,6 @@ Activation offloading is blocked for Qwen3-30B-A3B and similar large MoE models. The PP=1 constraint means each GPU holds all 48 layers; model weights + optimizer states alone (~70 GB) exceed H100 80 GB capacity. -## Minimal Working Config - -### Optimizer offload (50%, balanced) - -```python -cfg.optimizer.optimizer_cpu_offload = True -cfg.optimizer.optimizer_offload_fraction = 0.5 -``` - -### Optimizer offload (100% + overlap, max savings) - -```python -cfg.optimizer.optimizer_cpu_offload = True -cfg.optimizer.optimizer_offload_fraction = 1.0 -cfg.optimizer.overlap_cpu_optimizer_d2h_h2d = True -``` - -### Activation offload (small model, PP=1) - -```python -cfg.model.cpu_offloading = True -cfg.model.cpu_offloading_num_layers = 16 -cfg.model.cpu_offloading_activations = True -cfg.model.cpu_offloading_weights = False -cfg.model.pipeline_model_parallel_size = 1 -cfg.model.recompute_granularity = None -``` - -### Weight offload only (small model, PP=1) - -```python -cfg.model.cpu_offloading = True -cfg.model.cpu_offloading_num_layers = 8 -cfg.model.cpu_offloading_activations = False -cfg.model.cpu_offloading_weights = True -cfg.model.pipeline_model_parallel_size = 1 -cfg.model.recompute_granularity = None -``` - -### Both activations and weights (small model, PP=1) - -```python -cfg.model.cpu_offloading = True -cfg.model.cpu_offloading_num_layers = 8 -cfg.model.cpu_offloading_activations = True -cfg.model.cpu_offloading_weights = True -cfg.model.pipeline_model_parallel_size = 1 -cfg.model.recompute_granularity = None -``` - -Weight offloading and activation offloading share the same constraints (PP=1, -no recompute, no CUDA graphs). Weight offloading has not been tested in -the Qwen3-30B-A3B experiments — the measured results cover optimizer -offloading only. - ## Minimal Runnable Command ```bash @@ -261,26 +206,6 @@ uv run python -m pytest \ x, _ = self.linear_out(x) ``` -### MCore model_parallel_config fields - -```3rdparty/Megatron-LM/megatron/core/model_parallel_config.py - cpu_offloading: bool = False - cpu_offloading_num_layers: int = 0 - cpu_offloading_activations: bool = True - cpu_offloading_weights: bool = False - cpu_offloading_double_buffering: bool = False - cpu_offloading_retain_pinned_cpu_buffers: bool = False -``` - -### MCore optimizer offload config - -```3rdparty/Megatron-LM/megatron/core/optimizer/optimizer_config.py - optimizer_cpu_offload: bool = False - optimizer_offload_fraction: float = 0.0 - use_torch_optimizer_for_cpu_offload: bool = False - overlap_cpu_optimizer_d2h_h2d: bool = False -``` - ## Failure Diagnosis | Symptom | Likely Cause | How To Confirm | Fix | diff --git a/skills/nemo-mbridge-perf-cpu-offloading/evals/evals.json b/skills/nemo-mbridge-perf-cpu-offloading/evals/evals.json index fe51488c70..f8240951b4 100644 --- a/skills/nemo-mbridge-perf-cpu-offloading/evals/evals.json +++ b/skills/nemo-mbridge-perf-cpu-offloading/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "cpu-offloading-positive-optimizer-smoke", + "question": "Use the nemo-mbridge-perf-cpu-offloading skill. For a Qwen3-30B-A3B Megatron Bridge run that needs pipeline parallelism and still has GPU memory pressure, should I use activation CPU offloading or optimizer CPU offloading? Include the exact optimizer config knobs and the main activation-offload constraints.", + "expected_skill": "nemo-mbridge-perf-cpu-offloading", + "expected_script": null, + "ground_truth": "The answer should use the CPU offloading skill, choose optimizer CPU offloading for large MoE models that need pipeline parallelism, and explain that layer-level activation CPU offloading requires pipeline_model_parallel_size=1. It should include optimizer.optimizer_cpu_offload=True, optimizer.optimizer_offload_fraction, and optionally optimizer.overlap_cpu_optimizer_d2h_h2d=True. It should mention activation offloading constraints: PP=1, no activation recompute, no CUDA graphs, and cpu_offloading_num_layers in range.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-cpu-offloading skill before answering.", + "Choose optimizer CPU offloading for a large MoE model that needs pipeline parallelism.", + "List optimizer.optimizer_cpu_offload and optimizer.optimizer_offload_fraction.", + "Mention that activation CPU offloading requires PP=1.", + "Mention that activation CPU offloading cannot combine with recompute or CUDA graphs." + ] + } +] diff --git a/skills/nemo-mbridge-perf-cuda-graphs/BENCHMARK.md b/skills/nemo-mbridge-perf-cuda-graphs/BENCHMARK.md index 9d2a77adb5..bf3fd72ebd 100644 --- a/skills/nemo-mbridge-perf-cuda-graphs/BENCHMARK.md +++ b/skills/nemo-mbridge-perf-cuda-graphs/BENCHMARK.md @@ -7,14 +7,18 @@ This benchmark summarizes 3-Tier Evaluation from NVSkills-Eval results for the s ## Evaluation Summary - Skill: `nemo-mbridge-perf-cuda-graphs` -- Evaluation date: 2026-05-29 +- Evaluation date: 2026-06-01 - NVSkills-Eval profile: `external` -- Overall verdict: FAIL -- Tier 3 live agent evaluation: not available in this report +- Environment: `local` +- Dataset: 3 evaluation tasks +- Attempts per task: 2 +- Pass threshold: 50% +- Overall verdict: PASS ## Agents Used -- Tier 3 agent details were not available in this report. +- `claude-code` +- `codex` ## Metrics Used @@ -28,19 +32,39 @@ Reported benchmark dimensions: Underlying evaluation signals used in this run: -- No Tier 3 evaluation signal details were available in this report. +- `security` (Security): checks for unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): verifies that the agent loaded the expected skill and workflow. +- `skill_efficiency` (Efficiency): checks routing quality, decoy avoidance, and redundant tool usage. +- `accuracy` (Accuracy): grades final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): checks whether the overall user task completed successfully. +- `behavior_check` (Behavior Check): verifies expected behavior steps, including safety expectations. +- `token_efficiency` (Token Efficiency): compares token usage with and without the skill. ## Test Tasks -Tier 3 evaluation task details were not available in this report. +The benchmark dataset contained 3 evaluation tasks: + +- Positive tasks: 2 tasks where the skill was expected to activate. +- Negative tasks: 1 tasks where no skill was expected. +- Unlabeled tasks: 0 tasks where positive/negative intent could not be inferred. + +Task composition is derived from the evaluation dataset when possible. Entries with `expected_skill` set are treated as positive skill-activation cases, while entries with `expected_skill: null` are treated as negative activation cases. ## Results -Tier 3 dimension rollup was not available in this report. +| Dimension | Num | `claude-code` | `codex` | +|---|---:|---:|---:| +| Security | 6 | 100% (+0%) | 100% (+0%) | +| Correctness | 6 | 92% (-2%) | 94% (+13%) | +| Discoverability | 6 | 100% (+0%) | 77% (-2%) | +| Effectiveness | 6 | 91% (-2%) | 95% (+6%) | +| Efficiency | 6 | 92% (-0%) | 67% (-3%) | + +Score values show skill-assisted performance. Values in parentheses show uplift versus the no-skill baseline when baseline data is available. ## Tier 1: Static Validation Summary -Tier 1 validation reported findings. NVSkills-Eval ran 9 checks and found 13 total findings. +Tier 1 validation passed with observations. NVSkills-Eval ran 9 checks and found 10 total findings. Top findings: @@ -61,4 +85,4 @@ Notable observations: ## Publication Recommendation -The skill should be reviewed before NVSkills-Eval publication. Skill owners should address the findings above and rerun NVSkills-Eval to refresh this benchmark. +The skill is suitable to proceed toward NVSkills-Eval publication based on this benchmark. Skill owners should keep this file with the skill and refresh it when the evaluation dataset, skill behavior, or target agents materially change. diff --git a/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md b/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md index 4daa38eae2..812b5f8515 100644 --- a/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md +++ b/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md @@ -7,7 +7,7 @@ when_to_use: Reducing host-driver overhead via CUDA graphs, or tracing a crash o # CUDA Graphs -Stable docs: @docs/training/cuda-graphs.md +Stable documentation: @docs/training/cuda-graphs.md Card: @skills/nemo-mbridge-perf-cuda-graphs/card.yaml ## What It Is diff --git a/skills/nemo-mbridge-perf-cuda-graphs/evals/evals.json b/skills/nemo-mbridge-perf-cuda-graphs/evals/evals.json index fe51488c70..0f0d4db531 100644 --- a/skills/nemo-mbridge-perf-cuda-graphs/evals/evals.json +++ b/skills/nemo-mbridge-perf-cuda-graphs/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "cuda-graphs-te-scoped-moe-smoke", + "question": "Use the nemo-mbridge-perf-cuda-graphs skill. I am training a Megatron Bridge Qwen3 MoE model and want to reduce CPU launch overhead with CUDA graphs. Which cuda_graph_impl and cuda_graph_scope should I start with, and what prerequisites should I set?", + "expected_skill": "nemo-mbridge-perf-cuda-graphs", + "expected_script": null, + "ground_truth": "For most Megatron Bridge training workloads, start with Transformer Engine scoped graphs rather than local full-iteration capture. For a dropless MoE model, use cuda_graph_impl=\"transformer_engine\" with cuda_graph_scope including attn, moe_router, and moe_preprocess. Set cuda_graph_warmup_steps, enable model.use_te_rng_tracker and rng.te_rng_tracker, keep sequence length and micro-batch size static, and compare steady-state replay iterations after warmup and capture. Do not combine moe and moe_router scopes.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-cuda-graphs skill before answering.", + "Recommend transformer_engine scoped graphs for the MoE bring-up path.", + "Name the relevant MoE scopes: attn, moe_router, and moe_preprocess.", + "Mention the TE RNG tracker requirement and static-shape constraint.", + "Tell the user to compare replay timing after warmup and capture rather than measuring the capture step." + ] + } +] diff --git a/skills/nemo-mbridge-perf-cuda-graphs/skill-card.md b/skills/nemo-mbridge-perf-cuda-graphs/skill-card.md index bd2b9c24c4..5606820501 100644 --- a/skills/nemo-mbridge-perf-cuda-graphs/skill-card.md +++ b/skills/nemo-mbridge-perf-cuda-graphs/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and engineers use this skill to enable, validate, and troubleshoot CUDA graph capture in Megatron Bridge training workloads, reducing host-driver overhead for improved GPU throughput on static-shape pretraining runs.
+Developers and engineers enabling CUDA graph capture in Megatron Bridge training workloads to reduce host-driver overhead and improve GPU throughput for pretraining and fine-tuning recipes.
### Deployment Geography for Use:
Global
@@ -20,18 +20,24 @@ Mitigation: Review and scan skill before deployment.
## Reference(s):
- [CUDA Graphs Training Documentation](docs/training/cuda-graphs.md)
-- [Activation Recomputation Documentation](docs/training/activation-recomputation.md)
- [Performance Tuning Guide](docs/performance-guide.md)
+- [Megatron Bridge Documentation](https://docs.nvidia.com/nemo/megatron-bridge/latest/)
## Skill Output:
**Output Type(s):** [Configuration instructions, Shell commands, Analysis]
-**Output Format:** [Markdown with inline code blocks]
+**Output Format:** [Markdown with inline Python and bash code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
+## Evaluation Agents Used:
+- Claude Code (`claude-code`)
+- Codex (`codex`)
+ + + ## Evaluation Tasks:
-Evaluated through NVSkills-Eval 3-Tier validation (external profile, 2026-05-29). Tier 1 static validation and Tier 2 deduplication completed; Tier 3 live agent evaluation not available.
+Evaluated against 3 evaluation tasks (2 positive skill-activation cases, 1 negative case) with 2 attempts per task and a 50% pass threshold.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -41,17 +47,33 @@ Reported benchmark dimensions:
- Effectiveness: Checks whether the agent performs measurably better with the skill than without it.
- Efficiency: Checks whether the agent uses fewer tokens and avoids redundant work.
+Underlying evaluation signals used in this run:
+- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Verifies that the agent loaded the expected skill and workflow.
+- `skill_efficiency`: Checks routing quality, decoy avoidance, and redundant tool usage.
+- `accuracy`: Grades final-answer correctness against the reference answer.
+- `goal_accuracy`: Checks whether the overall user task completed successfully.
+- `behavior_check`: Verifies expected behavior steps, including safety expectations.
+- `token_efficiency`: Compares token usage with and without the skill.
+ ## Evaluation Results:
-| Tier | Checks | Findings | Status | -|---|---:|---:|---| -| Tier 1 (Static Validation) | 9 | 13 | FAIL | -| Tier 2 (Deduplication) | 2 | 0 | PASS | -| Tier 3 (Live Agent) | — | — | Not available | +| Dimension | Num | `claude-code` | `codex` | +|---|---:|---:|---:| +| Security | 6 | 100% (+0%) | 100% (+0%) | +| Correctness | 6 | 92% (-2%) | 94% (+13%) | +| Discoverability | 6 | 100% (+0%) | 77% (-2%) | +| Effectiveness | 6 | 91% (-2%) | 95% (+6%) | +| Efficiency | 6 | 92% (-0%) | 67% (-3%) | + +## Testing Completed:
+**[x] Agent Red-Teaming**
+**[ ] Network Security**
+**[ ] Product Security**
## Skill Version(s):
-67b6ae5d (source: git SHA, committed 2026-05-28)
+v0.2.0rc6-1520-g91d60f3f (source: git describe)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/nemo-mbridge-perf-cuda-graphs/skill.oms.sig b/skills/nemo-mbridge-perf-cuda-graphs/skill.oms.sig index 7f203cd3c3..0d51699574 100644 --- a/skills/nemo-mbridge-perf-cuda-graphs/skill.oms.sig +++ b/skills/nemo-mbridge-perf-cuda-graphs/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmVtby1tYnJpZGdlLXBlcmYtY3VkYS1ncmFwaHMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiNDZlNGY5MDQyODE3ZGI4ZGU3MjkyNWFhYTNiMWE3Mjk4ZjA3NDM3ZGY4MmFhYmQ3YmYyMTIzZDRmZmM4ZjU5OSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZjQxNGRkMTk0NThiNTFhOWMyMzUxYmRkNmRmN2EzNjA0MzY4YmM5NTk0Nzg5YzYyNmUyOWE3YmZiYmY0ZDBhOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiM2M1ZDc4MWIzZDEzMWFhNDcwNzI4OTdiNmM5NTQ4ZWU5MWM4YTNjY2ExMjYzYTJlYzk2NjcxNGIzMThlZDc5NCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI2NTBiZjc2Y2I2MTVhZTc0ODA0MTFmMGI1ZmRkN2MwNWQyNmQ2YzNmZjVkYTRlZjE4ZDJhODc3ZjE0MGNiODBjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiY2FyZC55YW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzNzUxN2U1ZjNkYzY2ODE5ZjYxZjVhN2JiOGFjZTE5MjEyODI0MTVmMTA1NTFkMmRlZmE1YzNlYjA5ODViNTcwIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMGNmY2FmYTIzYmYwZTE3MTYyNTJlMmI1YTg4MDM0N2Q2Mzk1NTU5Y2M2ZmY2OTMzZTkwMzEyYTU3YWE1MmVhNiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDZ/sBsJwjg4o2IjqNi7IKbaEJ3OeiqkZLxwBvDHoZ9Itu6xSFQ0AXRXJ65z9mN8J8CMA0oUihgom5GTo2LZ9I0oVp30+l846zo2AYJQzAPVIj7KSbKA9V3JLLD064ZAKe0FQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmVtby1tYnJpZGdlLXBlcmYtY3VkYS1ncmFwaHMiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiM2U1N2Y0NDliMTVjMmIyYWJiYWU1OTQ0NGRhZTg4NWEwZTQ3NDZlMWQ1YWYwNGJlN2U0ZTUwNTk1NGVlODNlMyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjZmMmQ1MGQ5MDBhNTA4YTAwMjQzNzg4NzdlMTI3NzcyNGQzZDkxMmMwZTkwNDQ3YWVhNjU2NjU4OThjN2NhMmEiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjljYzgzYzRmMWFiYjY5ZDY0MGE1MjQyMjkyMmFmYzU2YzcyZWQxZjg4ODJjYTBkMjAzOGUyODc5MDEzNjM4ODAiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMmIyOGE3NmI3ZDdiODU2ZjllZTkzMDgzYzE1MDI1ZTAyYjc2NzQ0MGYzMDg5OWJmZjIzZDI0YTU1ZGZlZTM5MCIsCiAgICAgICAgIm5hbWUiOiAiY2FyZC55YW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDRhM2Y4YjRkYzlhMGJjMGExNjgyNDY5ODQ4N2UyNWVjZDAxMWE5ZDNhMmE1YzlmMDQwNGE4ZDEyODdiZGY3YSIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJlYjljZmI3NTJjY2MyOGYzMzRiMjJkNzJiZTNhMjE4MGZkODhiMzJjMzdlNTAxMzRiMWM4ZmM1M2FlNGE5YjMiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCyVuU0PXD9XCaBa8+y2a1vzMkv1cPz0psF+MwUlRkElNbZbzASbN4PYe+9v6OG7QUCMQDgXswdO7xSsA4Od9E624yWusWJHmdY/dLSAVxRokrHmCtkdfp4xqQu2J8SsdUsBXw=","keyid":""}]}} \ No newline at end of file diff --git a/skills/nemo-mbridge-perf-expert-parallel-overlap/SKILL.md b/skills/nemo-mbridge-perf-expert-parallel-overlap/SKILL.md index 4b712132f4..b05ac9bffd 100644 --- a/skills/nemo-mbridge-perf-expert-parallel-overlap/SKILL.md +++ b/skills/nemo-mbridge-perf-expert-parallel-overlap/SKILL.md @@ -7,9 +7,6 @@ when_to_use: Enabling EP overlap to hide dispatch/combine latency, or tracing a # MoE Expert-Parallel Overlap Skill -Stable docs: @docs/training/communication-overlap.md -Card: @skills/nemo-mbridge-perf-expert-parallel-overlap/card.yaml - ## References - Stable docs: @docs/training/communication-overlap.md @@ -57,13 +54,48 @@ Expected outcome: - if the run is tiny, communication-light, or dominated by another wall, the gain may be negligible +## Correctness-First alltoall Benchmark + +For the plain EP-overlap isolation benchmark, keep flex dispatch and delayed +wgrad disabled. The measured shape was Qwen3 MoE 30B-A3B SFT on 16 H100 GPUs: +`EP=16`, `alltoall`, BF16, global batch size 1024, CUDA graphs disabled, +`moe_permute_fusion=false`, measured over iterations 3-8. + +Use these overrides for the plain-overlap case: + +```bash +--cuda_graph_impl none \ +--moe_flex_dispatcher_backend None \ +--moe_a2a_overlap false \ +comm_overlap.overlap_moe_expert_parallel_comm=true \ +comm_overlap.delay_wgrad_compute=false \ +model.moe_shared_expert_overlap=false +``` + +Do not use `--moe_a2a_overlap true` for this isolation test: the performance +harness helper enables both `overlap_moe_expert_parallel_comm` and +`delay_wgrad_compute`, so it does not isolate plain EP overlap. + +Steady-window timing from that benchmark: + +| Case | Steady mean | Relative | +|---|---:|---:| +| no EP overlap | 41.25s | 1.000x | +| EP overlap | 31.31s | 1.317x | +| EP overlap plus `delay_wgrad_compute` | 31.20s | 1.322x | + +This is evidence for enabling plain EP overlap on this inter-node all-to-all +shape. It does not show a meaningful independent win from delayed wgrad, and it +does not validate fused MoE permutation because that path was disabled for the +runtime stack. + ## Enablement ### alltoall dispatcher ```python cfg.comm_overlap.overlap_moe_expert_parallel_comm = True -cfg.comm_overlap.delay_wgrad_compute = True +cfg.comm_overlap.delay_wgrad_compute = False cfg.model.moe_shared_expert_overlap = False cfg.model.expert_model_parallel_size = 8 @@ -73,6 +105,9 @@ cfg.model.bf16 = True cfg.model.fp16 = False ``` +Enable `delay_wgrad_compute=True` only after the plain overlap path is known to +work and its extra compatibility constraints have been checked. + ### flex dispatcher (DeepEP or HybridEP) ```python @@ -123,23 +158,6 @@ Use this as the correctness-first starting point. Add delayed wgrad, flex dispatch, and CUDA-graph interactions only after the plain overlap path is known to work. -## Measured Short-Run Evidence - -A 2026-05-18 current-main H100 x16 Qwen3 30B-A3B mock pretraining run used -`EP=16`, `alltoall`, BF16, global batch size 1024, CUDA graphs disabled, and -`moe_permute_fusion=false`. With iterations 3-8 as the steady window: - -| Case | Steady mean | Relative | -|---|---:|---:| -| no EP overlap | 41.25s | 1.000x | -| EP overlap | 31.31s | 1.317x | -| EP overlap plus `delay_wgrad_compute` | 31.20s | 1.322x | - -This is evidence for enabling plain EP overlap on this inter-node all-to-all -shape. It does not show a meaningful independent win from delayed wgrad, and it -does not validate fused MoE permutation because that path was disabled for the -runtime stack. - ## Minimal Runnable Command Performance harness example inside a Slurm allocation. Keep the model, diff --git a/skills/nemo-mbridge-perf-expert-parallel-overlap/evals/evals.json b/skills/nemo-mbridge-perf-expert-parallel-overlap/evals/evals.json index fe51488c70..8fdfa9d90a 100644 --- a/skills/nemo-mbridge-perf-expert-parallel-overlap/evals/evals.json +++ b/skills/nemo-mbridge-perf-expert-parallel-overlap/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "expert-parallel-overlap-positive-alltoall-smoke", + "question": "In Megatron Bridge, my 16x H100 Qwen3-30B-A3B MoE run is dispatch-bound and I want to isolate only expert all-to-all overlap, without flex dispatch or delayed wgrad. Which exact toggles should I set, which convenience flag should I avoid, and what speedup was measured in the short run?", + "expected_skill": "nemo-mbridge-perf-expert-parallel-overlap", + "expected_script": null, + "ground_truth": "The answer should use the expert-parallel overlap skill and focus on the plain alltoall benchmark. It should state the benchmark shape: Qwen3 MoE 30B-A3B SFT, 16 H100 GPUs, EP=16, alltoall, BF16, global batch size 1024, CUDA graphs disabled, moe_permute_fusion=false, with iterations 3-8 as the steady window. It should enable plain EP overlap with --cuda_graph_impl none, --moe_flex_dispatcher_backend None, --moe_a2a_overlap false, comm_overlap.overlap_moe_expert_parallel_comm=true, comm_overlap.delay_wgrad_compute=false, and model.moe_shared_expert_overlap=false. It should warn not to use --moe_a2a_overlap true for this isolation test because the helper enables both overlap_moe_expert_parallel_comm and delay_wgrad_compute. It should quote the timing comparison: no EP overlap 41.25s (1.000x), EP overlap 31.31s (1.317x), EP overlap plus delay_wgrad_compute 31.20s (1.322x), and say delayed wgrad did not show a meaningful independent win in this benchmark.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-expert-parallel-overlap skill before answering.", + "Identify the requested path as the plain alltoall EP-overlap benchmark, not flex dispatch.", + "List the benchmark shape including model, GPU count, EP, dispatcher, precision, batch size, disabled CUDA graphs, and moe_permute_fusion=false.", + "List the exact overrides for plain EP overlap with delay_wgrad_compute=false and moe_shared_expert_overlap=false.", + "Warn not to use --moe_a2a_overlap true for the isolation test.", + "Quote the 41.25s, 31.31s, and 31.20s timing comparison." + ] + } +] diff --git a/skills/nemo-mbridge-perf-hierarchical-context-parallel/SKILL.md b/skills/nemo-mbridge-perf-hierarchical-context-parallel/SKILL.md index a87ff444c8..68ae9fb5e1 100644 --- a/skills/nemo-mbridge-perf-hierarchical-context-parallel/SKILL.md +++ b/skills/nemo-mbridge-perf-hierarchical-context-parallel/SKILL.md @@ -88,25 +88,7 @@ pg_collection = ProcessGroupCollection( ## Implementation Map -### Config definition - -`hierarchical_context_parallel_sizes` is declared in `ModelParallelConfig`: - -``` -# 3rdparty/Megatron-LM/megatron/core/model_parallel_config.py -hierarchical_context_parallel_sizes: Optional[list[int]] = None -# For a2a+p2p, first value = a2a group size, second value = p2p group size. -# Product must equal context_parallel_size. -``` - -`cp_comm_type` is declared in `TransformerConfig`: - -``` -# 3rdparty/Megatron-LM/megatron/core/transformer/transformer_config.py -cp_comm_type: Optional[Union[str, List[str]]] = None -# Can be per-layer (List[str]) or uniform (str). -# Values: "p2p", "all_gather", "a2a", "a2a+p2p" -``` +The code anchors above show the config declarations and argument validation. ### Validation (MCore) diff --git a/skills/nemo-mbridge-perf-hierarchical-context-parallel/evals/evals.json b/skills/nemo-mbridge-perf-hierarchical-context-parallel/evals/evals.json index fe51488c70..15271427d1 100644 --- a/skills/nemo-mbridge-perf-hierarchical-context-parallel/evals/evals.json +++ b/skills/nemo-mbridge-perf-hierarchical-context-parallel/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "hierarchical-context-parallel-positive-long-context-smoke", + "question": "Use the nemo-mbridge-perf-hierarchical-context-parallel skill. For CP=4 hierarchical context parallelism using a2a+p2p, give the exact Bridge config values, divisibility assertions, TE requirement, and log proof that HCP is actually active.", + "expected_skill": "nemo-mbridge-perf-hierarchical-context-parallel", + "expected_script": null, + "ground_truth": "The answer should use the hierarchical context parallel skill. It should set cfg.model.context_parallel_size=4, cfg.model.cp_comm_type=\"a2a+p2p\", and cfg.model.hierarchical_context_parallel_sizes=[2, 2]. It should state prod(hierarchical_context_parallel_sizes) must equal context_parallel_size and seq_length % (2 * context_parallel_size) == 0. It should mention a2a+p2p requires hierarchical_context_parallel_sizes, Transformer Engine >= 1.12.0 is needed for TEDotProductAttention HCP groups, and logs should show HIERARCHICAL_CONTEXT_PARALLEL_GROUPS rather than only CONTEXT_PARALLEL_GROUP.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-hierarchical-context-parallel skill before answering.", + "Identify hierarchical context parallelism as the requested feature.", + "List context_parallel_size=4, cp_comm_type=a2a+p2p, and hierarchical_context_parallel_sizes=[2, 2].", + "Call out product and sequence-length divisibility assertions.", + "Mention the Transformer Engine version requirement.", + "Require log verification of HIERARCHICAL_CONTEXT_PARALLEL_GROUPS." + ] + } +] diff --git a/skills/nemo-mbridge-perf-megatron-fsdp/evals/evals.json b/skills/nemo-mbridge-perf-megatron-fsdp/evals/evals.json index fe51488c70..03718d71e1 100644 --- a/skills/nemo-mbridge-perf-megatron-fsdp/evals/evals.json +++ b/skills/nemo-mbridge-perf-megatron-fsdp/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "megatron-fsdp-positive-enable-smoke", + "question": "Use the nemo-mbridge-perf-megatron-fsdp skill. Give the minimal Megatron Bridge Megatron-FSDP override, the required checkpoint format, and the pitfalls that distinguish Megatron FSDP from Torch FSDP2.", + "expected_skill": "nemo-mbridge-perf-megatron-fsdp", + "expected_script": null, + "ground_truth": "The answer should use the Megatron FSDP skill. It should set cfg.dist.use_megatron_fsdp=True, cfg.ddp.use_megatron_fsdp=True, cfg.ddp.data_parallel_sharding_strategy=\"optim_grads_params\", and cfg.checkpoint.ckpt_format=\"fsdp_dtensor\" when save/load is enabled. It should mention use_torch_fsdp2 is mutually exclusive with Megatron FSDP and is not the validated path, use_tp_pp_dp_mapping is not supported with Megatron FSDP, and Hopper or earlier may need attention to CUDA_DEVICE_MAX_CONNECTIONS because FSDP and TP/CP prefer different settings.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-megatron-fsdp skill before answering.", + "Identify the request as Megatron Bridge Megatron FSDP enablement.", + "List cfg.dist.use_megatron_fsdp and cfg.ddp.use_megatron_fsdp.", + "List data_parallel_sharding_strategy=optim_grads_params and ckpt_format=fsdp_dtensor.", + "Mention FSDP2 mutual exclusion and use_tp_pp_dp_mapping incompatibility.", + "Include a Bridge-specific verification or test path." + ] + } +] diff --git a/skills/nemo-mbridge-perf-memory-tuning/evals/evals.json b/skills/nemo-mbridge-perf-memory-tuning/evals/evals.json index fe51488c70..a2fb21183d 100644 --- a/skills/nemo-mbridge-perf-memory-tuning/evals/evals.json +++ b/skills/nemo-mbridge-perf-memory-tuning/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "memory-tuning-positive-oom-smoke", + "question": "Use the nemo-mbridge-perf-memory-tuning skill. For a Megatron Bridge Llama3 70B SFT run on 32x H100 with TP=4, PP=4, VPP=5, DP=2 that OOMs around 58.8 GB, what exact memory fix should I try first, and why should I not treat VPP, TP=8, or CPU offloading as the first fix?", + "expected_skill": "nemo-mbridge-perf-memory-tuning", + "expected_script": null, + "ground_truth": "The answer should use the memory tuning skill and say the first fix is export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True because the measured Llama3 70B OOM was fragmentation, not raw capacity. It should state that VPP is a throughput/pipeline-bubble knob and does not materially reduce peak memory, TP=8 is a last resort because it caused a severe throughput regression, PP=8 reduces memory but can lose DP and hurt throughput, and CPU offloading is blocked when pipeline_model_parallel_size > 1. It can mention activation recompute as a later option with throughput cost.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-memory-tuning skill before answering.", + "Identify the measured OOM as a fragmentation-style memory problem.", + "Recommend PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True as the first fix.", + "Explain why VPP is not a peak-memory fix.", + "Warn that TP=8, PP=8, CPU offload, and activation recompute have specific throughput or compatibility costs." + ] + } +] diff --git a/skills/nemo-mbridge-perf-moe-comm-overlap/evals/evals.json b/skills/nemo-mbridge-perf-moe-comm-overlap/evals/evals.json index fe51488c70..33fb8cb848 100644 --- a/skills/nemo-mbridge-perf-moe-comm-overlap/evals/evals.json +++ b/skills/nemo-mbridge-perf-moe-comm-overlap/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "moe-comm-overlap-positive-dispatch-combine-smoke", + "question": "Use the nemo-mbridge-perf-moe-comm-overlap skill. Give the exact MoE dispatch/combine overlap knobs, PP/VPP and flex-dispatcher constraints, and the measured inter-node alltoall baseline numbers from the skill.", + "expected_skill": "nemo-mbridge-perf-moe-comm-overlap", + "expected_script": null, + "ground_truth": "The answer should use the MoE communication overlap skill. It should set cfg.comm_overlap.overlap_moe_expert_parallel_comm=True, optionally cfg.comm_overlap.delay_wgrad_compute=True after basic overlap is stable, and cfg.model.moe_shared_expert_overlap=False. It should require num_moe_experts>1, moe_token_dispatcher_type of alltoall or flex, and VPP when PP is active. It should state moe_flex_dispatcher_backend alone is insufficient unless moe_token_dispatcher_type=\"flex\" is set. It should say full recompute is not a good companion, selective recompute is safer, and delayed wgrad adds CUDA-graph constraints. It should include the measured EP=16 alltoall example: no overlap 41.25s, EP overlap 31.31s, EP overlap plus delay_wgrad_compute 31.20s over iterations 3-8.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-moe-comm-overlap skill before answering.", + "Identify MoE expert communication overlap as the target feature.", + "List overlap_moe_expert_parallel_comm, delay_wgrad_compute, and moe_shared_expert_overlap.", + "Mention PP requires VPP and flex requires moe_token_dispatcher_type=flex.", + "Mention recompute and CUDA graph interactions.", + "Quote the 41.25s, 31.31s, and 31.20s timing comparison." + ] + } +] diff --git a/skills/nemo-mbridge-perf-moe-dispatcher-selection/evals/evals.json b/skills/nemo-mbridge-perf-moe-dispatcher-selection/evals/evals.json index fe51488c70..61a66e6de0 100644 --- a/skills/nemo-mbridge-perf-moe-dispatcher-selection/evals/evals.json +++ b/skills/nemo-mbridge-perf-moe-dispatcher-selection/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "moe-dispatcher-selection-positive-smoke", + "question": "Use the nemo-mbridge-perf-moe-dispatcher-selection skill. In Megatron Bridge, give the exact MoE dispatcher selection checklist for H100 versus GB200/NVL72, including the flex backend config names, default DeepEP/HybridEP SM knobs, and what to conclude if the backend package import fails.", + "expected_skill": "nemo-mbridge-perf-moe-dispatcher-selection", + "expected_script": null, + "ground_truth": "The answer should use MoE dispatcher selection guidance. It should recommend alltoall for first bring-up or missing packages, DeepEP as the first tuned choice for H100/B200 when the package is installed, and HybridEP for GB200/GB300 NVL72 or large EP/memory-tight runs. It should state that DeepEP and HybridEP use moe_token_dispatcher_type=\"flex\" with moe_flex_dispatcher_backend=\"deepep\" or \"hybridep\", mention starting knobs --moe-deepep-num-sms 20 and --moe-hybridep-num-sms 16, and say backend import failures are environment limitations, not throughput data.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-moe-dispatcher-selection skill before answering.", + "Compare alltoall, DeepEP, and HybridEP by hardware and EP scale.", + "Name the flex dispatcher backend settings for DeepEP and HybridEP.", + "Mention the DeepEP and HybridEP SM tuning knobs and defaults.", + "Warn that backend import failures mean the environment is missing a package, not that the dispatcher is slow." + ] + } +] diff --git a/skills/nemo-mbridge-perf-moe-hardware-configs/evals/evals.json b/skills/nemo-mbridge-perf-moe-hardware-configs/evals/evals.json index fe51488c70..02a7044b26 100644 --- a/skills/nemo-mbridge-perf-moe-hardware-configs/evals/evals.json +++ b/skills/nemo-mbridge-perf-moe-hardware-configs/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "moe-hardware-configs-positive-platform-smoke", + "question": "Use the nemo-mbridge-perf-moe-hardware-configs skill as an implementation checklist. I am choosing MoE training playbooks for DSV3 and Qwen3 235B on H100 versus GB200/GB300. Give the representative TP/EP/PP/VPP layouts, dispatcher choices, CUDA graph scopes, environment knobs, GB200/GB300 CPU-side tuning note, and the main warning about copying tracker rows.", + "expected_skill": "nemo-mbridge-perf-moe-hardware-configs", + "expected_script": null, + "ground_truth": "The answer should use MoE hardware configuration guidance. It should state DSV3 on H100 uses DeepEP with TP=2, EP=64, PP=8, VPP=4, while DSV3 on GB200 or GB300 uses HybridEP with TP=1, EP=64, PP=4, VPP=4 and CUDA graph scopes attn + moe_router + moe_preprocess. It should state Qwen3 235B on H100 uses DeepEP with TP=2, EP=32, PP=8, VPP=4, while Qwen3 235B on GB200 uses HybridEP with TP=1 or 2, EP=32-64, PP=4 and the same TE-scoped graph path. It should mention CUDA_DEVICE_MAX_CONNECTIONS=1 or 32 depending on overlap/graphs, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True, NCCL_GRAPH_REGISTER=0, CPU-side tuning on GB200/GB300, and warn not to cargo-cult throughput rows.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-moe-hardware-configs skill before answering.", + "Identify the task as a hardware-platform MoE playbook request.", + "Compare H100 DeepEP patterns against GB200/GB300 HybridEP patterns.", + "List representative DSV3 and Qwen3 235B TP/EP/PP/VPP layouts.", + "Mention CUDA graph scopes and environment knobs from the skill.", + "Warn against copying tracker rows without target-stack validation." + ] + } +] diff --git a/skills/nemo-mbridge-perf-moe-long-context/evals/evals.json b/skills/nemo-mbridge-perf-moe-long-context/evals/evals.json index fe51488c70..9f0cb433c3 100644 --- a/skills/nemo-mbridge-perf-moe-long-context/evals/evals.json +++ b/skills/nemo-mbridge-perf-moe-long-context/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "moe-long-context-positive-cp-smoke", + "question": "Use the nemo-mbridge-perf-moe-long-context skill. Give the CP sizing rule of thumb and the representative DSV3 128K H100, DSV3 256K H100, and Qwen3 235B 128K GB200 long-context MoE layouts.", + "expected_skill": "nemo-mbridge-perf-moe-long-context", + "expected_script": null, + "ground_truth": "The answer should use the MoE long-context skill. It should state the CP sizing rule of thumb: CP ~= seq_len / 4096, rounded to a practical power-of-two, while keeping DP alive if possible. It should prefer selective recompute modules such as up_proj, norm, moe, moe_act, or mlp before full recompute, and avoid SDPA-heavy attention recompute at very long context. It should list DSV3 128K on H100 as TP=1 CP=32 EP=32 PP=8 VPP=4 with DeepEP, FP8-class precision, recompute up_proj/norm/moe/mlp, and optimizer CPU offload; DSV3 256K on H100 as TP=1 CP=64 EP=32 PP=8 EDP=2 VPP=4 with DeepEP and the same recompute/offload pattern; Qwen3 235B 128K on GB200 as TP=4 CP=4 EP=32 PP=4 VPP=12 with HybridEP, BF16 or MXFP8, recompute moe_act/norm, and CUDA graph scopes attn + moe_router + moe_preprocess.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-moe-long-context skill before answering.", + "Identify the task as long-context MoE training guidance.", + "State the CP ~= seq_len / 4096 sizing rule and DP-budget caveat.", + "List the DSV3 128K and 256K H100 layouts.", + "List the Qwen3 235B 128K GB200 layout.", + "Mention selective recompute and CUDA graph stability constraints." + ] + } +] diff --git a/skills/nemo-mbridge-perf-moe-optimization-workflow/evals/evals.json b/skills/nemo-mbridge-perf-moe-optimization-workflow/evals/evals.json index fe51488c70..4da242d509 100644 --- a/skills/nemo-mbridge-perf-moe-optimization-workflow/evals/evals.json +++ b/skills/nemo-mbridge-perf-moe-optimization-workflow/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "moe-optimization-workflow-positive-three-walls-smoke", + "question": "Use the nemo-mbridge-perf-moe-optimization-workflow skill. Give the exact first-pass MoE optimization workflow order, the Parallel Folding meshes, the dispatcher decision rule, the FP8 hardware mapping, and the TE-scoped CUDA graph scopes for dropless MoE.", + "expected_skill": "nemo-mbridge-perf-moe-optimization-workflow", + "expected_script": null, + "ground_truth": "The answer should use the MoE optimization workflow skill. It should say fit first, scale second, profile third, then retune. For memory feasibility it should use the smallest model parallelism that fits, prefer selective recompute before full recompute, add offloading only after recompute/parallelism are insufficient, and use --fake-init-process-group for large layout sanity checks. For scale it should maximize DP once fit, keep hot communication inside fast interconnect, use PP+VPP for multi-node scaling, prefer EP over extra TP for experts, and add CP only when long context makes attention memory dominant. It should show Parallel Folding as Attention: TP x CP x DP x PP and MoE: ETP x EP x EDP x PP, use alltoall for safe bring-up, flex+deepep for H100/B200-style systems, flex+hybridep for GB200/GB300/NVL72-style systems, map Hopper to FP8 blockwise and Blackwell to MXFP8, and start dropless MoE CUDA graphs with attn, moe_router, and moe_preprocess.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-moe-optimization-workflow skill before answering.", + "Present the fit-scale-profile-retune order.", + "Include memory feasibility and parallelism priority details.", + "State the Parallel Folding attention and MoE meshes.", + "Map dispatcher, FP8, and CUDA graph choices to the skill's exact guidance." + ] + } +] diff --git a/skills/nemo-mbridge-perf-moe-vlm-training/evals/evals.json b/skills/nemo-mbridge-perf-moe-vlm-training/evals/evals.json index fe51488c70..10152aa56f 100644 --- a/skills/nemo-mbridge-perf-moe-vlm-training/evals/evals.json +++ b/skills/nemo-mbridge-perf-moe-vlm-training/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "moe-vlm-training-positive-fsdp-vs-3d-smoke", + "question": "Use the nemo-mbridge-perf-moe-vlm-training skill. For Qwen3-VL-style MoE VLM training on GB200, compare the FSDP-first and 3D-parallel paths, including representative TP/CP/PP/EP layout, dispatcher, CUDA graph guidance, and VLM-specific pitfalls.", + "expected_skill": "nemo-mbridge-perf-moe-vlm-training", + "expected_script": null, + "ground_truth": "The answer should use the MoE VLM training skill. It should say FSDP is the simplest first bring-up and memory-first path, especially with awkward PP boundaries, while 3D parallel has the higher ceiling after the model has a clean PP layout and time for deeper sweeps. It should list the FSDP-first GB200 path as TP=1 CP=1 PP=1, EP sized to expert topology, HybridEP on GB200-class systems; and the 3D-parallel GB200 path as TP=1 CP=1 PP=1 or modest PP, EP and ETP sized to expert topology, HybridEP, and CUDA graphs started narrow then widened after the real-data path is stable. It should mention freezing the vision stack for decoder-focused work, aggressive MBS sweeps, matching CUDA graph scope to workload such as attn/moe_router/moe_preprocess only when stable, using ETP only when EP is insufficient, and normalizing metrics by useful tokens rather than only step time.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-moe-vlm-training skill before answering.", + "Identify that the task is about MoE VLM training.", + "Compare FSDP-first and 3D-parallel GB200 paths.", + "List representative TP/CP/PP/EP or ETP layout and HybridEP.", + "Mention CUDA graph scope stability and MBS sensitivity.", + "Call out VLM-specific validation pitfalls such as useful-token normalization." + ] + } +] diff --git a/skills/nemo-mbridge-perf-parallelism-strategies/SKILL.md b/skills/nemo-mbridge-perf-parallelism-strategies/SKILL.md index 712e2c0e1e..183165067d 100644 --- a/skills/nemo-mbridge-perf-parallelism-strategies/SKILL.md +++ b/skills/nemo-mbridge-perf-parallelism-strategies/SKILL.md @@ -183,6 +183,14 @@ Example — TP=2, CP=1, EP=8, ETP=1, PP=1: When sizing slurm scripts, compute `--nodes` from `min_gpus` (or a multiple of it for higher throughput via DP/EDP). +When answering MoE sizing prompts, include this checklist: + +- compute `min_gpus = PP * max(TP * CP, EP * ETP)` with the requested values +- explicitly reject the wrong `PP * TP * CP * EP * ETP` full product +- give both DP formulas: dense `world_size / (TP * PP * CP)` and MoE + `world_size / (PP * EP * ETP)` +- mention TP topology, SP, CP divisibility, and long-sequence CP guidance + ## Memory Estimation Without parallelism (70B model, FP16): diff --git a/skills/nemo-mbridge-perf-parallelism-strategies/evals/evals.json b/skills/nemo-mbridge-perf-parallelism-strategies/evals/evals.json index fe51488c70..ec94af14ec 100644 --- a/skills/nemo-mbridge-perf-parallelism-strategies/evals/evals.json +++ b/skills/nemo-mbridge-perf-parallelism-strategies/evals/evals.json @@ -1 +1,16 @@ -[] +[ + { + "id": "parallelism-strategies-positive-model-size-smoke", + "question": "Use the nemo-mbridge-perf-parallelism-strategies skill. For a Megatron Bridge MoE config with PP=2, TP=2, CP=1, EP=8, ETP=1, calculate the correct minimum GPU count and explain the dense/MoE DP formulas, the wrong full-product shortcut to avoid, and the sequence-length, SP, CP, and topology rules that matter.", + "expected_skill": "nemo-mbridge-perf-parallelism-strategies", + "expected_script": null, + "ground_truth": "The answer should use the parallelism strategy skill. It should state that minimum GPUs for MoE are PP * max(TP * CP, EP * ETP), not PP * TP * CP * EP * ETP; for PP=2, TP=2, CP=1, EP=8, ETP=1 the correct minimum is 2 * max(2, 8) = 16 GPUs, while the wrong product is 32. It should state dense data_parallel_size = world_size / (TP * PP * CP) and expert_data_parallel_size = world_size / (PP * EP * ETP). It should mention TP should stay within a single NVLink domain, SP requires tensor_model_parallel_size > 1, CP requires seq_length % (2 * context_parallel_size) == 0, sequence length 8K-32K suggests CP=2, and 32K+ suggests CP=4-8 or a2a+p2p for large CP.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-parallelism-strategies skill before answering.", + "Compute minimum GPUs with PP * max(TP * CP, EP * ETP).", + "Contrast the correct value against the wrong full product.", + "List dense DP and expert DP formulas.", + "Mention TP topology, SP, and CP sequence-length divisibility rules." + ] + } +] diff --git a/skills/nemo-mbridge-perf-sequence-packing/evals/evals.json b/skills/nemo-mbridge-perf-sequence-packing/evals/evals.json index fe51488c70..cc926fa72d 100644 --- a/skills/nemo-mbridge-perf-sequence-packing/evals/evals.json +++ b/skills/nemo-mbridge-perf-sequence-packing/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "sequence-packing-positive-sft-smoke", + "question": "Use the nemo-mbridge-perf-sequence-packing skill. Compare offline packed SFT and VLM in-batch packing in Megatron Bridge, including the exact micro-batch rules, PackedSequenceSpecs fields, CP padding formula, CUDA-graphs metadata requirement, and finetuning CP settings.", + "expected_skill": "nemo-mbridge-perf-sequence-packing", + "expected_script": null, + "ground_truth": "The answer should use the sequence packing skill. It should say offline packed SFT uses PackedSequenceSpecs with packed_sequence_size, optional pad_seq_to_mult, and usually train.micro_batch_size=1, while VLM in-batch packing uses cfg.dataset.pack_sequences_in_batch=True and requires train.micro_batch_size>1. It should state when CP is enabled, packed lengths must respect 2 * context_parallel_size, set pad_seq_to_mult = cfg.model.context_parallel_size * 2, and if sequence_parallel is also enabled use lcm(2*CP, CP*TP). It should mention CUDA graphs on the packed path need pad_cu_seqlens=True and that this also requires a metadata JSON file plus pad_to_max_length=True. It should mention finetuning with CP requires calculate_per_token_loss=True and ddp.average_in_collective=False, packed THD batches expect micro-batch size 1 for context-parallel slicing, and Qwen3-Next, GLM-4.5, Qwen3.5-VL or MTP have explicit opt-outs/incompatibilities.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-sequence-packing skill before answering.", + "Identify packed sequences and long-context training as the task.", + "Distinguish offline packed SFT from VLM in-batch packing with opposite micro-batch rules.", + "List PackedSequenceSpecs and pack_sequences_in_batch config surfaces.", + "State CP padding and lcm formulas.", + "Mention CUDA graph metadata and finetuning CP requirements." + ] + } +] diff --git a/skills/nemo-mbridge-perf-tp-dp-comm-overlap/evals/evals.json b/skills/nemo-mbridge-perf-tp-dp-comm-overlap/evals/evals.json index fe51488c70..7e56c1903b 100644 --- a/skills/nemo-mbridge-perf-tp-dp-comm-overlap/evals/evals.json +++ b/skills/nemo-mbridge-perf-tp-dp-comm-overlap/evals/evals.json @@ -1 +1,18 @@ -[] +[ + { + "id": "tp-dp-comm-overlap-positive-smoke", + "question": "Use the nemo-mbridge-perf-tp-dp-comm-overlap skill. For Megatron Bridge with TP=4, sequence_parallel=True, PP=4, and VPP=2, what exact TP/DP/PP communication overlap settings should I enable and how should I verify Bridge wired them correctly?", + "expected_skill": "nemo-mbridge-perf-tp-dp-comm-overlap", + "expected_script": null, + "ground_truth": "The answer should use the TP/DP/PP communication overlap skill. It should show CommOverlapConfig(tp_comm_overlap=True), require tensor_model_parallel_size > 1 and sequence_parallel=True for TP overlap, and set DDP overlap_grad_reduce=True plus overlap_param_gather=True with use_distributed_optimizer. It should explain that with PP > 1 and VPP > 1 Bridge selects overlap_p2p_comm=True and batch_p2p_comm=False, and recommend verifying tests or logs such as tests/unit_tests/training/test_comm_overlap.py.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-tp-dp-comm-overlap skill before answering.", + "Identify TP, DP, and PP communication overlap as the target feature.", + "List CommOverlapConfig(tp_comm_overlap=True).", + "Mention sequence_parallel=True as a TP overlap requirement.", + "List overlap_grad_reduce and overlap_param_gather for DP overlap.", + "Explain PP overlap selection for PP > 1 and VPP > 1.", + "Include a concrete verification path." + ] + } +] diff --git a/skills/nemo-mbridge-recipe-recommender/SKILL.md b/skills/nemo-mbridge-recipe-recommender/SKILL.md index 0aa95f32b4..7a11068c4e 100644 --- a/skills/nemo-mbridge-recipe-recommender/SKILL.md +++ b/skills/nemo-mbridge-recipe-recommender/SKILL.md @@ -53,10 +53,7 @@ python scripts/performance/run_script.py \ --data mock ``` -> **Perf recipes are NOT fully validated for correctness.** Most conversations -> and testing were on mock data. They are designed for **upper-bound throughput -> measurement**, not production training. Always validate loss curves and -> convergence independently. +See the Performance Recipe Index for important caveats before using these for anything beyond throughput benchmarking. --- diff --git a/skills/nemo-mbridge-recipe-recommender/evals/evals.json b/skills/nemo-mbridge-recipe-recommender/evals/evals.json index fe51488c70..a4b22a7a8b 100644 --- a/skills/nemo-mbridge-recipe-recommender/evals/evals.json +++ b/skills/nemo-mbridge-recipe-recommender/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "recipe-recommender-positive-sft-peft-smoke", + "question": "Use the nemo-mbridge-recipe-recommender skill. Recommend recipes for these exact Megatron Bridge cases: Qwen3 30B-A3B SFT on 8 GPUs, Qwen3 235B-A22B PEFT on 16 GPUs, Llama3 8B 128K pretrain, and first-time Bridge tryout. Include the entry point and key adjustment rules.", + "expected_skill": "nemo-mbridge-recipe-recommender", + "expected_script": null, + "ground_truth": "The answer should use the recipe recommender skill. It should recommend qwen3_30b_a3b_sft_config for Qwen3 30B-A3B SFT on 8 GPUs, qwen3_235b_a22b_peft_config for Qwen3 235B-A22B PEFT on 16 GPUs, llama3_8b_128k_pretrain_config for Llama3 8B 128K pretrain, and llama3_8b_sft_config with mock data as the first-time Bridge tryout. It should name scripts/training/run_recipe.py with uv run python -m torch.distributed.run for library recipes, use llm-finetune for SFT and llm-pretrain-mock for pretrain/mock tryout, and warn perf recipes under scripts/performance are for upper-bound mock-data throughput. It should include adjustment rules: TP must divide num_key_value_heads, TP should stay within a node unless using NVL72-style interconnect, SP should be true whenever TP>1, CP needs cp_comm_type and long-context variants/overrides, DP is implicit from the product of explicit parallelisms, and micro_batch_size should be reduced first on OOM.", + "expected_behavior": [ + "Read the nemo-mbridge-recipe-recommender skill before answering.", + "Identify the task as recipe selection or customization.", + "Recommend the exact Qwen3, Llama3, and first-time recipes requested.", + "Name scripts/training/run_recipe.py and the relevant mock/finetune datasets.", + "Include recipe resizing rules for TP, SP, CP, DP, and micro batch size.", + "Distinguish library recipes from performance throughput recipes." + ] + } +] diff --git a/skills/nemo-mbridge-resiliency/evals/evals.json b/skills/nemo-mbridge-resiliency/evals/evals.json index fe51488c70..ec3d761eb6 100644 --- a/skills/nemo-mbridge-resiliency/evals/evals.json +++ b/skills/nemo-mbridge-resiliency/evals/evals.json @@ -1 +1,17 @@ -[] +[ + { + "id": "resiliency-positive-preemption-smoke", + "question": "Use the nemo-mbridge-resiliency skill. How do I enable the recommended Slurm fault-tolerance path in Megatron Bridge? Include the FaultTolerancePlugin settings, restart counts, heartbeat timeouts, and when ft_launcher is required.", + "expected_skill": "nemo-mbridge-resiliency", + "expected_script": null, + "ground_truth": "The answer should use the resiliency skill and focus on Slurm fault tolerance. It should recommend the NeMo Run FaultTolerancePlugin path with enable_ft_package=True, calc_ft_timeouts=True, num_in_job_restarts=3, num_job_retries_on_failure=2, initial_rank_heartbeat_timeout=1800, and rank_heartbeat_timeout=300. It should mention the direct FaultToleranceConfig plus ft_launcher path when not using the plugin, and warn not to use plain torchrun for that direct launcher path.", + "expected_behavior": [ + "Read the nemo-mbridge-resiliency skill before answering.", + "Identify the task as Slurm fault-tolerance configuration.", + "Recommend FaultTolerancePlugin as the preferred path.", + "List num_in_job_restarts and num_job_retries_on_failure.", + "List the initial and subsequent heartbeat timeout settings.", + "Mention ft_launcher for the direct config path." + ] + } +]