diff --git a/.github/workflows/fern-docs-ci.yml b/.github/workflows/fern-docs-ci.yml index 65a8b4a893..795d2cf006 100644 --- a/.github/workflows/fern-docs-ci.yml +++ b/.github/workflows/fern-docs-ci.yml @@ -27,6 +27,9 @@ on: permissions: contents: read +env: + FERN_NO_VERSION_REDIRECTION: "true" + jobs: check: runs-on: ubuntu-latest diff --git a/.github/workflows/fern-docs-preview-build.yml b/.github/workflows/fern-docs-preview-build.yml deleted file mode 100644 index b900041c4f..0000000000 --- a/.github/workflows/fern-docs-preview-build.yml +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Workflow 1 of 2 for Fern doc previews. -# -# Collects the docs/fern sources, referenced skills, and PR metadata from the -# (possibly untrusted) PR branch and uploads them as an artifact. No secrets are used here, so this is -# safe to run on fork PRs via the regular pull_request trigger. -# -# The companion workflow (fern-docs-preview-comment.yml) picks up the artifact, -# builds the preview with DOCS_FERN_TOKEN, and posts the PR comment. - -name: "Preview Fern Docs: Build" - -on: - pull_request: - paths: - - 'docs/**' - - 'skills/**' - - '.github/workflows/fern-docs-preview-build.yml' - -permissions: - contents: read - -jobs: - collect: - runs-on: ubuntu-latest - steps: - - name: Checkout PR - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - fetch-depth: 0 - - - name: Save PR metadata - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_REF: ${{ github.head_ref }} - BASE_REF: ${{ github.base_ref }} - run: | - mkdir -p preview-metadata - echo "$PR_NUMBER" > preview-metadata/pr_number - echo "$HEAD_REF" > preview-metadata/head_ref - git diff --name-only "origin/${BASE_REF}...HEAD" -- '*.md' '*.mdx' > preview-metadata/changed_mdx_files 2>/dev/null || true - - - name: Upload fern sources, skills, and metadata - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: fern-preview - path: | - docs/fern/ - skills/ - preview-metadata/ - retention-days: 1 diff --git a/.github/workflows/fern-docs-preview-comment.yml b/.github/workflows/fern-docs-preview-comment.yml deleted file mode 100644 index d7bc2dd5fe..0000000000 --- a/.github/workflows/fern-docs-preview-comment.yml +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Workflow 2 of 2 for Fern doc previews. -# -# Triggered by workflow_run after "Preview Fern Docs: Build" completes. -# Downloads the docs/fern plus skills artifact, builds a preview with -# DOCS_FERN_TOKEN, and posts a stable :herb: comment on the PR. This workflow never checks out the -# PR branch directly, keeping secrets isolated from untrusted code. -# -# Required configuration: -# - Organization secret: DOCS_FERN_TOKEN (from `fern token` for the nvidia Fern org) - -name: "Preview Fern Docs: Comment" - -on: - workflow_run: - workflows: ["Preview Fern Docs: Build"] - types: [completed] - -permissions: - pull-requests: write - actions: read - -jobs: - preview: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Download fern sources and metadata - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 - with: - name: fern-preview - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Read PR metadata - id: metadata - run: | - echo "pr_number=$(cat preview-metadata/pr_number)" >> "$GITHUB_OUTPUT" - echo "head_ref=$(cat preview-metadata/head_ref)" >> "$GITHUB_OUTPUT" - - - name: Setup Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: '20' - - - name: Install Fern CLI - run: npm install -g fern-api@$(jq -r .version docs/fern/fern.config.json) - - - name: Generate preview URL - id: generate-docs - env: - FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} - HEAD_REF: ${{ steps.metadata.outputs.head_ref }} - working-directory: ./docs/fern - run: | - OUTPUT=$(fern generate --docs --preview --id "$HEAD_REF" 2>&1) - echo "$OUTPUT" - URL=$(echo "$OUTPUT" | grep -oP 'Published docs to \K.*(?= \()') - if [ -z "$URL" ]; then - echo "::error::Failed to generate preview URL. See fern output above." - exit 1 - fi - echo "preview_url=$URL" >> "$GITHUB_OUTPUT" - - - name: Build page links for changed MDX files - id: page-links - env: - FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} - PREVIEW_URL: ${{ steps.generate-docs.outputs.preview_url }} - run: | - CHANGED_FILES="" - if [ -f preview-metadata/changed_mdx_files ]; then - CHANGED_FILES=$(cat preview-metadata/changed_mdx_files) - fi - - if [ -z "$CHANGED_FILES" ] || [ -z "$PREVIEW_URL" ]; then - echo "page_links=" >> "$GITHUB_OUTPUT"; exit 0 - fi - - BASE_URL=$(echo "$PREVIEW_URL" | grep -oP 'https?://[^/]+') - FILES_PARAM=$(echo "$CHANGED_FILES" | tr '\n' ',' | sed 's/,$//' \ - | python3 -c "import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=',/'))") - RESPONSE=$(curl -sf -H "FERN_TOKEN: $FERN_TOKEN" "${PREVIEW_URL}/api/fern-docs/get-slug-for-file?files=${FILES_PARAM}" 2>/dev/null) || { - echo "page_links=" >> "$GITHUB_OUTPUT"; exit 0 - } - - PAGE_LINKS=$(echo "$RESPONSE" | jq -r --arg url "$BASE_URL" \ - '.mappings[] | select(.slug != null) | "- [\(.slug)](\($url)/\(.slug))"') - - if [ -n "$PAGE_LINKS" ]; then - { echo "page_links<> "$GITHUB_OUTPUT" - else - echo "page_links=" >> "$GITHUB_OUTPUT" - fi - - - name: Post or update PR comment - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ steps.metadata.outputs.pr_number }} - PREVIEW_URL: ${{ steps.generate-docs.outputs.preview_url }} - PAGE_LINKS: ${{ steps.page-links.outputs.page_links }} - run: | - # Build comment body - BODY=":herb: **Preview your docs:** <${PREVIEW_URL}>" - if [ -n "${PAGE_LINKS}" ]; then - BODY="${BODY} - - Here are the markdown pages you've updated: - ${PAGE_LINKS}" - fi - - # Hidden marker for upsert - MARKER="" - BODY="${BODY} - - ${MARKER}" - - # Find existing comment with marker - COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \ - --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | tr -d '\r' | head -1) - - if [ -n "$COMMENT_ID" ]; then - gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ - -X PATCH -f body="$BODY" - else - gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \ - -f body="$BODY" - fi diff --git a/.github/workflows/fern-docs-preview.yml b/.github/workflows/fern-docs-preview.yml new file mode 100644 index 0000000000..e28778e999 --- /dev/null +++ b/.github/workflows/fern-docs-preview.yml @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Preview Fern Docs + +on: + push: + branches: + - "pull-request/[0-9]+" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FERN_NO_VERSION_REDIRECTION: "true" + +jobs: + changed-files: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + docs: ${{ steps.changed-files.outputs.docs_any_changed }} + steps: + - name: Checkout approved PR mirror + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Get PR info + id: get-pr-info + uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main + - name: Get changed files + id: changed-files + uses: step-security/changed-files@2e07db73e5ccdb319b9a6c7766bd46d39d304bad # v47.0.5 + with: + base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + sha: ${{ github.sha }} + files_yaml: | + docs: + - docs/** + - skills/** + - .github/workflows/fern-docs-ci.yml + - .github/workflows/fern-docs-preview.yml + - name: Report docs preview gate + env: + DOCS_CHANGED: ${{ steps.changed-files.outputs.docs_any_changed }} + run: | + printf 'Docs or preview workflow changed: %s\n' "$DOCS_CHANGED" | tee -a "$GITHUB_STEP_SUMMARY" + + preview: + needs: changed-files + # An approved upstream mirror is the trust boundary, even for a fork PR. + # The original PR head repository does not change when its commit is mirrored. + if: >- + github.repository == 'NVIDIA-NeMo/Megatron-Bridge' && + needs.changed-files.outputs.docs == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + steps: + - name: Verify current PR against mirror push + id: pr + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + [[ "$GITHUB_REF" =~ ^refs/heads/pull-request/([1-9][0-9]*)$ ]] + PR_NUMBER=${BASH_REMATCH[1]} + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" | jq -e \ + --arg repo "$GITHUB_REPOSITORY" --arg sha "$GITHUB_SHA" --argjson number "$PR_NUMBER" \ + '.number == $number and .state == "open" and .head.sha == $sha and + .base.repo.full_name == $repo' + echo "number=$PR_NUMBER" >> "$GITHUB_OUTPUT" + + - name: Checkout trusted configuration + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: main + path: trusted-source + persist-credentials: false + + - name: Checkout approved PR content + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + path: pr-source + fetch-depth: 0 + persist-credentials: false + + - name: Prepare preview docs + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/fern-preview" + CONTENT='.*\.(md|mdx|png|jpg|jpeg|gif|svg|webp|ico|pdf)' + for ROOT in docs/fern skills; do + test -d "pr-source/$ROOT" + test -z "$(find "pr-source/$ROOT" -type l -print -quit)" + mkdir -p "$RUNNER_TEMP/fern-preview/$(dirname "$ROOT")" + cp -R "trusted-source/$ROOT" "$RUNNER_TEMP/fern-preview/$ROOT" + # Keep newer main pages as fallbacks, but honor explicit PR deletions. + BASE=$(git -C pr-source merge-base HEAD refs/remotes/origin/main) + git -C pr-source diff --no-renames --diff-filter=D --name-only -z \ + "$BASE" HEAD -- "$ROOT/" > "$RUNNER_TEMP/fern-deleted-paths" + while IFS= read -r -d '' path; do + case "$path" in + */.*|*/components/*) continue ;; + esac + case "$path" in + *.md|*.mdx|*.png|*.jpg|*.jpeg|*.gif|*.svg|*.webp|*.ico|*.pdf) + rm -f -- "$RUNNER_TEMP/fern-preview/$path" ;; + esac + done < "$RUNNER_TEMP/fern-deleted-paths" + # Navigation, components and tooling stay on the trusted branch. + (cd pr-source && find "$ROOT" -regextype posix-extended -type f \ + -regex "$CONTENT" ! -path '*/.*' ! -path '*/components/*' \ + -exec cp --parents -- '{}' "$RUNNER_TEMP/fern-preview/" \;) + done + printf '%s\n' '{"organization":"nvidia","version":"5.40.0"}' \ + > "$RUNNER_TEMP/fern-preview/docs/fern/fern.config.json" + + - name: Setup Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: '20' + + - name: Install fixed Fern CLI + working-directory: ${{ runner.temp }} + run: npm install --prefix "$RUNNER_TEMP/fern-cli" --ignore-scripts --no-audit --no-fund fern-api@5.40.0 + + - name: Generate preview + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + working-directory: ${{ runner.temp }}/fern-preview/docs/fern + run: | + set -euo pipefail + "$RUNNER_TEMP/fern-cli/node_modules/.bin/fern" generate --docs --preview \ + --id "pr-${PR_NUMBER}-${GITHUB_SHA:0:12}" 2>&1 | tee "$RUNNER_TEMP/fern-preview.log" + + - name: Read preview URL without credentials + id: preview + env: + PR_NUMBER: ${{ steps.pr.outputs.number }} + run: | + set -euo pipefail + URL=$(sed -n 's/^.*Published docs to \(https:\/\/[^ ]*\) (.*/\1/p' "$RUNNER_TEMP/fern-preview.log") + HOST="nvidia-preview-pr-${PR_NUMBER}-${GITHUB_SHA:0:12}.docs.buildwithfern.com" + [[ "$URL" == "https://$HOST" || "$URL" == "https://$HOST/"* ]] + PATH_PART=${URL#"https://$HOST"} + [[ "$PATH_PART" =~ ^(/[a-zA-Z0-9/_-]*)?$ ]] + echo "url=$URL" >> "$GITHUB_OUTPUT" + + - name: Post or update preview comment + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + PREVIEW_URL: ${{ steps.preview.outputs.url }} + run: | + set -euo pipefail + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" | jq -e \ + --arg sha "$GITHUB_SHA" '.state == "open" and .head.sha == $sha' + BODY=$(printf ':herb: **Preview your docs:** <%s>\n\n' "$PREVIEW_URL") + COMMENT_ID=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.user.login == "github-actions[bot]" and (.body | contains(""))) | .id' | head -1) + if [ -n "$COMMENT_ID" ]; then + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID" -X PATCH -f body="$BODY" + else + gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" -f body="$BODY" + fi + + clear-preview: + needs: changed-files + if: >- + github.repository == 'NVIDIA-NeMo/Megatron-Bridge' && + needs.changed-files.outputs.docs == 'false' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Remove stale preview comments after docs changes are reverted + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + [[ "$GITHUB_REF" =~ ^refs/heads/pull-request/([1-9][0-9]*)$ ]] + PR_NUMBER=${BASH_REMATCH[1]} + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" | jq -e \ + --arg repo "$GITHUB_REPOSITORY" --arg sha "$GITHUB_SHA" --argjson number "$PR_NUMBER" \ + '.number == $number and .state == "open" and .head.sha == $sha and + .base.repo.full_name == $repo' + gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.user.login == "github-actions[bot]" and (.body | contains(""))) | .id' \ + > "$RUNNER_TEMP/fern-preview-comments" + while IFS= read -r comment; do + [[ "$comment" =~ ^[1-9][0-9]*$ ]] + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$comment" -X DELETE + done < "$RUNNER_TEMP/fern-preview-comments" diff --git a/.github/workflows/publish-fern-docs.yml b/.github/workflows/publish-fern-docs.yml index ebdf3603e2..266375c7d2 100644 --- a/.github/workflows/publish-fern-docs.yml +++ b/.github/workflows/publish-fern-docs.yml @@ -36,6 +36,9 @@ concurrency: group: fern-publish cancel-in-progress: true +env: + FERN_NO_VERSION_REDIRECTION: "true" + jobs: publish: runs-on: ubuntu-latest diff --git a/docs/conf.py b/docs/conf.py index e4e18a6377..e94d16b7ee 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,6 +59,7 @@ ".DS_Store", "skills/linting-and-formatting/SKILL.md", "skills/parity-testing/SKILL.md", + "skills/pr-review/SKILL.md", "skills/**/BENCHMARK.md", "skills/**/skill-card.md", ] diff --git a/docs/fern/versions/nightly/pages/models/README.mdx b/docs/fern/versions/nightly/pages/models/README.mdx index 04ede80d6f..20c45c5640 100644 --- a/docs/fern/versions/nightly/pages/models/README.mdx +++ b/docs/fern/versions/nightly/pages/models/README.mdx @@ -1,4 +1,4 @@ - +{/* Generated by scripts/docs/generate_model_verification_catalog.py. Do not edit. */} # Supported Models @@ -128,11 +128,13 @@ Choose a model to inspect its recorded import/export, training, and precision co ✓ Verified - NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + NVIDIA-Nemotron-3-Super-120B-A12B-BF16 {/* pragma: allowlist secret */ + } ◐ Partial - NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 + NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 {/* pragma: allowlist secret */ + } ◐ Partial diff --git a/docs/fern/versions/nightly/pages/models/bagel/bagel.mdx b/docs/fern/versions/nightly/pages/models/bagel/bagel.mdx index 71b7cf99a6..9bf5fa05a6 100644 --- a/docs/fern/versions/nightly/pages/models/bagel/bagel.mdx +++ b/docs/fern/versions/nightly/pages/models/bagel/bagel.mdx @@ -12,7 +12,7 @@ for dependency setup, checkpoint initialization, training, and direct inference. See the [BAGEL data tutorial](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/tutorials/data/bagel/README.md) for dataset preparation. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -131,8 +131,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

BAGEL native-checkpoint initialization currently runs as part of GPU model construction and is not registered with the maintained public CPU conversion launcher. -

+

BAGEL native-checkpoint initialization currently runs as part of GPU model construction and is not registered with the maintained public CPU conversion launcher.

@@ -157,8 +156,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The public GPU importer consumed the pinned official EMA state, reported 1,223 source tensors consumed and 943 target tensors verified, and wrote a standalone torch_dist checkpoint. A clean one-GPU reload completed without missing or unexpected checkpoint keys; all 56 MoT language-layer input-norm tensors used as regression sentinels exactly matched the official EMA values. -

+

The public GPU importer consumed the pinned official EMA state, reported 1,223 source tensors consumed and 943 target tensors verified, and wrote a standalone torch_dist checkpoint. A clean one-GPU reload completed without missing or unexpected checkpoint keys; all 56 MoT language-layer input-norm tensors used as regression sentinels exactly matched the official EMA values.

@@ -177,8 +175,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Exporting a BAGEL Megatron checkpoint to a complete reloadable Hugging Face checkpoint is not implemented by the public CPU converter. -

+

Exporting a BAGEL Megatron checkpoint to a complete reloadable Hugging Face checkpoint is not implemented by the public CPU converter.

@@ -197,8 +194,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Exporting a BAGEL Megatron checkpoint to a complete reloadable Hugging Face checkpoint is not implemented by the public GPU converter. -

+

Exporting a BAGEL Megatron checkpoint to a complete reloadable Hugging Face checkpoint is not implemented by the public GPU converter.

@@ -256,8 +252,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The clean public-data run completes 10 BF16 optimizer steps on 8 H100s at TP1/PP1/CP1/DP8 and GBS/MBS 8/1. Every step has finite CE, MSE, and total loss with no skipped or NaN iterations. The final step reports CE 0.9156674, MSE 0.3265290, and total loss 1.242196. The final-10 averages include first-step compilation and are 16,383.960 ms and 195.540 model TFLOP/s/GPU; 2,250.005 token slots/GPU/s uses the fixed 36,864 maximum packed sequence slots rather than variable logical tokens. The post-setup configuration and iteration-10 model, optimizer, RNG, and per-DP dataloader state are persisted. A separate same-topology public-launcher load restored step 10 and exited without executing an additional optimizer step. -

+

The clean public-data run completes 10 BF16 optimizer steps on 8 H100s at TP1/PP1/CP1/DP8 and GBS/MBS 8/1. Every step has finite CE, MSE, and total loss with no skipped or NaN iterations. The final step reports CE 0.9156674, MSE 0.3265290, and total loss 1.242196. The final-10 averages include first-step compilation and are 16,383.960 ms and 195.540 model TFLOP/s/GPU; 2,250.005 token slots/GPU/s uses the fixed 36,864 maximum packed sequence slots rather than variable logical tokens. The post-setup configuration and iteration-10 model, optimizer, RNG, and per-DP dataloader state are persisted. A separate same-topology public-launcher load restored step 10 and exited without executing an additional optimizer step.

@@ -301,8 +296,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The BAGEL fine-tuning recipe must complete a bounded pinned-data run with finite losses, all required metrics, and a reloadable checkpoint. -

+

The BAGEL fine-tuning recipe must complete a bounded pinned-data run with finite losses, all required metrics, and a reloadable checkpoint.

@@ -346,8 +340,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The current BAGEL integration does not expose a maintained packed long-context fine-tuning recipe with context parallelism. -

+

The current BAGEL integration does not expose a maintained packed long-context fine-tuning recipe with context parallelism.

@@ -391,8 +384,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

BAGEL LoRA and other parameter-efficient fine-tuning workflows are not implemented by the current integration. -

+

BAGEL LoRA and other parameter-efficient fine-tuning workflows are not implemented by the current integration.

@@ -450,11 +442,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 32-GPU TP1/PP1/CP1/DP32 real-data run completes exactly 30 BF16 steps at GBS/MBS 32/1 with one microbatch, block-23 language-model recompute, full vision recompute, Megatron FSDP, finite CE/MSE/total losses, and no skipped or NaN iterations. Steps 21-30 average 7,697.160 ms, 234.268 model TFLOP/s/GPU, 23.687% MFU, and 4,789.299 token slots/GPU/s using 36,864 maximum packed sequence slots and GBS 32 across 32 GPUs. The separately measured logical throughput is 4,428.810 tokens/GPU/s from 1,090,856.3 logical tokens per step. Logical tokens include text, ViT, VAE-latent, and special tokens and exclude physical padding. The variable packed sequence length averages 34,070.15 tokens/GPU over steps 6-30. Peak allocated/reserved memory is 58.594/62.919 GiB. The same steps 6-30 workload reports standalone raw metrics of 10,167.941 ms, 176.936 model TFLOP/s/GPU, and 17.890% MFU under official BAGEL. Do not compute a relative speedup: that run uses full language-model recompute and performs an EMA update, while the Bridge run uses block-23 recompute without EMA. -

+

The 32-GPU TP1/PP1/CP1/DP32 real-data run completes exactly 30 BF16 steps at GBS/MBS 32/1 with one microbatch, block-23 language-model recompute, full vision recompute, Megatron FSDP, finite CE/MSE/total losses, and no skipped or NaN iterations. Steps 21-30 average 7,697.160 ms, 234.268 model TFLOP/s/GPU, 23.687% MFU, and 4,789.299 token slots/GPU/s using 36,864 maximum packed sequence slots and GBS 32 across 32 GPUs. The separately measured logical throughput is 4,428.810 tokens/GPU/s from 1,090,856.3 logical tokens per step. Logical tokens include text, ViT, VAE-latent, and special tokens and exclude physical padding. The variable packed sequence length averages 34,070.15 tokens/GPU over steps 6-30. Peak allocated/reserved memory is 58.594/62.919 GiB. The same steps 6-30 workload reports standalone raw metrics of 10,167.941 ms, 176.936 model TFLOP/s/GPU, and 17.890% MFU under official BAGEL. Do not compute a relative speedup: that run uses full language-model recompute and performs an EMA update, while the Bridge run uses block-23 recompute without EMA.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v3.mdx b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v3.mdx index 7c958d1818..7bb2f871ed 100644 --- a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v3.mdx +++ b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v3.mdx @@ -4,7 +4,7 @@ DeepSeek V3 models are supported via the Bridge system with auto-detected configuration and weight mapping. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -139,8 +139,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A CPU import of the pinned blockwise-FP8 Hugging Face checkpoint must dequantize every mapped weight correctly, persist a reloadable Megatron checkpoint, and pass a complete tensor audit. This workflow is deferred by this card. -

+

A CPU import of the pinned blockwise-FP8 Hugging Face checkpoint must dequantize every mapped weight correctly, persist a reloadable Megatron checkpoint, and pass a complete tensor audit. This workflow is deferred by this card.

@@ -165,8 +164,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 32-GPU import completed all 4,583 mapping tasks, blockwise-dequantized the pinned FP8 source into BF16, and persisted a reloadable 32-shard distributed Megatron checkpoint. A strict paired export subsequently matched all 46,183 model tensors and 684,489,845,504 values exactly. -

+

The 32-GPU import completed all 4,583 mapping tasks, blockwise-dequantized the pinned FP8 source into BF16, and persisted a reloadable 32-shard distributed Megatron checkpoint. A strict paired export subsequently matched all 46,183 model tensors and 684,489,845,504 values exactly.

@@ -191,8 +189,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 32-process distributed CPU export exits successfully and writes all 163 source-shaped shards. The index contains exactly 46,183 BF16 model tensors; an exhaustive audit matches all 684,489,845,504 values against the pinned source after blockwise dequantization of all 45,808 FP8 tensors and 59 other declared dtype normalizations. Transformers strictly reloads the output as DeepseekV3ForCausalLM with all 967 state tensors and no loading discrepancies. -

+

The 32-process distributed CPU export exits successfully and writes all 163 source-shaped shards. The index contains exactly 46,183 BF16 model tensors; an exhaustive audit matches all 684,489,845,504 values against the pinned source after blockwise dequantization of all 45,808 FP8 tensors and 59 other declared dtype normalizations. Transformers strictly reloads the output as DeepseekV3ForCausalLM with all 967 state tensors and no loading discrepancies.

@@ -217,8 +214,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exited successfully and wrote all 163 source-shaped shards. The index contained exactly 46,183 BF16 model tensors; an exhaustive 32-rank audit matched all 684,489,845,504 values against the pinned source after blockwise dequantization of all 45,808 FP8 tensors. The exported configuration instantiated natively as DeepseekV3ForCausalLM. -

+

The command exited successfully and wrote all 163 source-shaped shards. The index contained exactly 46,183 BF16 model tensors; an exhaustive 32-rank audit matched all 684,489,845,504 values against the pinned source after blockwise dequantization of all 45,808 FP8 tensors. The exported configuration instantiated natively as DeepseekV3ForCausalLM.

@@ -262,8 +258,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A bounded public-data DeepSeek V3 pretraining run must complete with finite loss, zero skipped or NaN iterations, all five metrics, a saved post-setup configuration, and a reloadable final checkpoint. Functional training is deferred by this card. -

+

A bounded public-data DeepSeek V3 pretraining run must complete with finite loss, zero skipped or NaN iterations, all five metrics, a saved post-setup configuration, and a reloadable final checkpoint. Functional training is deferred by this card.

@@ -307,8 +302,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A pinned-data full-SFT run must finish with finite loss, zero skipped or NaN iterations, all five metrics, a saved post-setup configuration, and a reloadable final checkpoint. Fine-tuning is deferred by this card. -

+

A pinned-data full-SFT run must finish with finite loss, zero skipped or NaN iterations, all five metrics, a saved post-setup configuration, and a reloadable final checkpoint. Fine-tuning is deferred by this card.

@@ -352,8 +346,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A dedicated packed long-context SFT run must complete with context parallelism, finite loss, zero skipped or NaN iterations, all five metrics, and a reloadable checkpoint. This workflow is deferred by this card. -

+

A dedicated packed long-context SFT run must complete with context parallelism, finite loss, zero skipped or NaN iterations, all five metrics, and a reloadable checkpoint. This workflow is deferred by this card.

@@ -397,8 +390,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A DeepSeek V3 PEFT recipe with an audited model-native attention adapter target set must complete with finite loss, zero skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint. This workflow is deferred by this card. -

+

A DeepSeek V3 PEFT recipe with an audited model-native attention adapter target set must complete with finite loss, zero skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint. This workflow is deferred by this card.

@@ -448,8 +440,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 1,024 H100 GPUs, the canonical 50-step mock-data BF16 recipe completed at TP2/PP8/VP4/CP1/EP64/ETP1 and GBS/MBS 16384/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 61,756.130 ms and 276.030 model TFLOP/s/GPU, passing gates of at most 62,000 ms and at least 275 TFLOP/s/GPU. Mock data and forced expert balancing make this benchmark-only evidence, not convergence evidence. -

+

On exactly 1,024 H100 GPUs, the canonical 50-step mock-data BF16 recipe completed at TP2/PP8/VP4/CP1/EP64/ETP1 and GBS/MBS 16384/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 61,756.130 ms and 276.030 model TFLOP/s/GPU, passing gates of at most 62,000 ms and at least 275 TFLOP/s/GPU. Mock data and forced expert balancing make this benchmark-only evidence, not convergence evidence.

@@ -499,8 +490,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 256 GB200 GPUs, the canonical 50-step mock-data MXFP8 recipe completed at TP1/PP4/VP4/CP1/EP64/ETP1 and GBS/MBS 4096/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 13,841.180 ms and 1,231.040 model TFLOP/s/GPU, passing gates of at most 14,000 ms and at least 1,200 TFLOP/s/GPU. Mock data, forced expert balancing, and reduced-precision optimizer moments make this benchmark-only evidence, not convergence evidence. -

+

On exactly 256 GB200 GPUs, the canonical 50-step mock-data MXFP8 recipe completed at TP1/PP4/VP4/CP1/EP64/ETP1 and GBS/MBS 4096/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 13,841.180 ms and 1,231.040 model TFLOP/s/GPU, passing gates of at most 14,000 ms and at least 1,200 TFLOP/s/GPU. Mock data, forced expert balancing, and reduced-precision optimizer moments make this benchmark-only evidence, not convergence evidence.

@@ -550,11 +540,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 256 GB300 GPUs, the canonical 50-step mock-data MXFP8 recipe completed at TP1/PP2/VP8/CP1/EP32/ETP1 and GBS/MBS 4096/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 10,667.300 ms and 1,597.310 model TFLOP/s/GPU, passing gates of at most 11,000 ms and at least 1,550 TFLOP/s/GPU. Mock data, forced expert balancing, and reduced-precision optimizer moments make this benchmark-only evidence, not convergence evidence. -

+

On exactly 256 GB300 GPUs, the canonical 50-step mock-data MXFP8 recipe completed at TP1/PP2/VP8/CP1/EP32/ETP1 and GBS/MBS 4096/1. All 50 keyed rows had finite loss and throughput with zero skipped or NaN iterations, and the resolved configuration was saved. Steps 41-50 averaged 10,667.300 ms and 1,597.310 model TFLOP/s/GPU, passing gates of at most 11,000 ms and at least 1,550 TFLOP/s/GPU. Mock data, forced expert balancing, and reduced-precision optimizer moments make this benchmark-only evidence, not convergence evidence.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx index 67a4a677e2..a395e3dc94 100644 --- a/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx +++ b/docs/fern/versions/nightly/pages/models/deepseek/deepseek-v4.mdx @@ -4,7 +4,7 @@ DeepSeek V4 models are supported via the Bridge system with auto-detected configuration and weight mapping. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -137,8 +137,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU import requires approximately 570 GB of CPU RAM for the full BF16 weight materialisation (285B parameters x 2 bytes). Verification requires a high-memory node with enough additional headroom for conversion workspace and strict checkpoint reload. -

+

CPU import requires approximately 570 GB of CPU RAM for the full BF16 weight materialisation (285B parameters x 2 bytes). Verification requires a high-memory node with enough additional headroom for conversion workspace and strict checkpoint reload.

@@ -163,8 +162,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully and creates a reloadable iter_0000000 Megatron checkpoint. -

+

The command exits successfully and creates a reloadable iter_0000000 Megatron checkpoint.

@@ -189,8 +187,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU export requires a high-memory node with enough headroom for the full BF16 weight set and tensor-merging workspace, followed by strict Hugging Face reload. -

+

CPU export requires a high-memory node with enough headroom for the full BF16 weight set and tensor-merging workspace, followed by strict Hugging Face reload.

@@ -215,8 +212,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Two GB200 nodes produced 46 complete safetensors shards containing all 35,020 expected non-scale source keys. BF16 export intentionally omitted 34,167 quantization-scale companions; 35,017 exported tensors are BF16 and the three I32 tid2eid routing tables exactly match their source values after the expected integer cast. Two independent Transformers processes each strictly reloaded all 1,500 model modules across four GPUs with no CPU or disk placement and produced the same bounded greedy token. --not-strict permits only the intentional scale omission; exact key, shard, dtype, routing-table value, and reload checks remain required correctness gates. -

+

Two GB200 nodes produced 46 complete safetensors shards containing all 35,020 expected non-scale source keys. BF16 export intentionally omitted 34,167 quantization-scale companions; 35,017 exported tensors are BF16 and the three I32 tid2eid routing tables exactly match their source values after the expected integer cast. Two independent Transformers processes each strictly reloaded all 1,500 model modules across four GPUs with no CPU or disk placement and produced the same bounded greedy token. --not-strict permits only the intentional scale omission; exact key, shard, dtype, routing-table value, and reload checks remain required correctness gates.

@@ -266,8 +262,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 64 GB200 (16 nodes x 4 GPUs), TP1/PP4/VPP4/EP16/CP1 uses dense DP16, expert DP1, and GBS/MBS 256/1. The command loads the imported model weights while starting optimizer and RNG state fresh. It uses HybridEP natural routing, grouped GEMM, TE fused cross entropy, selective recompute over mhc and mla_up_proj, and MXFP8 parameter gather and gradient-buffer reuse without activation offload. Expert capacity, paged stash, CUDA graphs, and forced load balancing remain disabled. The run completed 100 finite LM/MTP-loss steps with no skipped or NaN iterations and wrote complete grouped-MXFP8 checkpoints containing model, optimizer, scheduler, data-order, and RNG state at steps 50 and 100. The item-specific Bridge revision pins the compatible Megatron-LM dev revision used by this workload without changing the card's default environment for other verification items. -

+

On 64 GB200 (16 nodes x 4 GPUs), TP1/PP4/VPP4/EP16/CP1 uses dense DP16, expert DP1, and GBS/MBS 256/1. The command loads the imported model weights while starting optimizer and RNG state fresh. It uses HybridEP natural routing, grouped GEMM, TE fused cross entropy, selective recompute over mhc and mla_up_proj, and MXFP8 parameter gather and gradient-buffer reuse without activation offload. Expert capacity, paged stash, CUDA graphs, and forced load balancing remain disabled. The run completed 100 finite LM/MTP-loss steps with no skipped or NaN iterations and wrote complete grouped-MXFP8 checkpoints containing model, optimizer, scheduler, data-order, and RNG state at steps 50 and 100. The item-specific Bridge revision pins the compatible Megatron-LM dev revision used by this workload without changing the card's default environment for other verification items.

@@ -325,8 +320,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 32 GB200 (8 nodes x 4 GPUs), TP1/PP4/EP8/CP1 with dense DP8 and expert DP1, GBS/MBS 128/1. OpenMathInstruct-2 thinking data uses offline packing at seq_length=1024 with fixed token and cumulative-boundary shapes. The GB200 recipe uses selective recompute over moe, mhc, mla_up_proj, and layernorm, attention activation offload, HybridEP dispatch, and DSA kernel fusion without enabling the optional DSA indexer loss. The run completed 100 steps with finite LM and MTP losses, no skipped or NaN iterations, and a complete model checkpoint at step 100. A fresh process reloaded that checkpoint and completed finite step 101. moe_grouped_gemm=True is a recipe default. dist.distributed_timeout_minutes must be 180 or above to allow offline data packing (approximately 63 minutes) before training begins. -

+

On 32 GB200 (8 nodes x 4 GPUs), TP1/PP4/EP8/CP1 with dense DP8 and expert DP1, GBS/MBS 128/1. OpenMathInstruct-2 thinking data uses offline packing at seq_length=1024 with fixed token and cumulative-boundary shapes. The GB200 recipe uses selective recompute over moe, mhc, mla_up_proj, and layernorm, attention activation offload, HybridEP dispatch, and DSA kernel fusion without enabling the optional DSA indexer loss. The run completed 100 steps with finite LM and MTP losses, no skipped or NaN iterations, and a complete model checkpoint at step 100. A fresh process reloaded that checkpoint and completed finite step 101. moe_grouped_gemm=True is a recipe default. dist.distributed_timeout_minutes must be 180 or above to allow offline data packing (approximately 63 minutes) before training begins.

@@ -376,8 +370,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CP=2 with contiguous partitioning and offline-packed OpenMathInstruct-2 at seq_length=1024 demonstrates sequence packing and context parallelism working together over 100 training steps with finite loss and no skipped or NaN iterations. -

+

CP=2 with contiguous partitioning and offline-packed OpenMathInstruct-2 at seq_length=1024 demonstrates sequence packing and context parallelism working together over 100 training steps with finite loss and no skipped or NaN iterations.

@@ -435,8 +428,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 32 GB200 (8 nodes x 4 GPUs), TP1/PP4/EP8/CP1 with dense DP8 and expert DP1, GBS/MBS 128/1. The recipe applies rank-32, alpha-32 LoRA with zero dropout to linear_q_down_proj, linear_q_up_proj, linear_kv_proj, linear_proj, linear_fc1, and linear_fc2. Routed experts use separate per-expert adapters; shared experts are also adapted. The run preserves the packed OpenMath thinking data, objective, natural routing, seed, and warmup/decay horizon used by the verified SFT item while freezing the base model and using PEFT's own 1e-4 peak learning rate. It disables activation recompute and offload because the reduced PEFT training-state footprint fits with 53.712 GiB peak allocated memory. Transformer Engine CUDA graphs cover moe_router and moe_preprocess, while HybridEP uses 32 flex-dispatcher SMs, 8 ranks per NVLink domain, 128-token combine chunks, a 72-GPU domain, and MNNVL. All 100 steps completed with finite LM/MTP losses and zero skipped or NaN iterations. LM/MTP losses were 0.9435847/0.2250465 at step 1 and 0.2914867/0.02638751 at step 100. The adapter-only checkpoint reloaded after the same base checkpoint in a fresh process and completed finite step 101 with LM loss 0.2827368 and MTP loss 0.02594211. -

+

On 32 GB200 (8 nodes x 4 GPUs), TP1/PP4/EP8/CP1 with dense DP8 and expert DP1, GBS/MBS 128/1. The recipe applies rank-32, alpha-32 LoRA with zero dropout to linear_q_down_proj, linear_q_up_proj, linear_kv_proj, linear_proj, linear_fc1, and linear_fc2. Routed experts use separate per-expert adapters; shared experts are also adapted. The run preserves the packed OpenMath thinking data, objective, natural routing, seed, and warmup/decay horizon used by the verified SFT item while freezing the base model and using PEFT's own 1e-4 peak learning rate. It disables activation recompute and offload because the reduced PEFT training-state footprint fits with 53.712 GiB peak allocated memory. Transformer Engine CUDA graphs cover moe_router and moe_preprocess, while HybridEP uses 32 flex-dispatcher SMs, 8 ranks per NVLink domain, 128-token combine chunks, a 72-GPU domain, and MNNVL. All 100 steps completed with finite LM/MTP losses and zero skipped or NaN iterations. LM/MTP losses were 0.9435847/0.2250465 at step 1 and 0.2914867/0.02638751 at step 100. The adapter-only checkpoint reloaded after the same base checkpoint in a fresh process and completed finite step 101 with LM loss 0.2827368 and MTP loss 0.02594211.

@@ -486,8 +478,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 128 GB200, PP1/EP64/CP1/TP1/DP2 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence. -

+

On 128 GB200, PP1/EP64/CP1/TP1/DP2 with GBS/MBS 2048/1, FP8-MX, HybridEP dispatch, and full-iteration CUDA graphs completes 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. This benchmark uses forced routing, a static expert-rank capacity, paged stash, and a different batch/objective contract, so its metrics must not be compared with natural-routing library pretraining as convergence evidence.

@@ -537,11 +528,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 128 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 2048/1, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.59616 to 3.160061; the final ten steps average 7840.320 ms, 759.280 TFLOP/s/GPU, and 8358.842 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 128 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 2048/1, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.59616 to 3.160061; the final ten steps average 7840.320 ms, 759.280 TFLOP/s/GPU, and 8358.842 tokens/s/GPU. The resolved configuration persists.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/exaone/exaone.mdx b/docs/fern/versions/nightly/pages/models/exaone/exaone.mdx index 821e14a983..c128d2bec7 100644 --- a/docs/fern/versions/nightly/pages/models/exaone/exaone.mdx +++ b/docs/fern/versions/nightly/pages/models/exaone/exaone.mdx @@ -17,7 +17,7 @@ their model-specific distributed round-trip workflows. K-EXAONE 2.0 also has a c [model verification card](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/model_verification_cards/k-exaone-2/card.yaml) that records the current verification status of individual workflows. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -129,8 +129,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No CPU Hugging Face-to-Megatron conversion result is claimed. Future verification must import the complete immutable-revision checkpoint, persist iter_0000000, reload it, and audit all mapped keys, shapes, dtypes, and values. -

+

No CPU Hugging Face-to-Megatron conversion result is claimed. Future verification must import the complete immutable-revision checkpoint, persist iter_0000000, reload it, and audit all mapped keys, shapes, dtypes, and values.

@@ -155,8 +154,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned 59,396-tensor BF16 checkpoint imported and persisted iter_0000000 on 48 H100 GPUs, and that artifact subsequently reloaded for a full distributed GPU re-export audit. The audit compared all 59,396 tensors and 1,498,715,010,176 payload bytes with zero missing, unexpected, duplicate, shape, dtype, or value mismatches. -

+

The pinned 59,396-tensor BF16 checkpoint imported and persisted iter_0000000 on 48 H100 GPUs, and that artifact subsequently reloaded for a full distributed GPU re-export audit. The audit compared all 59,396 tensors and 1,498,715,010,176 payload bytes with zero missing, unexpected, duplicate, shape, dtype, or value mismatches.

@@ -175,8 +173,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No CPU Megatron-to-Hugging-Face conversion result is claimed. Future verification depends on a complete CPU import and must export, strictly reload, and compare all lossless tensors against the pinned source revision. -

+

No CPU Megatron-to-Hugging-Face conversion result is claimed. Future verification depends on a complete CPU import and must export, strictly reload, and compare all lossless tensors against the pinned source revision.

@@ -201,8 +198,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A complete distributed export produced 59,396 indexed keys in 17 safetensors shards. Every exported tensor matches the pinned source bitwise across 1,498,715,008,512 payload bytes, and the corrected config/tokenizer reload successfully. The run exposed and fixed preservation of one physical repeated MTP layer across four speculative steps. The config fix is committed; this item remains unverified until the clean-commit export and exact audit rerun complete. -

+

A complete distributed export produced 59,396 indexed keys in 17 safetensors shards. Every exported tensor matches the pinned source bitwise across 1,498,715,008,512 payload bytes, and the corrected config/tokenizer reload successfully. The run exposed and fixed preservation of one physical repeated MTP layer across four speculative steps. The config fix is committed; this item remains unverified until the clean-commit export and exact audit rerun complete.

@@ -246,8 +242,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The exported 512-GPU H100 recipe must complete a bounded public-data pretraining run at TP4/PP16/EP32 with natural routing, finite loss, no skipped or NaN iterations, all five required metrics, and reloadable middle and final checkpoints. -

+

The exported 512-GPU H100 recipe must complete a bounded public-data pretraining run at TP4/PP16/EP32 with natural routing, finite loss, no skipped or NaN iterations, all five required metrics, and reloadable middle and final checkpoints.

@@ -291,8 +286,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The exported 512-GPU H100 full-SFT recipe must load the pinned checkpoint and complete a bounded immutable-dataset run at TP4/PP16/EP32 with finite loss, no skipped or NaN iterations, all four required metrics, and a reloadable final full-model checkpoint. -

+

The exported 512-GPU H100 full-SFT recipe must load the pinned checkpoint and complete a bounded immutable-dataset run at TP4/PP16/EP32 with finite loss, no skipped or NaN iterations, all four required metrics, and a reloadable final full-model checkpoint.

@@ -336,8 +330,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No long-context SFT result is claimed. Future verification must first resolve a supported long-context topology, then verify sequence packing and context parallelism together with finite loss, no skipped or NaN iterations, and all five required metrics. -

+

No long-context SFT result is claimed. Future verification must first resolve a supported long-context topology, then verify sequence packing and context parallelism together with finite loss, no skipped or NaN iterations, and all five required metrics.

@@ -381,11 +374,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The exported 128-GPU H100 PEFT recipe must load the pinned checkpoint, verify that LoRA targets the model-native attention projections, and complete a bounded immutable-dataset run at TP4/PP8/EP16 with finite loss, no skipped or NaN iterations, all five required metrics, and a reloadable adapter checkpoint. -

+

The exported 128-GPU H100 PEFT recipe must load the pinned checkpoint, verify that LoRA targets the model-native attention projections, and complete a bounded immutable-dataset run at TP4/PP8/EP16 with finite loss, no skipped or NaN iterations, all five required metrics, and a reloadable adapter checkpoint.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/gemma/gemma3-vl.mdx b/docs/fern/versions/nightly/pages/models/gemma/gemma3-vl.mdx index dcba759c16..8d62ab5254 100644 --- a/docs/fern/versions/nightly/pages/models/gemma/gemma3-vl.mdx +++ b/docs/fern/versions/nightly/pages/models/gemma/gemma3-vl.mdx @@ -6,7 +6,7 @@ Gemma 3 VL models support multimodal tasks including image captioning, visual qu Gemma family models are supported via the Bridge system with auto-detected configuration and weight mapping. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -124,8 +124,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Exact-revision BF16 CPU import loaded all 781 Bridge tensors and saved one reloadable DCP shard. An independent Gloo reload mapped all 883 source tensors with exact keys, shapes, dtypes, and torch.equal values, including all 437 legacy vision-tower keys. -

+

Exact-revision BF16 CPU import loaded all 781 Bridge tensors and saved one reloadable DCP shard. An independent Gloo reload mapped all 883 source tensors with exact keys, shapes, dtypes, and torch.equal values, including all 437 legacy vision-tower keys.

@@ -150,8 +149,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Exact-revision distributed BF16 import completed with TP2/PP2/EP1/ETP1. All 781 Bridge tensors loaded and all four ranks saved a reloadable checkpoint with the vision tower and multimodal projector on the owning pipeline stage. Independent TP2/PP2 and Gloo reloads retained FusedAttention and matched all 883 mapped tensors exactly. -

+

Exact-revision distributed BF16 import completed with TP2/PP2/EP1/ETP1. All 781 Bridge tensors loaded and all four ranks saved a reloadable checkpoint with the vision tower and multimodal projector on the owning pipeline stage. Independent TP2/PP2 and Gloo reloads retained FusedAttention and matched all 883 mapped tensors exactly.

@@ -176,8 +174,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The exact-revision CPU export exits successfully and writes all 883 tensors, 4,300,079,472 values, and 8,600,158,944 tensor-payload bytes. Keys, shapes, dtypes, and values are bitwise identical to the pinned BF16 source with no dtype conversions, including all 437 legacy vision_tower.vision_model.* tensors. Transformers strictly reloads the output natively as Gemma3ForConditionalGeneration with all 884 state entries and no loading discrepancies. -

+

The exact-revision CPU export exits successfully and writes all 883 tensors, 4,300,079,472 values, and 8,600,158,944 tensor-payload bytes. Keys, shapes, dtypes, and values are bitwise identical to the pinned BF16 source with no dtype conversions, including all 437 legacy vision_tower.vision_model.* tensors. Transformers strictly reloads the output natively as Gemma3ForConditionalGeneration with all 884 state entries and no loading discrepancies.

@@ -202,8 +199,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

TP2/PP2 distributed export completed without a missing PP owner. Source and export contained the same 883 keys with exact shapes, dtypes, and torch.equal values, including all 437 legacy vision_tower.vision_model.* tensors. Transformers exposes 884 logical state entries after reload because tied lm_head.weight aliases the single stored language_model.model.embed_tokens.weight tensor; no extra tensor is serialized. Native Gemma3ForConditionalGeneration strict reload reported no missing, unexpected, mismatched, or errored tensors. -

+

TP2/PP2 distributed export completed without a missing PP owner. Source and export contained the same 883 keys with exact shapes, dtypes, and torch.equal values, including all 437 legacy vision_tower.vision_model.* tensors. Transformers exposes 884 logical state entries after reload because tied lm_head.weight aliases the single stored language_model.model.embed_tokens.weight tensor; no extra tensor is serialized. Native Gemma3ForConditionalGeneration strict reload reported no missing, unexpected, mismatched, or errored tensors.

@@ -247,8 +243,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Megatron Bridge does not provide a public Gemma 3 VL 4B pretraining recipe. The available exact-model recipes cover SFT and PEFT, so a supported pretraining command and checkpoint contract cannot be stated. -

+

Megatron Bridge does not provide a public Gemma 3 VL 4B pretraining recipe. The available exact-model recipes cover SFT and PEFT, so a supported pretraining command and checkpoint contract cannot be stated.

@@ -293,13 +288,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gemma3_vl_4b_sft_config --mode sft --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-3-4b-it/sft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gemma3_vl_4b_sft_config --mode sft --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-3-4b-it/sft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Pending exact-revision CORD-v2 full-SFT verification with the public Gemma 3 VL 4B H100 recipe. The run must complete at least ten optimizer steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final full-model checkpoint. -

+

Pending exact-revision CORD-v2 full-SFT verification with the public Gemma 3 VL 4B H100 recipe. The run must complete at least ten optimizer steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final full-model checkpoint.

@@ -343,8 +337,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Gemma 3 VL image-bidirectional attention uses dense local/global additive biases with FusedAttention. The provider rejects context parallelism greater than one, and the public recipes disable in-batch THD packing, so there is no supported exact-model long-context SFT contract. -

+

Gemma 3 VL image-bidirectional attention uses dense local/global additive biases with FusedAttention. The provider rejects context parallelism greater than one, and the public recipes disable in-batch THD packing, so there is no supported exact-model long-context SFT contract.

@@ -389,16 +382,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gemma3_vl_4b_peft_config --mode lora --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-3-4b-it/peft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gemma3_vl_4b_peft_config --mode lora --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-3-4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-3-4b-it/peft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Pending exact-revision CORD-v2 LoRA verification with the public Gemma 3 VL 4B H100 recipe. The run must complete at least ten optimizer steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint. -

+

Pending exact-revision CORD-v2 LoRA verification with the public Gemma 3 VL 4B H100 recipe. The run must complete at least ten optimizer steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/gemma/gemma4-vl.mdx b/docs/fern/versions/nightly/pages/models/gemma/gemma4-vl.mdx index d25f5c2944..3da81fb969 100644 --- a/docs/fern/versions/nightly/pages/models/gemma/gemma4-vl.mdx +++ b/docs/fern/versions/nightly/pages/models/gemma/gemma4-vl.mdx @@ -14,7 +14,7 @@ For the full setup, conversion, inference, training, and LoRA merge / adapter export workflows, see [`examples/models/gemma/gemma4_vl/README.md`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/models/gemma/gemma4_vl/README.md). - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -132,8 +132,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command completed all 8,548 conversion mappings and saved iter_0000000. All 383 tensor entries in the distributed-checkpoint metadata were torch.bfloat16. The run configuration recorded bf16=true, fp16=false, and BF16 parameter and autocast dtypes. The artifact reloaded successfully for CPU export. -

+

The command completed all 8,548 conversion mappings and saved iter_0000000. All 383 tensor entries in the distributed-checkpoint metadata were torch.bfloat16. The run configuration recorded bf16=true, fp16=false, and BF16 parameter and autocast dtypes. The artifact reloaded successfully for CPU export.

@@ -158,8 +157,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command completed successfully at TP4/PP2/EP1. It loaded 8,548 conversion mappings from the immutable Hugging Face revision and saved the full multimodal checkpoint as iter_0000000. All 383 tensor entries in the distributed-checkpoint metadata were torch.bfloat16. The artifact then reloaded successfully for distributed export and direct-model inference. -

+

The command completed successfully at TP4/PP2/EP1. It loaded 8,548 conversion mappings from the immutable Hugging Face revision and saved the full multimodal checkpoint as iter_0000000. All 383 tensor entries in the distributed-checkpoint metadata were torch.bfloat16. The artifact then reloaded successfully for distributed export and direct-model inference.

@@ -184,8 +182,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command completed all 8,548 conversion mappings and wrote 11 BF16 shards. All 1,013 exported tensor entries matched the immutable source exactly in key, shape, dtype, and value. Native Gemma4ForConditionalGeneration reload reported model.dtype=torch.bfloat16 with no missing, unexpected, mismatched, or error keys. -

+

The command completed all 8,548 conversion mappings and wrote 11 BF16 shards. All 1,013 exported tensor entries matched the immutable source exactly in key, shape, dtype, and value. Native Gemma4ForConditionalGeneration reload reported model.dtype=torch.bfloat16 with no missing, unexpected, mismatched, or error keys.

@@ -210,8 +207,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed TP2/PP1/EP4 export completed all 2,788 conversion mappings. The exported config preserved sliding-attention rope_theta 10000 and full-attention rope_theta 1000000. All 1,013 exported tensor entries were BF16 and matched the immutable source exactly in key, shape, dtype, and value. Native Gemma4ForConditionalGeneration reload reported model.dtype=torch.bfloat16 with no missing, unexpected, mismatched, or error keys. -

+

Distributed TP2/PP1/EP4 export completed all 2,788 conversion mappings. The exported config preserved sliding-attention rope_theta 10000 and full-attention rope_theta 1000000. All 1,013 exported tensor entries were BF16 and matched the immutable source exactly in key, shape, dtype, and value. Native Gemma4ForConditionalGeneration reload reported model.dtype=torch.bfloat16 with no missing, unexpected, mismatched, or error keys.

@@ -255,8 +251,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Megatron Bridge does not publish a pretraining recipe for this Gemma 4 conditional-generation variant; the current model package provides SFT and PEFT recipes only. -

+

Megatron Bridge does not publish a pretraining recipe for this Gemma 4 conditional-generation variant; the current model package provides SFT and PEFT recipes only.

@@ -309,13 +304,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gemma4_vl_26b_sft_config --mode sft --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-4-26b-a4b-it/sft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/gemma-4-26b-a4b-it/sft-post-setup.yaml
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gemma4_vl_26b_sft_config --mode sft --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-4-26b-a4b-it/sft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/gemma-4-26b-a4b-it/sft-post-setup.yaml
Expected result
-

On 2026-08-11 the single-node TP4/PP1/EP8/ETP1 run loaded the immutable BF16 checkpoint and real CORD-v2 revision, then completed exactly 10 optimizer steps at MBS1 and GBS32. The persisted post-setup config confirmed a frozen vision encoder, trainable projection and language model, expandable CUDA allocator segments, and full-layer uniform recompute with one layer per recompute unit directly from the recipe defaults, while CUDA graphs and CPU offload remained disabled. Loss moved from 1.282612 to 0.09127883, final grad norm was 3.090, and all 10 steps reported zero skipped and zero NaN iterations. After iteration 2, logger-reported ranks returned to 62.134-62.145 GiB allocated, while cumulative peak allocated and reserved memory remained at or below 65.223 and 72.356 GiB. The run saved a complete iter_0000010 torch_dist checkpoint with eight rank shards, finalized metadata, run config, training state, and a latest-checkpoint tracker containing iteration 10. -

+

On 2026-08-11 the single-node TP4/PP1/EP8/ETP1 run loaded the immutable BF16 checkpoint and real CORD-v2 revision, then completed exactly 10 optimizer steps at MBS1 and GBS32. The persisted post-setup config confirmed a frozen vision encoder, trainable projection and language model, expandable CUDA allocator segments, and full-layer uniform recompute with one layer per recompute unit directly from the recipe defaults, while CUDA graphs and CPU offload remained disabled. Loss moved from 1.282612 to 0.09127883, final grad norm was 3.090, and all 10 steps reported zero skipped and zero NaN iterations. After iteration 2, logger-reported ranks returned to 62.134-62.145 GiB allocated, while cumulative peak allocated and reserved memory remained at or below 65.223 and 72.356 GiB. The run saved a complete iter_0000010 torch_dist checkpoint with eight rank shards, finalized metadata, run config, training state, and a latest-checkpoint tracker containing iteration 10.

@@ -359,8 +353,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No Gemma 4 VL long-context recipe or run was verified. A future workload must establish sequence-packing and context-parallel settings, complete at least 10 optimizer steps with finite loss and no skipped or NaN iterations, report all five required metrics, and save a reloadable final checkpoint. -

+

No Gemma 4 VL long-context recipe or run was verified. A future workload must establish sequence-packing and context-parallel settings, complete at least 10 optimizer steps with finite loss and no skipped or NaN iterations, report all five required metrics, and save a reloadable final checkpoint.

@@ -413,16 +406,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe gemma4_vl_26b_peft_config --mode lora --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-4-26b-a4b-it/peft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/gemma-4-26b-a4b-it/peft-post-setup.yaml
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe gemma4_vl_26b_peft_config --mode lora --dataset cord-v2 --pretrained_checkpoint work/model-verification/gemma-4-26b-a4b-it/imported-megatron/iter_0000000 --max_steps 10 --seq_length 4096 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gemma-4-26b-a4b-it/peft-checkpoints --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/gemma-4-26b-a4b-it/peft-post-setup.yaml
Expected result
-

On 2026-08-04 the 4-GPU TP2/PP1/EP4/ETP1 LoRA run loaded the immutable BF16 checkpoint and real CORD-v2 revision, then completed exactly 10 optimizer steps at MBS1 and GBS32. The persisted post-setup config recorded rank-32 adapters with alpha 32 and zero dropout on linear_qkv, linear_proj, linear_fc1, and linear_fc2. Runtime transformation froze the base model and reported 22,272,000 trainable adapter parameters, 0.30% of the local model shard. Loss moved from 1.288328 to 0.1072377, final grad norm was 22.994, and every step reported zero skipped and zero NaN iterations. Across all four ranks, allocated memory returned to 14.582 GiB after each step, while cumulative peak allocated and reserved memory remained at or below 44.252 and 45.637 GiB. The run saved a complete iter_0000010 adapter checkpoint with four rank shards, finalized metadata, training state, and a latest-checkpoint tracker containing iteration 10. -

+

On 2026-08-04 the 4-GPU TP2/PP1/EP4/ETP1 LoRA run loaded the immutable BF16 checkpoint and real CORD-v2 revision, then completed exactly 10 optimizer steps at MBS1 and GBS32. The persisted post-setup config recorded rank-32 adapters with alpha 32 and zero dropout on linear_qkv, linear_proj, linear_fc1, and linear_fc2. Runtime transformation froze the base model and reported 22,272,000 trainable adapter parameters, 0.30% of the local model shard. Loss moved from 1.288328 to 0.1072377, final grad norm was 22.994, and every step reported zero skipped and zero NaN iterations. Across all four ranks, allocated memory returned to 14.582 GiB after each step, while cumulative peak allocated and reserved memory remained at or below 44.252 and 45.637 GiB. The run saved a complete iter_0000010 adapter checkpoint with four rank shards, finalized metadata, training state, and a latest-checkpoint tracker containing iteration 10.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/glm/glm5-2.mdx b/docs/fern/versions/nightly/pages/models/glm/glm5-2.mdx index 256757bb55..fff772d55c 100644 --- a/docs/fern/versions/nightly/pages/models/glm/glm5-2.mdx +++ b/docs/fern/versions/nightly/pages/models/glm/glm5-2.mdx @@ -33,7 +33,7 @@ The recorded commands and results below belong to the checkpoint named in each v [Production qualification tracker #5476](https://github.com/NVIDIA-NeMo/Megatron-Bridge/issues/5476) remains open. It includes repeated-MTP training semantics, precision controls, dynamic context parallelism, production recipes, checkpoint continuity, convergence, and hardware-specific performance qualification. See also the open [export context-length fix #5996](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/5996), [router-bias fix #6036](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/6036), and [recipe-selection fix #5608](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/5608). - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -180,8 +180,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Import exits successfully and creates a reloadable CPU checkpoint. Its strict round-trip comparison matches all 59,585 BF16 tensors and 753,329,940,480 elements bitwise against the immutable HF revision. -

+

Import exits successfully and creates a reloadable CPU checkpoint. Its strict round-trip comparison matches all 59,585 BF16 tensors and 753,329,940,480 elements bitwise against the immutable HF revision.

@@ -206,8 +205,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Import exits successfully at TP1/PP13/EP8/ETP1 and creates a reloadable iter_0000000 checkpoint. Its strict round-trip comparison matches all 59,585 BF16 tensors and 753,329,940,480 elements bitwise against the immutable HF revision. -

+

Import exits successfully at TP1/PP13/EP8/ETP1 and creates a reloadable iter_0000000 checkpoint. Its strict round-trip comparison matches all 59,585 BF16 tensors and 753,329,940,480 elements bitwise against the immutable HF revision.

@@ -232,8 +230,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Export exits successfully; all 59,585 BF16 tensors and 753,329,940,480 elements match the immutable HF revision bitwise. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 743,377,000,704 parameters. -

+

Export exits successfully; all 59,585 BF16 tensors and 753,329,940,480 elements match the immutable HF revision bitwise. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 743,377,000,704 parameters.

@@ -258,8 +255,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed export exits successfully; all 59,585 BF16 tensors and 753,329,940,480 elements match the immutable HF revision bitwise. Transformers strictly reloads the output as GlmMoeDsaForCausalLM with 743,377,000,704 parameters. -

+

Distributed export exits successfully; all 59,585 BF16 tensors and 753,329,940,480 elements match the immutable HF revision bitwise. Transformers strictly reloads the output as GlmMoeDsaForCausalLM with 743,377,000,704 parameters.

@@ -309,8 +305,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The uninterrupted 352-H100 TP1/PP11/CP1/EP32/ETP1 run completed 100 real-data steps at GBS/MBS 1024/1 with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and flex/DeepEP dispatch. Loss remained finite from 13.15907 to 6.385243, final grad norm was 0.796, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 31,204.920 ms and 54.595 TFLOPS/GPU; peak observed GPU memory was 80,999 MiB. The matching step-50 and step-100 checkpoints contain model, optimizer, scheduler, data-order, and RNG state; the step-50 checkpoint reloads successfully. -

+

The uninterrupted 352-H100 TP1/PP11/CP1/EP32/ETP1 run completed 100 real-data steps at GBS/MBS 1024/1 with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and flex/DeepEP dispatch. Loss remained finite from 13.15907 to 6.385243, final grad norm was 0.796, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 31,204.920 ms and 54.595 TFLOPS/GPU; peak observed GPU memory was 80,999 MiB. The matching step-50 and step-100 checkpoints contain model, optimizer, scheduler, data-order, and RNG state; the step-50 checkpoint reloads successfully.

@@ -360,8 +355,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 192-GB200 TP1/PP6/CP1/EP32/ETP1 run completed exactly 100 RP2 head_01 pretraining steps at GBS/MBS 1024/1 with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and all-to-all expert dispatch. The RP2 Llama SentencePiece tokenizer is used while retaining GLM-5.2's configured vocabulary size, so this is support verification rather than a cross-tokenizer convergence claim. Loss remained finite from 13.168150 to 6.523522, final grad norm was 2.775, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 64,786.540 ms and 102.520 TFLOPS/GPU. Step-50 and step-100 checkpoints contain model, optimizer, scheduler, data-order, and RNG state. -

+

The 192-GB200 TP1/PP6/CP1/EP32/ETP1 run completed exactly 100 RP2 head_01 pretraining steps at GBS/MBS 1024/1 with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and all-to-all expert dispatch. The RP2 Llama SentencePiece tokenizer is used while retaining GLM-5.2's configured vocabulary size, so this is support verification rather than a cross-tokenizer convergence claim. Loss remained finite from 13.168150 to 6.523522, final grad norm was 2.775, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 64,786.540 ms and 102.520 TFLOPS/GPU. Step-50 and step-100 checkpoints contain model, optimizer, scheduler, data-order, and RNG state.

@@ -411,8 +405,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The immutable-revision 416-H100 run completes exactly 100 full-SFT steps at TP1/PP13/CP2/EP32/ETP1, GBS/MBS 32/1, with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and natural routing. Pad-32 packing is 99.60% efficient and contains 4,097,215 supervised tokens. Loss remains finite from 1.909875 to 0.6634203 with zero skipped or NaN iterations; the final grad norm is 0.895 and the step-100 checkpoint reloads for export. -

+

The immutable-revision 416-H100 run completes exactly 100 full-SFT steps at TP1/PP13/CP2/EP32/ETP1, GBS/MBS 32/1, with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and natural routing. Pad-32 packing is 99.60% efficient and contains 4,097,215 supervised tokens. Loss remains finite from 1.909875 to 0.6634203 with zero skipped or NaN iterations; the final grad norm is 0.895 and the step-100 checkpoint reloads for export.

@@ -462,8 +455,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The immutable-revision 192-GB200 run completed exactly 100 full-SFT steps at TP1/PP6/CP4/EP32/ETP1, DP8, and GBS/MBS 8/1 without gradient accumulation. It used the pinned Tulu 3 train[:10000] split with the GLM assistant-only chat template, 8,192-token offline packs, pad-8 sequence alignment, an 8,184-token single-sequence cap, constant-shape cu_seqlens, cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, natural routing, and HybridEP. Packing is 99.72% efficient across 421 rows with a 23.75 packing factor. The 100-step window contains 6,553,600 token slots and 4,283,561 supervised tokens. Metadata and Parquet SHA-256 values are cbd964c686b74f82804ceea2df75bb00fbe28648f0458a641b9db27977a55b1b and b2c6f71a6b5add331750956622073cc8ced0bb6b2742e1d7a0e1a9f763cbaa0c. Loss remained finite from 1.704979 to 0.3682597, final grad norm was 0.873, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 8,593.990 ms and 12.446 TFLOPS/GPU. A complete step-100 distributed checkpoint was written for the future GB200 export/reload gate. -

+

The immutable-revision 192-GB200 run completed exactly 100 full-SFT steps at TP1/PP6/CP4/EP32/ETP1, DP8, and GBS/MBS 8/1 without gradient accumulation. It used the pinned Tulu 3 train[:10000] split with the GLM assistant-only chat template, 8,192-token offline packs, pad-8 sequence alignment, an 8,184-token single-sequence cap, constant-shape cu_seqlens, cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, natural routing, and HybridEP. Packing is 99.72% efficient across 421 rows with a 23.75 packing factor. The 100-step window contains 6,553,600 token slots and 4,283,561 supervised tokens. Metadata and Parquet SHA-256 values are cbd964c686b74f82804ceea2df75bb00fbe28648f0458a641b9db27977a55b1b and b2c6f71a6b5add331750956622073cc8ced0bb6b2742e1d7a0e1a9f763cbaa0c. Loss remained finite from 1.704979 to 0.3682597, final grad norm was 0.873, and every step recorded zero skipped and NaN iterations. The final 10 steps averaged 8,593.990 ms and 12.446 TFLOPS/GPU. A complete step-100 distributed checkpoint was written for the future GB200 export/reload gate.

@@ -513,8 +505,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 608-H100 TP1/PP19/CP32/EP32/ETP1 run completed exactly 20 optimizer steps over 200,000-token packed rows with cuDNN DSA, sparse indexer loss coefficient 0.001, and MTP1. Loss remained finite from 8.283127 to 2.557671, final grad norm was 10838.703, and every step recorded zero skipped and NaN iterations. The last 10 steps averaged 36,603.370 ms and 39.648 TFLOPS/GPU; peak observed GPU memory was 81,005 MiB. -

+

The 608-H100 TP1/PP19/CP32/EP32/ETP1 run completed exactly 20 optimizer steps over 200,000-token packed rows with cuDNN DSA, sparse indexer loss coefficient 0.001, and MTP1. Loss remained finite from 8.283127 to 2.557671, final grad norm was 10838.703, and every step recorded zero skipped and NaN iterations. The last 10 steps averaged 36,603.370 ms and 39.648 TFLOPS/GPU; peak observed GPU memory was 81,005 MiB.

@@ -564,8 +555,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The immutable-revision 192-GB200 run completed exactly 20 optimizer steps at TP1/PP6/CP32/EP32/ETP1, DP1, and GBS/MBS 56/1 with cuDNN DSA, MTP1, HybridEP, and full activation recompute. The explicit PP6 decoder layout [14, 16, 12, 12, 12, 12] completed without OOM using 32 single-sequence, 131,072-token synthetic packs. Loss remained finite from 0.4159648 to 0.000511673 with zero skipped or NaN iterations across 146,786,080 supervised tokens. The final 10 steps averaged 143,248.820 ms and 88.848 TFLOPS/GPU. This verifies long-context execution only; the 8K pinned-Tulu run provides real-data convergence evidence. -

+

The immutable-revision 192-GB200 run completed exactly 20 optimizer steps at TP1/PP6/CP32/EP32/ETP1, DP1, and GBS/MBS 56/1 with cuDNN DSA, MTP1, HybridEP, and full activation recompute. The explicit PP6 decoder layout [14, 16, 12, 12, 12, 12] completed without OOM using 32 single-sequence, 131,072-token synthetic packs. Loss remained finite from 0.4159648 to 0.000511673 with zero skipped or NaN iterations across 146,786,080 supervised tokens. The final 10 steps averaged 143,248.820 ms and 88.848 TFLOPS/GPU. This verifies long-context execution only; the 8K pinned-Tulu run provides real-data convergence evidence.

@@ -615,8 +605,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The immutable-revision 208-H100 run completes exactly 100 LoRA steps at TP1/PP13/CP1/EP16/ETP1, GBS/MBS 32/1, with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and natural routing. Pad-4 packing is 99.45% efficient and contains 4,251,129 supervised tokens. Loss remains finite from 1.909421 to 0.8835775 with zero skipped or NaN iterations. The final grad norm is 0.101; the checkpoint contains exactly 790 BF16 rank-8 adapter tensors and no base-model weights. -

+

The immutable-revision 208-H100 run completes exactly 100 LoRA steps at TP1/PP13/CP1/EP16/ETP1, GBS/MBS 32/1, with cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and natural routing. Pad-4 packing is 99.45% efficient and contains 4,251,129 supervised tokens. Loss remains finite from 1.909421 to 0.8835775 with zero skipped or NaN iterations. The final grad norm is 0.101; the checkpoint contains exactly 790 BF16 rank-8 adapter tensors and no base-model weights.

@@ -666,11 +655,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The immutable-revision 192-GB200 run completed exactly 100 LoRA steps at TP1/PP6/CP1/EP32/ETP1, GBS/MBS 32/1, with offline-packed Tulu 3 data, cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and all-to-all expert dispatch. Loss remained finite from 1.831100 to 0.8957523, final grad norm was 0.101, and every step recorded zero skipped and NaN iterations. The step-100 adapter checkpoint was written for reload validation. -

+

The immutable-revision 192-GB200 run completed exactly 100 LoRA steps at TP1/PP6/CP1/EP32/ETP1, GBS/MBS 32/1, with offline-packed Tulu 3 data, cuDNN DSA, sparse indexer loss coefficient 0.001, MTP1, and all-to-all expert dispatch. Loss remained finite from 1.831100 to 0.8957523, final grad norm was 0.101, and every step recorded zero skipped and NaN iterations. The step-100 adapter checkpoint was written for reload validation.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/glm/glm5.mdx b/docs/fern/versions/nightly/pages/models/glm/glm5.mdx index ed69e6e9fa..cf63825abc 100644 --- a/docs/fern/versions/nightly/pages/models/glm/glm5.mdx +++ b/docs/fern/versions/nightly/pages/models/glm/glm5.mdx @@ -33,7 +33,7 @@ The recorded commands and results below belong to the checkpoint named in each v [Production qualification tracker #5476](https://github.com/NVIDIA-NeMo/Megatron-Bridge/issues/5476) remains open. It includes repeated-MTP training semantics, precision controls, dynamic context parallelism, production recipes, checkpoint continuity, convergence, and hardware-specific performance qualification. See also the open [export context-length fix #5996](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/5996), [router-bias fix #6036](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/6036), and [recipe-selection fix #5608](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/5608). - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -145,8 +145,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A CPU import of the pinned Hugging Face revision must complete every mapping and create a reloadable Megatron checkpoint. This workflow is deferred by this card. -

+

A CPU import of the pinned Hugging Face revision must complete every mapping and create a reloadable Megatron checkpoint. This workflow is deferred by this card.

@@ -171,8 +170,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned 32-H100 import exits successfully at TP1/PP2/EP8/ETP2, completes all 6,201 distributed mapping tasks, and persists a reloadable iter_0000000 checkpoint. Reload plus exact export projection covers all 59,079 tensors and 1,487,822,475,264 tensor-payload bytes in the 78-layer inference graph with matching keys, shapes, dtypes, and values. The 791 source tensors under model.layers.78 belong only to the intentionally disabled appended MTP auxiliary layer and are outside this item. -

+

The pinned 32-H100 import exits successfully at TP1/PP2/EP8/ETP2, completes all 6,201 distributed mapping tasks, and persists a reloadable iter_0000000 checkpoint. Reload plus exact export projection covers all 59,079 tensors and 1,487,822,475,264 tensor-payload bytes in the 78-layer inference graph with matching keys, shapes, dtypes, and values. The 791 source tensors under model.layers.78 belong only to the intentionally disabled appended MTP auxiliary layer and are outside this item.

@@ -197,8 +195,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 32-process distributed CPU export exits successfully and writes 280 safetensors shards. The exhaustive projection audit covers all 59,079 tensors and 1,487,822,475,264 tensor-payload bytes in the 78-layer inference graph with zero missing, unexpected, shape, dtype, or value mismatches. The 791 tensors under model.layers.78 belong only to the intentionally disabled appended MTP auxiliary layer and remain outside this item. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 1,629 state tensors, 743,911,199,232 parameters, and no loading discrepancies. -

+

The 32-process distributed CPU export exits successfully and writes 280 safetensors shards. The exhaustive projection audit covers all 59,079 tensors and 1,487,822,475,264 tensor-payload bytes in the 78-layer inference graph with zero missing, unexpected, shape, dtype, or value mismatches. The 791 tensors under model.layers.78 belong only to the intentionally disabled appended MTP auxiliary layer and remain outside this item. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 1,629 state tensors, 743,911,199,232 parameters, and no loading discrepancies.

@@ -223,8 +220,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 32-H100 distributed export exits successfully after all 6,201 mapping tasks and writes 280 safetensors shards. Of those, 278 are byte-for-byte identical to the pinned source shards; the two shards that also contain excluded MTP keys match all 63 common tensors exactly. The composite audit covers all 59,079 inference-graph tensors and 1,487,822,475,264 tensor-payload bytes with zero key, shape, dtype, or value mismatch. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 1,629 state tensors, 743,911,199,232 parameters, and no loading discrepancies. -

+

The 32-H100 distributed export exits successfully after all 6,201 mapping tasks and writes 280 safetensors shards. Of those, 278 are byte-for-byte identical to the pinned source shards; the two shards that also contain excluded MTP keys match all 63 common tensors exactly. The composite audit covers all 59,079 inference-graph tensors and 1,487,822,475,264 tensor-payload bytes with zero key, shape, dtype, or value mismatch. Transformers 5.12.1 strictly reloads the output as GlmMoeDsaForCausalLM with 1,629 state tensors, 743,911,199,232 parameters, and no loading discrepancies.

@@ -268,8 +264,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A public GLM-5 H100 recipe must complete a bounded 100-step run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card. -

+

A public GLM-5 H100 recipe must complete a bounded 100-step run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.

@@ -313,8 +308,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A pinned-data 100-step full-SFT run must finish with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card. -

+

A pinned-data 100-step full-SFT run must finish with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.

@@ -358,8 +352,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A dedicated packed long-context SFT run must complete a bounded run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable checkpoint. Training is deferred by this card. -

+

A dedicated packed long-context SFT run must complete a bounded run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable checkpoint. Training is deferred by this card.

@@ -403,11 +396,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A GLM-5 PEFT recipe with an audited adapter target set must complete a bounded run with finite loss, all five metrics, and a reloadable adapter checkpoint. Training is deferred by this card. -

+

A GLM-5 PEFT recipe with an audited adapter target set must complete a bounded run with finite loss, all five metrics, and a reloadable adapter checkpoint. Training is deferred by this card.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-120b.mdx b/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-120b.mdx index e1a424b8e5..9b36f5c356 100644 --- a/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-120b.mdx +++ b/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-120b.mdx @@ -6,7 +6,7 @@ The GPT OSS models feature decoder-only architectures with routed expert layers, GPT OSS models are supported via the Bridge system with specialized configurations for MoE optimizations and long-context training. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -140,8 +140,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Future verification must import the pinned MXFP4 Hugging Face checkpoint, dequantize it into a reloadable BF16 Megatron checkpoint at iter_0000000, and audit keys, shapes, dtypes, and values against the recorded source revision. -

+

Future verification must import the pinned MXFP4 Hugging Face checkpoint, dequantize it into a reloadable BF16 Megatron checkpoint at iter_0000000, and audit keys, shapes, dtypes, and values against the recorded source revision.

@@ -166,8 +165,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Imports the pinned MXFP4 checkpoint on 32 H100 GPUs and creates a reloadable TP1/PP4/EP8/ETP1 BF16 Megatron checkpoint at iter_0000000 with 32 DistCP shards, metadata, train-state, tokenizer, and latest-marker artifacts. -

+

Imports the pinned MXFP4 checkpoint on 32 H100 GPUs and creates a reloadable TP1/PP4/EP8/ETP1 BF16 Megatron checkpoint at iter_0000000 with 32 DistCP shards, metadata, train-state, tokenizer, and latest-marker artifacts.

@@ -192,8 +190,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The single-process CPU export exits successfully and writes the complete BF16 Hugging Face checkpoint. Exhaustive comparison against the pinned unsloth/gpt-oss-120b-BF16 reference covers all 615 tensors and 116,829,156,672 values with exact keys, shapes, dtypes, and bitwise-equal values. Transformers strictly reloads all 615 state tensors natively as GptOssForCausalLM with attention_bias=true and no loading discrepancies, and the tokenizer also reloads successfully. -

+

The single-process CPU export exits successfully and writes the complete BF16 Hugging Face checkpoint. Exhaustive comparison against the pinned unsloth/gpt-oss-120b-BF16 reference covers all 615 tensors and 116,829,156,672 values with exact keys, shapes, dtypes, and bitwise-equal values. Transformers strictly reloads all 615 state tensors natively as GptOssForCausalLM with attention_bias=true and no loading discrepancies, and the tokenizer also reloads successfully.

@@ -218,8 +215,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Exports the distributed Megatron checkpoint to a reloadable BF16 Hugging Face checkpoint with attention_bias=true. The exported checkpoint has 73 safetensors shards and 615 BF16 tensors totaling 116,829,156,672 parameters, with no missing, unexpected, shape, dtype, or value mismatches against unsloth/gpt-oss-120b-BF16. -

+

Exports the distributed Megatron checkpoint to a reloadable BF16 Hugging Face checkpoint with attention_bias=true. The exported checkpoint has 73 safetensors shards and 615 BF16 tensors totaling 116,829,156,672 parameters, with no missing, unexpected, shape, dtype, or value mismatches against unsloth/gpt-oss-120b-BF16.

@@ -269,8 +265,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 64x H100, the command completes exactly 100 bounded pretraining optimizer steps using the recipe-owned GBS/MBS, natural routing, and BF16 configuration. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 10.91136 to 5.828481, all five metrics are recorded, the post-setup run_config.yaml persists, and complete 64-shard iter_0000050 and iter_0000100 checkpoints are saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers suitable for direct resume. -

+

On 64x H100, the command completes exactly 100 bounded pretraining optimizer steps using the recipe-owned GBS/MBS, natural routing, and BF16 configuration. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 10.91136 to 5.828481, all five metrics are recorded, the post-setup run_config.yaml persists, and complete 64-shard iter_0000050 and iter_0000100 checkpoints are saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers suitable for direct resume.

@@ -315,13 +310,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe gpt_oss_120b_sft_32gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-sft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 model.tensor_model_parallel_size=2 model.sequence_parallel=true model.recompute_granularity=selective 'model.recompute_modules=[moe_act]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-120b/sft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe gpt_oss_120b_sft_32gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-sft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 model.tensor_model_parallel_size=2 model.sequence_parallel=true model.recompute_granularity=selective 'model.recompute_modules=[moe_act]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-120b/sft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Completed 100 full-SFT steps from the imported checkpoint on 32x H100 with immutable Tulu3 selection, assistant-token masking, offline packing, TP2/SP, and selective MoE-activation recompute. All 100 optimizer-step rows are present with finite losses, zero skipped iterations, and zero NaN iterations; loss decreases from 6.751403 to 0.7468084. All five metrics are recorded, the checkpoint-local post-setup run_config.yaml persists, and the final full-model checkpoint at iter_0000100 is complete with 32 DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint marker 100. -

+

Completed 100 full-SFT steps from the imported checkpoint on 32x H100 with immutable Tulu3 selection, assistant-token masking, offline packing, TP2/SP, and selective MoE-activation recompute. All 100 optimizer-step rows are present with finite losses, zero skipped iterations, and zero NaN iterations; loss decreases from 6.751403 to 0.7468084. All five metrics are recorded, the checkpoint-local post-setup run_config.yaml persists, and the final full-model checkpoint at iter_0000100 is complete with 32 DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint marker 100.

@@ -366,13 +360,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 6 --gpus-per-node 8 --recipe gpt_oss_120b_sft_48gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-sft-long-context-32k-cp4-pp6-nofuse-gbs4-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=8 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 6 --gpus-per-node 8 --recipe gpt_oss_120b_sft_48gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-sft-long-context-32k-cp4-pp6-nofuse-gbs4-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=8 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Completed 20 long-context full-SFT optimizer steps from the imported checkpoint on 48x H100 with 32K sequence length, immutable Tulu3 selection, assistant-token masking, offline packing, TP2/SP, CP4 with a2a context-parallel communication, PP6, EP4, disabled cross-entropy and bias-activation fusions, and selective MoE-activation recompute. All 20 optimizer-step rows are present exactly once with finite losses, zero skipped iterations, and zero NaN iterations. Loss decreases from 6.666666 to 1.238030, all five metrics are recorded, and no checkpoint is saved because this bounded long-context verification run disables checkpoint output. -

+

Completed 20 long-context full-SFT optimizer steps from the imported checkpoint on 48x H100 with 32K sequence length, immutable Tulu3 selection, assistant-token masking, offline packing, TP2/SP, CP4 with a2a context-parallel communication, PP6, EP4, disabled cross-entropy and bias-activation fusions, and selective MoE-activation recompute. All 20 optimizer-step rows are present exactly once with finite losses, zero skipped iterations, and zero NaN iterations. Loss decreases from 6.666666 to 1.238030, all five metrics are recorded, and no checkpoint is saved because this bounded long-context verification run disables checkpoint output.

@@ -417,13 +410,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_120b_peft_8gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-peft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-120b/peft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_120b_peft_8gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-120b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a"' dataset.hf_output_root=work/data/tulu3/gpt-oss-120b-peft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-120b/peft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Completed 100 LoRA steps from the imported checkpoint with the base frozen, model-native target modules, immutable Tulu3 selection, assistant-token masking, and offline packing. All 100 optimizer-step rows are present with finite losses, zero skipped iterations, and zero NaN iterations; the checkpoint-local post-setup run_config.yaml persists and latest_checkpointed_iteration.txt points to the reloadable final adapter checkpoint at iter_0000100. -

+

Completed 100 LoRA steps from the imported checkpoint with the base frozen, model-native target modules, immutable Tulu3 selection, assistant-token masking, and offline packing. All 100 optimizer-step rows are present with finite losses, zero skipped iterations, and zero NaN iterations; the checkpoint-local post-setup run_config.yaml persists and latest_checkpointed_iteration.txt points to the reloadable final adapter checkpoint at iter_0000100.

@@ -473,8 +465,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 64 GB200s, the canonical BF16 mock-data performance recipe completes 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 1280/4, forced router balancing, selective layernorm and MoE activation recompute, and expert-parallel communication overlap. All 50 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 12.79104 to 0.03246120. The final ten logged step times average 4995.630 ms, measured model throughput averages 535.230 TFLOP/s/GPU, and derived token-slot throughput averages 16398.332 tokens/s/GPU over the same final-ten window. The resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence. -

+

On exactly 64 GB200s, the canonical BF16 mock-data performance recipe completes 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 1280/4, forced router balancing, selective layernorm and MoE activation recompute, and expert-parallel communication overlap. All 50 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 12.79104 to 0.03246120. The final ten logged step times average 4995.630 ms, measured model throughput averages 535.230 TFLOP/s/GPU, and derived token-slot throughput averages 16398.332 tokens/s/GPU over the same final-ten window. The resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence.

@@ -524,11 +515,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 64 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP16/ETP1, GBS/MBS 1280/4, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.78424 to 0.0985439; the final ten steps average 2481.140 ms, 1077.350 TFLOP/s/GPU, and 33017.081 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 64 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP16/ETP1, GBS/MBS 1280/4, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.78424 to 0.0985439; the final ten steps average 2481.140 ms, 1077.350 TFLOP/s/GPU, and 33017.081 tokens/s/GPU. The resolved configuration persists.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-20b.mdx b/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-20b.mdx index 87ac0db6f8..8fa77ca40e 100644 --- a/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-20b.mdx +++ b/docs/fern/versions/nightly/pages/models/gpt_oss/gpt-oss-20b.mdx @@ -6,7 +6,7 @@ The GPT OSS models feature decoder-only architectures with routed expert layers, GPT OSS models are supported via the Bridge system with specialized configurations for MoE optimizations and long-context training. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -124,8 +124,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Future verification must import the pinned MXFP4 Hugging Face checkpoint, dequantize it into a reloadable BF16 Megatron checkpoint at iter_0000000, and audit keys, shapes, dtypes, and values against the recorded source revision. -

+

Future verification must import the pinned MXFP4 Hugging Face checkpoint, dequantize it into a reloadable BF16 Megatron checkpoint at iter_0000000, and audit keys, shapes, dtypes, and values against the recorded source revision.

@@ -150,8 +149,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, imports the pinned MXFP4 Hugging Face checkpoint on 8 GPUs, and creates iter_0000000 as a distributed TP2/PP2/EP2/ETP1 Megatron checkpoint with eight DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint markers. -

+

The command exits successfully, imports the pinned MXFP4 Hugging Face checkpoint on 8 GPUs, and creates iter_0000000 as a distributed TP2/PP2/EP2/ETP1 Megatron checkpoint with eight DistCP shards, metadata, train state, tokenizer artifacts, and latest-checkpoint markers.

@@ -176,8 +174,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The synchronous CPU workflow exports the verified distributed Megatron checkpoint to the pinned public unquantized BF16 Hugging Face layout. Its 411 tensors, shapes, dtypes, and all 20,914,757,184 values match the BF16 reference bitwise at zero tolerance. Native Transformers loading succeeds as GptOssForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export. -

+

The synchronous CPU workflow exports the verified distributed Megatron checkpoint to the pinned public unquantized BF16 Hugging Face layout. Its 411 tensors, shapes, dtypes, and all 20,914,757,184 values match the BF16 reference bitwise at zero tolerance. Native Transformers loading succeeds as GptOssForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export.

@@ -202,8 +199,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, exports the distributed Megatron checkpoint to the public unquantized BF16 Hugging Face layout, and writes a reloadable GptOssForCausalLM checkpoint with attention_bias=true. The export has 411 BF16 tensors, no missing or unexpected keys relative to unsloth/gpt-oss-20b-BF16, and all 20,914,757,184 parameters match that BF16 reference bitwise. -

+

The command exits successfully, exports the distributed Megatron checkpoint to the public unquantized BF16 Hugging Face layout, and writes a reloadable GptOssForCausalLM checkpoint with attention_bias=true. The export has 411 BF16 tensors, no missing or unexpected keys relative to unsloth/gpt-oss-20b-BF16, and all 20,914,757,184 parameters match that BF16 reference bitwise.

@@ -253,8 +249,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 16x H100, the command completes exactly 100 bounded pretraining optimizer steps. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 10.90364 to 5.725391, all five metrics are recorded, the post-setup run_config.yaml persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers suitable for direct resume. -

+

On 16x H100, the command completes exactly 100 bounded pretraining optimizer steps. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 10.90364 to 5.725391, all five metrics are recorded, the post-setup run_config.yaml persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers suitable for direct resume.

@@ -299,13 +294,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_20b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="6cee5e81ee83917806bbde320786a8fb61efebee"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-sft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-20b/sft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_20b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="6cee5e81ee83917806bbde320786a8fb61efebee"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-sft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-20b/sft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

On 8x H100, the command completes exactly 100 full-SFT optimizer steps from the imported GPT-OSS 20B checkpoint with immutable Tulu3 data selection, assistant-token loss masking, and offline packing. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 2.130852 to 0.758708, all five metrics are recorded, packing is 98.56% efficient, and a complete 8-shard iter_0000100 full-model checkpoint is saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers. -

+

On 8x H100, the command completes exactly 100 full-SFT optimizer steps from the imported GPT-OSS 20B checkpoint with immutable Tulu3 data selection, assistant-token loss masking, and offline packing. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 2.130852 to 0.758708, all five metrics are recorded, packing is 98.56% efficient, and a complete 8-shard iter_0000100 full-model checkpoint is saved with metadata, train state, tokenizer artifacts, and latest-checkpoint markers.

@@ -350,13 +344,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_20b_sft_8gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --tensor_model_parallel_size 4 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="6cee5e81ee83917806bbde320786a8fb61efebee"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe gpt_oss_20b_sft_8gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --tensor_model_parallel_size 4 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="6cee5e81ee83917806bbde320786a8fb61efebee"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Completed 20 optimizer steps using the 32K SFT recipe, immutable Tulu3 data selection, offline packing, TP4, CP2, sequence parallelism, non-fused cross entropy, and full activation recompute. The run loaded 120 packed rows, recorded 99.79% packing efficiency with 83.333 source sequences per pack on average, produced finite loss from 2.235548 to 1.175326, and reported zero skipped and zero NaN iterations. -

+

Completed 20 optimizer steps using the 32K SFT recipe, immutable Tulu3 data selection, offline packing, TP4, CP2, sequence parallelism, non-fused cross entropy, and full activation recompute. The run loaded 120 packed rows, recorded 99.79% packing efficiency with 83.333 source sequences per pack on average, produced finite loss from 2.235548 to 1.175326, and reported zero skipped and zero NaN iterations.

@@ -401,16 +394,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gpt_oss_20b_peft_1gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="6cee5e81ee83917806bbde320786a8fb61efebee"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-peft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-20b/peft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe gpt_oss_20b_peft_1gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/gpt-oss-20b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="6cee5e81ee83917806bbde320786a8fb61efebee"' dataset.hf_output_root=work/data/tulu3/gpt-oss-20b-peft-2k-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/gpt-oss-20b/peft-2k-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

On 1x H100, the command completes exactly 100 LoRA PEFT optimizer steps from the imported GPT-OSS 20B checkpoint with the base model frozen, LoRA dim 32, alpha 32, dropout 0.0, target modules linear_qkv, linear_proj, linear_fc1, and linear_fc2, immutable Tulu3 data selection, assistant-token loss masking, and offline packing. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 2.129699 to 1.075515, all five metrics are recorded, packing is 98.56% efficient, and a complete single-shard iter_0000100 adapter checkpoint is saved with metadata, train state, run_config.yaml, and latest-checkpoint markers. -

+

On 1x H100, the command completes exactly 100 LoRA PEFT optimizer steps from the imported GPT-OSS 20B checkpoint with the base model frozen, LoRA dim 32, alpha 32, dropout 0.0, target modules linear_qkv, linear_proj, linear_fc1, and linear_fc2, immutable Tulu3 data selection, assistant-token loss masking, and offline packing. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 2.129699 to 1.075515, all five metrics are recorded, packing is 98.56% efficient, and a complete single-shard iter_0000100 adapter checkpoint is saved with metadata, train state, run_config.yaml, and latest-checkpoint markers.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/kimi/kimi-k3.mdx b/docs/fern/versions/nightly/pages/models/kimi/kimi-k3.mdx index b794666795..f042c2b81a 100644 --- a/docs/fern/versions/nightly/pages/models/kimi/kimi-k3.mdx +++ b/docs/fern/versions/nightly/pages/models/kimi/kimi-k3.mdx @@ -6,7 +6,7 @@ Support for this model is in progress. Conversion (HF → Megatron) and Megatron greedy inference are verified; strict full-checkpoint export, exact round-trip parity, and every training workflow are not. See Known Limitations and the machine-readable [verification card](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/model_verification_cards/kimi-k3/card.yaml) before relying on this path. ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -118,8 +118,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No CPU Hugging Face-to-Megatron conversion result is claimed. Future verification must import the complete pinned language backbone with the public conversion launcher, persist iter_0000000, reload it, and audit all mapped keys, shapes, dtypes, and values. -

+

No CPU Hugging Face-to-Megatron conversion result is claimed. Future verification must import the complete pinned language backbone with the public conversion launcher, persist iter_0000000, reload it, and audit all mapped keys, shapes, dtypes, and values.

@@ -144,8 +143,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 48x GB200, the pinned complete checkpoint imports at TP2/PP3/EP8/ETP2 and exits successfully in 27:01. Published MXFP4 routed-expert weights are dequantized to BF16, and iter_0000000 is persisted with 48 distributed-checkpoint shards, metadata, and the recorded 93-layer run configuration. Exact Hugging Face round-trip parity is tracked separately by megatron_to_hf_gpu. -

+

On 48x GB200, the pinned complete checkpoint imports at TP2/PP3/EP8/ETP2 and exits successfully in 27:01. Published MXFP4 routed-expert weights are dequantized to BF16, and iter_0000000 is persisted with 48 distributed-checkpoint shards, metadata, and the recorded 93-layer run configuration. Exact Hugging Face round-trip parity is tracked separately by megatron_to_hf_gpu.

@@ -164,8 +162,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No CPU Megatron-to-Hugging-Face conversion result is claimed. Future verification depends on a complete persisted Megatron import and must write, strictly reload, and exactly audit the exported Hugging Face language-backbone weights against the pinned source revision. -

+

No CPU Megatron-to-Hugging-Face conversion result is claimed. Future verification depends on a complete persisted Megatron import and must write, strictly reload, and exactly audit the exported Hugging Face language-backbone weights against the pinned source revision.

@@ -184,8 +181,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No GPU Megatron-to-Hugging-Face conversion result is claimed. All 69 source KDA A_log tensors were audited: each stores 96 active heads plus 32 zero-valued inactive entries, for 2,208 checked padding values. Import validation drops only this zero padding, and export unit tests restore it with zeros. A four-layer diagnostic proxy completed low-memory GPU import and distributed streaming export. Its seven safetensors shards and index, totaling 58,482,941,904 bytes, were byte-identical to an export from the previous default-save import; this validates the low-memory I/O behavior, not full-model parity. Against the published proxy source, all 16,402 keys and shapes matched and A_log padding was valid. Three o_norm tensors changed from FP32 to BF16, and 16,128 value mismatches were concentrated in MXFP4 expert packed weights and scales because import dequantizes them to BF16 before export requantizes them. Full verification therefore still requires the pinned complete model to export and strictly reload, exact comparison of lossless tensors, and a declared numerical criterion for lossy MXFP4 reconstruction. -

+

No GPU Megatron-to-Hugging-Face conversion result is claimed. All 69 source KDA A_log tensors were audited: each stores 96 active heads plus 32 zero-valued inactive entries, for 2,208 checked padding values. Import validation drops only this zero padding, and export unit tests restore it with zeros. A four-layer diagnostic proxy completed low-memory GPU import and distributed streaming export. Its seven safetensors shards and index, totaling 58,482,941,904 bytes, were byte-identical to an export from the previous default-save import; this validates the low-memory I/O behavior, not full-model parity. Against the published proxy source, all 16,402 keys and shapes matched and A_log padding was valid. Three o_norm tensors changed from FP32 to BF16, and 16,128 value mismatches were concentrated in MXFP4 expert packed weights and scales because import dequantizes them to BF16 before export requantizes them. Full verification therefore still requires the pinned complete model to export and strictly reload, exact comparison of lossless tensors, and a declared numerical criterion for lossy MXFP4 reconstruction.

@@ -229,8 +225,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Training was intentionally excluded from this model-only verification pass. A future public Kimi K3 language-backbone recipe must complete a bounded 100-step pretrain run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. -

+

Training was intentionally excluded from this model-only verification pass. A future public Kimi K3 language-backbone recipe must complete a bounded 100-step pretrain run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint.

@@ -274,8 +269,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Training was intentionally excluded from this model-only verification pass. A future public Kimi K3 language-backbone recipe must complete a pinned-data 100-step full-SFT run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. -

+

Training was intentionally excluded from this model-only verification pass. A future public Kimi K3 language-backbone recipe must complete a pinned-data 100-step full-SFT run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint.

@@ -319,8 +313,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Training was intentionally excluded from this model-only verification pass. A future public long-context SFT recipe must define supported sequence packing and context parallelism, complete 100 steps with finite loss and no skipped or NaN iterations, and record all five metrics. -

+

Training was intentionally excluded from this model-only verification pass. A future public long-context SFT recipe must define supported sequence packing and context parallelism, complete 100 steps with finite loss and no skipped or NaN iterations, and record all five metrics.

@@ -364,11 +357,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Training was intentionally excluded from this model-only verification pass. A future Kimi K3 PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint. -

+

Training was intentionally excluded from this model-only verification pass. A future Kimi K3 PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable adapter checkpoint.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/minimax/minimax-m3.mdx b/docs/fern/versions/nightly/pages/models/minimax/minimax-m3.mdx index 22594d3db5..0a8f08da38 100644 --- a/docs/fern/versions/nightly/pages/models/minimax/minimax-m3.mdx +++ b/docs/fern/versions/nightly/pages/models/minimax/minimax-m3.mdx @@ -2,7 +2,7 @@ [MiniMax-M3](https://huggingface.co/MiniMaxAI/MiniMax-M3) is a natively multimodal sparse MoE model from MiniMaxAI (428B total, ~23B active parameters). `MiniMaxM3Bridge` converts the vision tower, both multimodal projector stages, and sparse-MoE text backbone into a `MiniMaxM3VLModel`. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -114,8 +114,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No CPU HF-to-Megatron result is claimed. The CPU-only attempt enabled CPU weight initialization, but the current distributed-model initialization path still selected a CUDA device before tensor loading. With no GPU available, it exited without persisting iter_0000000, so no checkpoint reload or exact tensor audit completed. CPU conversion remains unverified rather than unsupported. -

+

No CPU HF-to-Megatron result is claimed. The CPU-only attempt enabled CPU weight initialization, but the current distributed-model initialization path still selected a CUDA device before tensor loading. With no GPU available, it exited without persisting iter_0000000, so no checkpoint reload or exact tensor audit completed. CPU conversion remains unverified rather than unsupported.

@@ -140,8 +139,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned 32-H100 full-VLM import exited successfully, persisted iter_0000000, and reloaded it at TP1/PP1/EP32/ETP1. A strict projection of the reloaded Megatron model covered exactly 23,416 of 23,416 published language, vision, projector, and lightning-indexer tensors and 854,172,958,720 actual tensor-payload bytes, with zero duplicate, missing, unexpected, shape, invalid-dtype, or value mismatches. The source index's declared total_size is 869,157,697,024 bytes. No dtype widening was required. The audit covered 1,804 total Megatron state tensors; model-state structure matched across all 32 EP ranks, and all 1,348 replicated tensors comprising 13,917,973,376 elements matched exactly. -

+

The pinned 32-H100 full-VLM import exited successfully, persisted iter_0000000, and reloaded it at TP1/PP1/EP32/ETP1. A strict projection of the reloaded Megatron model covered exactly 23,416 of 23,416 published language, vision, projector, and lightning-indexer tensors and 854,172,958,720 actual tensor-payload bytes, with zero duplicate, missing, unexpected, shape, invalid-dtype, or value mismatches. The source index's declared total_size is 869,157,697,024 bytes. No dtype widening was required. The audit covered 1,804 total Megatron state tensors; model-state structure matched across all 32 EP ranks, and all 1,348 replicated tensors comprising 13,917,973,376 elements matched exactly.

@@ -166,8 +164,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The single-node CPU export exited successfully and wrote the complete indexed MiniMax-M3 VLM checkpoint. An exhaustive audit matched all 23,416 source and export tensors, 427,040,140,160 values, and 854,172,958,720 tensor-payload bytes with zero missing, unexpected, shape, dtype, or value mismatches and no dtype conversions (atol=0, rtol=0). A meta-device Transformers 5.14.0 reload instantiated MiniMaxM3SparseForConditionalGeneration with 1,582 state entries and zero missing, unexpected, mismatched, or error keys. -

+

The single-node CPU export exited successfully and wrote the complete indexed MiniMax-M3 VLM checkpoint. An exhaustive audit matched all 23,416 source and export tensors, 427,040,140,160 values, and 854,172,958,720 tensor-payload bytes with zero missing, unexpected, shape, dtype, or value mismatches and no dtype conversions (atol=0, rtol=0). A meta-device Transformers 5.14.0 reload instantiated MiniMaxM3SparseForConditionalGeneration with 1,582 state entries and zero missing, unexpected, mismatched, or error keys.

@@ -192,8 +189,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 32-H100 distributed export exited successfully and wrote an indexed full-VLM checkpoint at TP1/PP1/EP32/ETP1. Its key-to-shard map and all 59 safetensors shards are byte-for-byte identical to the pinned source, covering all 23,416 tensors and 854,172,958,720 actual tensor-payload bytes. The exported configuration preserves 60 text layers, 128 top-4 experts, 32 vision layers, both projector stages, and 57 lightning-indexer layers. Three tokenizer probes and multimodal processor inputs match the source. Transformers 5.14.0 natively reloads both source and export as MiniMaxM3SparseForConditionalGeneration through its checkpoint conversion mapping with zero missing, unexpected, mismatched, or error keys. The export consolidates the tokenizer in tokenizer.json and does not reproduce four redundant source-side legacy tokenizer files. -

+

The 32-H100 distributed export exited successfully and wrote an indexed full-VLM checkpoint at TP1/PP1/EP32/ETP1. Its key-to-shard map and all 59 safetensors shards are byte-for-byte identical to the pinned source, covering all 23,416 tensors and 854,172,958,720 actual tensor-payload bytes. The exported configuration preserves 60 text layers, 128 top-4 experts, 32 vision layers, both projector stages, and 57 lightning-indexer layers. Three tokenizer probes and multimodal processor inputs match the source. Transformers 5.14.0 natively reloads both source and export as MiniMaxM3SparseForConditionalGeneration through its checkpoint conversion mapping with zero missing, unexpected, mismatched, or error keys. The export consolidates the tokenizer in tokenizer.json and does not reproduce four redundant source-side legacy tokenizer files.

@@ -237,8 +233,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Complete a 100-step HybridEP language-backbone run with finite loss, reported metrics, and a reloadable checkpoint. -

+

Complete a 100-step HybridEP language-backbone run with finite loss, reported metrics, and a reloadable checkpoint.

@@ -282,8 +277,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Complete a 100-step packed HybridEP language-backbone SFT run with finite loss, reported metrics, and a reloadable checkpoint. -

+

Complete a 100-step packed HybridEP language-backbone SFT run with finite loss, reported metrics, and a reloadable checkpoint.

@@ -327,8 +321,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A dedicated packed long-context recipe must complete 100 steps with context parallelism, finite loss, no skipped or NaN iterations, and all five metrics. Beyond 2,048 tokens, Bridge uses full causal attention instead of MiniMax-M3's lightning-indexer sparse attention. -

+

A dedicated packed long-context recipe must complete 100 steps with context parallelism, finite loss, no skipped or NaN iterations, and all five metrics. Beyond 2,048 tokens, Bridge uses full causal attention instead of MiniMax-M3's lightning-indexer sparse attention.

@@ -372,11 +365,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A MiniMax-M3 PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, no skipped or NaN iterations, all four metrics, and a reloadable adapter checkpoint. -

+

A MiniMax-M3 PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, no skipped or NaN iterations, all four metrics, and a reloadable adapter checkpoint.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/moonlight/moonlight.mdx b/docs/fern/versions/nightly/pages/models/moonlight/moonlight.mdx index e04c74e33f..410b26289d 100644 --- a/docs/fern/versions/nightly/pages/models/moonlight/moonlight.mdx +++ b/docs/fern/versions/nightly/pages/models/moonlight/moonlight.mdx @@ -6,7 +6,7 @@ The model features 27 decoder layers with 64 routed experts and 8 shared experts Moonlight models are supported via the Bridge system with specialized configurations for MoE and MLA optimizations. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -124,8 +124,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with exact model configuration. After native reload, all 377 registered parameters and persistent buffers match bitwise by key, shape, dtype, and value. In the serialized files, 26 router correction-bias buffers widen losslessly from BF16 to FP32. -

+

The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with exact model configuration. After native reload, all 377 registered parameters and persistent buffers match bitwise by key, shape, dtype, and value. In the serialized files, 26 router correction-bias buffers widen losslessly from BF16 to FP32.

@@ -150,8 +149,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, creates a reloadable iter_0000000, and an exact audit finds all 5,344 serialized source tensors and the model configuration unchanged after the paired GPU export. -

+

The command exits successfully, creates a reloadable iter_0000000, and an exact audit finds all 5,344 serialized source tensors and the model configuration unchanged after the paired GPU export.

@@ -176,8 +174,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, the model configuration is exact, and all 377 registered parameters and persistent buffers match bitwise by key, shape, dtype, and value after native reload. Transformers strictly reloads the output as DeepseekV3ForCausalLM with trust_remote_code disabled. In the serialized files, 26 router correction-bias buffers widen losslessly from BF16 to FP32 and 27 stale, nonpersistent source inv_freq tensors are omitted. -

+

The command exits successfully, the model configuration is exact, and all 377 registered parameters and persistent buffers match bitwise by key, shape, dtype, and value after native reload. Transformers strictly reloads the output as DeepseekV3ForCausalLM with trust_remote_code disabled. In the serialized files, 26 router correction-bias buffers widen losslessly from BF16 to FP32 and 27 stale, nonpersistent source inv_freq tensors are omitted.

@@ -202,8 +199,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict distributed export exits successfully, all 5,344 serialized tensor keys, shapes, dtypes, and values match the recorded HF revision, and native Transformers strictly reloads the output as DeepseekV3ForCausalLM with trust_remote_code disabled and no missing or unexpected keys. -

+

Strict distributed export exits successfully, all 5,344 serialized tensor keys, shapes, dtypes, and values match the recorded HF revision, and native Transformers strictly reloads the output as DeepseekV3ForCausalLM with trust_remote_code disabled and no missing or unexpected keys.

@@ -253,8 +249,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The uninterrupted 16-GPU bounded RP2 run completes exactly 100 steps at TP1/PP1/CP1/EP8/ETP1, DP16, GBS/MBS 1024/2, and 32-way gradient accumulation with natural MoE routing and the HybridEP flex dispatcher. It reaches peak learning rate at step 40 and completes cosine decay at step 100. LM loss is finite from 12.43248 to 5.158420 with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved. -

+

The uninterrupted 16-GPU bounded RP2 run completes exactly 100 steps at TP1/PP1/CP1/EP8/ETP1, DP16, GBS/MBS 1024/2, and 32-way gradient accumulation with natural MoE routing and the HybridEP flex dispatcher. It reaches peak learning rate at step 40 and completes cosine decay at step 100. LM loss is finite from 12.43248 to 5.158420 with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved.

@@ -299,13 +294,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe moonlight_16b_sft_8gpu_h100_bf16_tp1_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="476b36a473d4467f94469414bef6cee75c9c8172"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-sft-hybridep-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 'dataset.dataset_kwargs={pad_to_max_length:true}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/moonlight-16b-a3b/sft-convergence-hybridep-v3-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe moonlight_16b_sft_8gpu_h100_bf16_tp1_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="476b36a473d4467f94469414bef6cee75c9c8172"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-sft-hybridep-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 'dataset.dataset_kwargs={pad_to_max_length:true}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/moonlight-16b-a3b/sft-convergence-hybridep-v3-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

The immutable-revision 8-GPU run completes exactly 100 full-SFT steps at TP1/PP1/CP1/EP8/ETP1, DP8, SP off, GBS/MBS 8/1, and no gradient accumulation, using the HybridEP flex dispatcher with overlapped expert communication and delayed weight-gradient compute. Fixed-width 8K, pad-1 offline packing is 99.73% efficient and averages 21.550 source sequences per pack while retaining 6,553,600 token slots across the 100 updates. LM loss is finite from 1.770799 to 1.180197 with no skipped or NaN iterations, all five metrics are recorded, and the complete eight-shard iter_0000100 full-model checkpoint saves successfully. -

+

The immutable-revision 8-GPU run completes exactly 100 full-SFT steps at TP1/PP1/CP1/EP8/ETP1, DP8, SP off, GBS/MBS 8/1, and no gradient accumulation, using the HybridEP flex dispatcher with overlapped expert communication and delayed weight-gradient compute. Fixed-width 8K, pad-1 offline packing is 99.73% efficient and averages 21.550 source sequences per pack while retaining 6,553,600 token slots across the 100 updates. LM loss is finite from 1.770799 to 1.180197 with no skipped or NaN iterations, all five metrics are recorded, and the complete eight-shard iter_0000100 full-model checkpoint saves successfully.

@@ -350,13 +344,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe moonlight_16b_sft_8k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 8192 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="476b36a473d4467f94469414bef6cee75c9c8172"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe moonlight_16b_sft_8k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 8192 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="476b36a473d4467f94469414bef6cee75c9c8172"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true
Expected result
-

The immutable-revision 8-GPU run completes exactly 20 Tulu3 SFT steps at the model's 8192-token context limit with recipe-owned TP2/PP1/CP2/EP8/SP-on, GBS/MBS 128/1, and explicit pad-4 offline packing. LM loss is 1.306440 to 1.228019; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.69%, and the sampled training window contains 13,765,732 actual supervised tokens. -

+

The immutable-revision 8-GPU run completes exactly 20 Tulu3 SFT steps at the model's 8192-token context limit with recipe-owned TP2/PP1/CP2/EP8/SP-on, GBS/MBS 128/1, and explicit pad-4 offline packing. LM loss is 1.306440 to 1.228019; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.69%, and the sampled training window contains 13,765,732 actual supervised tokens.

@@ -401,16 +394,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe moonlight_16b_peft_4gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="476b36a473d4467f94469414bef6cee75c9c8172"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/moonlight-16b-a3b/peft-convergence-v1-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe moonlight_16b_peft_4gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/moonlight-16b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="476b36a473d4467f94469414bef6cee75c9c8172"' dataset.hf_output_root=work/data/tulu3/moonlight-16b-a3b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/moonlight-16b-a3b/peft-convergence-v1-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

The immutable-revision 4-GPU run completes exactly 100 PEFT steps at TP1/PP1/CP1/EP4/ETP1, DP4, SP off, GBS/MBS 32/1, and eight-way gradient accumulation. Pad-4 offline packing is 99.29% efficient, and the sampled 6,553,600 token slots contain 4,291,091 supervised tokens after label masking. Only rank-8, alpha-16, zero-dropout LoRA on linear_q_proj, linear_kv_down_proj, linear_kv_up_proj, and linear_proj is trainable. LM loss is finite from 1.264324 to 1.100755 with no skipped or NaN iterations, all five metrics are recorded, and the complete four-shard iter_0000100 adapter checkpoint covers all 216 expected adapter entries. -

+

The immutable-revision 4-GPU run completes exactly 100 PEFT steps at TP1/PP1/CP1/EP4/ETP1, DP4, SP off, GBS/MBS 32/1, and eight-way gradient accumulation. Pad-4 offline packing is 99.29% efficient, and the sampled 6,553,600 token slots contain 4,291,091 supervised tokens after label masking. Only rank-8, alpha-16, zero-dropout LoRA on linear_q_proj, linear_kv_down_proj, linear_kv_up_proj, and linear_proj is trainable. LM loss is finite from 1.264324 to 1.100755 with no skipped or NaN iterations, all five metrics are recorded, and the complete four-shard iter_0000100 adapter checkpoint covers all 216 expected adapter entries.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/muse_glimmer/muse-glimmer.mdx b/docs/fern/versions/nightly/pages/models/muse_glimmer/muse-glimmer.mdx index cad46d051f..22e6263aa4 100644 --- a/docs/fern/versions/nightly/pages/models/muse_glimmer/muse-glimmer.mdx +++ b/docs/fern/versions/nightly/pages/models/muse_glimmer/muse-glimmer.mdx @@ -5,7 +5,7 @@ vision-language model with a roughly 2B-parameter vision encoder and a 28B-param decoder. Megatron Bridge models the complete checkpoint: the vision tower, adapter, projection, and text decoder all participate in checkpoint conversion. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -130,8 +130,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On a CPU-only node with no CUDA runtime, the command builds a native 52-layer HybridModel, completes all 1,228 mappings for 29,776,626,688 parameters, and persists iter_0000000. The checkpoint reloads for CPU export, whose exact audit covers all 1,436 source tensors and 29,776,626,688 elements with zero missing, unexpected, shape, dtype, or value mismatches. -

+

On a CPU-only node with no CUDA runtime, the command builds a native 52-layer HybridModel, completes all 1,228 mappings for 29,776,626,688 parameters, and persists iter_0000000. The checkpoint reloads for CPU export, whose exact audit covers all 1,436 source tensors and 29,776,626,688 elements with zero missing, unexpected, shape, dtype, or value mismatches.

@@ -156,8 +155,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The eight-GPU command builds the complete native HybridModel, completes all 1,228 mappings, validates the TP8 distributed state, and persists iter_0000000. The checkpoint reloads at TP8 for strict GPU export; its complete round-trip audit is bitwise exact across all 1,436 tensors and 29,776,626,688 elements. -

+

The eight-GPU command builds the complete native HybridModel, completes all 1,228 mappings, validates the TP8 distributed state, and persists iter_0000000. The checkpoint reloads at TP8 for strict GPU export; its complete round-trip audit is bitwise exact across all 1,436 tensors and 29,776,626,688 elements.

@@ -182,8 +180,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict export exits successfully. All 1,436 BF16 tensors and 29,776,626,688 elements match the pinned source exactly in keys, shapes, dtypes, and values (maximum absolute difference 0). Transformers 5.15.0 reloads the result as MuseGlimmerForConditionalGeneration with no missing, unexpected, mismatched, or error keys and the same parameter count. -

+

Strict export exits successfully. All 1,436 BF16 tensors and 29,776,626,688 elements match the pinned source exactly in keys, shapes, dtypes, and values (maximum absolute difference 0). Transformers 5.15.0 reloads the result as MuseGlimmerForConditionalGeneration with no missing, unexpected, mismatched, or error keys and the same parameter count.

@@ -208,8 +205,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict distributed export exits successfully. Every language, vision, adapter, and projection tensor matches the pinned source bitwise: 1,436 tensors and 29,776,626,688 elements with zero missing, unexpected, shape, dtype, or value mismatches. Transformers 5.15.0 strictly reloads the result with the same class and parameter count. -

+

Strict distributed export exits successfully. Every language, vision, adapter, and projection tensor matches the pinned source bitwise: 1,436 tensors and 29,776,626,688 elements with zero missing, unexpected, shape, dtype, or value mismatches. Transformers 5.15.0 strictly reloads the result with the same class and parameter count.

@@ -259,8 +255,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 32 H100 GPUs at TP4/PP4/DP2 with sequence parallelism, a 9|15|15|13 split of the 52 decoder layers, MBS2/GBS1024 at sequence length 4096, and four data-loader workers, the command trains the complete randomly initialized HybridModel for exactly 100 optimizer steps on the revision-pinned public CORD-v2 multimodal dataset. All losses and gradient norms remain finite, with zero skipped and zero NaN iterations. Loss decreases from 12.23791 to 0.6403766; steps 91-100 average 105,995.34 ms, 205.59 model TFLOP/s/GPU, and 1,236.583 token slots/s/GPU. The run used a NeMo 26.10 release-candidate base container with the pinned Megatron-Core submodule. The persisted post-setup config records the exact runtime. Complete step-50 and step-100 checkpoints each contain 32 nonempty distributed shards and 1,566 state entries, including model, optimizer, scheduler, and RNG state, plus metadata, train state, and the native HybridModel run config. Data-loader worker state is not checkpointed, so a resume from step 50 is not guaranteed to replay the identical sample stream. -

+

On 32 H100 GPUs at TP4/PP4/DP2 with sequence parallelism, a 9|15|15|13 split of the 52 decoder layers, MBS2/GBS1024 at sequence length 4096, and four data-loader workers, the command trains the complete randomly initialized HybridModel for exactly 100 optimizer steps on the revision-pinned public CORD-v2 multimodal dataset. All losses and gradient norms remain finite, with zero skipped and zero NaN iterations. Loss decreases from 12.23791 to 0.6403766; steps 91-100 average 105,995.34 ms, 205.59 model TFLOP/s/GPU, and 1,236.583 token slots/s/GPU. The run used a NeMo 26.10 release-candidate base container with the pinned Megatron-Core submodule. The persisted post-setup config records the exact runtime. Complete step-50 and step-100 checkpoints each contain 32 nonempty distributed shards and 1,566 state entries, including model, optimizer, scheduler, and RNG state, plus metadata, train state, and the native HybridModel run config. Data-loader worker state is not checkpointed, so a resume from step 50 is not guaranteed to replay the identical sample stream.

@@ -310,8 +305,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 32 H100 GPUs at TP8/PP2/DP2, the command loads the complete imported HybridModel and performs exactly 100 full-model image-conditioned SFT optimizer steps on the revision-pinned public CORD-v2 dataset. All losses and gradient norms remain finite, with zero skipped and zero NaN iterations. Loss decreases from 0.9772241 to 0.02695329; steps 91-100 average 1,826.64 ms, 94.03 model TFLOP/s/GPU, and 560.592 token slots/s/GPU. The complete 32-shard step-100 full-model checkpoint, metadata, train state, and resolved config are saved and reloadable. -

+

On 32 H100 GPUs at TP8/PP2/DP2, the command loads the complete imported HybridModel and performs exactly 100 full-model image-conditioned SFT optimizer steps on the revision-pinned public CORD-v2 dataset. All losses and gradient norms remain finite, with zero skipped and zero NaN iterations. Loss decreases from 0.9772241 to 0.02695329; steps 91-100 average 1,826.64 ms, 94.03 model TFLOP/s/GPU, and 560.592 token slots/s/GPU. The complete 32-shard step-100 full-model checkpoint, metadata, train state, and resolved config are saved and reloadable.

@@ -361,8 +355,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 32 H100 GPUs, the command loads the complete imported HybridModel and completes exactly 100 full-model SFT optimizer steps at sequence length 8192 with in-batch packing, TP1/PP4/CP2, A2A context-parallel communication, and selective core-attention recompute. Every loss and gradient norm is finite, with zero skipped and zero NaN iterations. Loss decreases from 1.034497 to 0.03382061; steps 91-100 average 1,419.91 ms, 16.07 model TFLOP/s/GPU, and 1,442.345 token slots/s/GPU. The complete step-100 full-model checkpoint, metadata, train state, and resolved config are saved and reloadable. -

+

On 32 H100 GPUs, the command loads the complete imported HybridModel and completes exactly 100 full-model SFT optimizer steps at sequence length 8192 with in-batch packing, TP1/PP4/CP2, A2A context-parallel communication, and selective core-attention recompute. Every loss and gradient norm is finite, with zero skipped and zero NaN iterations. Loss decreases from 1.034497 to 0.03382061; steps 91-100 average 1,419.91 ms, 16.07 model TFLOP/s/GPU, and 1,442.345 token slots/s/GPU. The complete step-100 full-model checkpoint, metadata, train state, and resolved config are saved and reloadable.

@@ -412,8 +405,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command loads the complete imported native HybridModel checkpoint, freezes the base model, and inserts LoRA dim 8 / alpha 16 adapters only into the native linear_qkv and linear_proj attention projections. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 0.7263421 to 0.03298394; steps 91-100 average 4,823.07 ms, 283.36 model TFLOP/s/GPU, and 1,698.503 token slots/s/GPU. The persisted post-setup config records the exact runtime. The complete eight-shard iter_0000100 checkpoint contains all 208 expected adapter tensors, its run config, metadata, and train state. A direct reload restores model and optimizer at step 100, completes finite step 101 with zero skipped or NaN iterations, and saves a distinct complete eight-shard checkpoint. -

+

The command loads the complete imported native HybridModel checkpoint, freezes the base model, and inserts LoRA dim 8 / alpha 16 adapters only into the native linear_qkv and linear_proj attention projections. All 100 optimizer-step rows are present exactly once with finite loss, zero skipped iterations, and zero NaN iterations. Loss decreases from 0.7263421 to 0.03298394; steps 91-100 average 4,823.07 ms, 283.36 model TFLOP/s/GPU, and 1,698.503 token slots/s/GPU. The persisted post-setup config records the exact runtime. The complete eight-shard iter_0000100 checkpoint contains all 208 expected adapter tensors, its run config, metadata, and train state. A direct reload restores model and optimizer at step 100, completes finite step 101 with zero skipped or NaN iterations, and saves a distinct complete eight-shard checkpoint.

@@ -463,11 +455,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The exact 32-H100 command completes all 50 fixed-shape mock-token dense decoder steps at TP4/PP4/CP2, a 9|15|15|13 split of all 52 language layers, sequence length 4096, MBS6/GBS192, BF16 model and gradient state, and FP8 current-scaling Transformer GEMMs. The vision and projection towers are frozen and excluded from the fixed-shape decoder workload. Loss remains finite from 12.26882 to 2.442728 with zero skipped or NaN iterations. Steps 41-50 average 9,373.06 ms, 428.24 model TFLOP/s/GPU, and 2,621.983 token slots/s/GPU. The persisted post-setup config records the exact runtime, and the process exits successfully without checkpoint output. -

+

The exact 32-H100 command completes all 50 fixed-shape mock-token dense decoder steps at TP4/PP4/CP2, a 9|15|15|13 split of all 52 language layers, sequence length 4096, MBS6/GBS192, BF16 model and gradient state, and FP8 current-scaling Transformer GEMMs. The vision and projection towers are frozen and excluded from the fixed-shape decoder workload. Loss remains finite from 12.26882 to 2.442728 with zero skipped or NaN iterations. Steps 41-50 average 9,373.06 ms, 428.24 model TFLOP/s/GPU, and 2,621.983 token slots/s/GPU. The persisted post-setup config records the exact runtime, and the process exits successfully without checkpoint output.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron-3-omni.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron-3-omni.mdx index d5e0ab0583..bf4287309f 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron-3-omni.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron-3-omni.mdx @@ -25,7 +25,7 @@ For the full setup, conversion, inference, training, evaluation, and LoRA merge / adapter export workflows, see [`examples/models/nemotron/nemotron_3_omni/README.md`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/models/nemotron/nemotron_3_omni/README.md). - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -143,8 +143,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The offline, immutable-revision CPU import exits successfully after mapping 7,333 model parameters and creates a reloadable iter_0000000 torch_dist checkpoint with 33,015,546,816 parameters on the single model parallel rank. -

+

The offline, immutable-revision CPU import exits successfully after mapping 7,333 model parameters and creates a reloadable iter_0000000 torch_dist checkpoint with 33,015,546,816 parameters on the single model parallel rank.

@@ -169,8 +168,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully at TP2/PP1/EP4/ETP1 and creates a reloadable iter_0000000 checkpoint. The paired strict GPU export contains the source checkpoint's exact 7,349-key set, shapes, dtypes, and values. -

+

The command exits successfully at TP2/PP1/EP4/ETP1 and creates a reloadable iter_0000000 checkpoint. The paired strict GPU export contains the source checkpoint's exact 7,349-key set, shapes, dtypes, and values.

@@ -195,8 +193,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU export completes in 14 indexed shards totaling 66,031,270,520 bytes. Its exact comparison contains all 7,349 source tensors and 33,015,632,238 values, with identical keys, shapes, dtypes, and values and no dtype conversions. Transformers reloads the local custom-code export natively with the source-equivalent warning set: one generated vision summary buffer is missing and four source-only audio/vision buffers are unexpected. -

+

CPU export completes in 14 indexed shards totaling 66,031,270,520 bytes. Its exact comparison contains all 7,349 source tensors and 33,015,632,238 values, with identical keys, shapes, dtypes, and values and no dtype conversions. Transformers reloads the local custom-code export natively with the source-equivalent warning set: one generated vision summary buffer is missing and four source-only audio/vision buffers are unexpected.

@@ -221,8 +218,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed export completes in 17 indexed shards totaling 66,031,270,520 bytes. All 7,349 tensors and 33,015,632,238 values match the immutable HF source exactly in keys, shapes, dtypes, and values with no dtype conversions. Transformers reloads the local custom-code export natively with the same one missing generated buffer and four unexpected source-only buffers reported when reloading the pinned source itself. -

+

Distributed export completes in 17 indexed shards totaling 66,031,270,520 bytes. All 7,349 tensors and 33,015,632,238 values match the immutable HF source exactly in keys, shapes, dtypes, and values with no dtype conversions. Transformers reloads the local custom-code export natively with the same one missing generated buffer and four unexpected source-only buffers reported when reloading the pinned source itself.

@@ -266,8 +262,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Megatron Bridge does not publish a pretraining recipe for this multimodal conditional-generation model; the supported package provides supervised CORD v2 SFT and LoRA PEFT workflows. -

+

Megatron Bridge does not publish a pretraining recipe for this multimodal conditional-generation model; the supported package provides supervised CORD v2 SFT and LoRA PEFT workflows.

@@ -312,13 +307,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_sft_8gpu_h100_bf16_config --mode sft --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-4k-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678
+ 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_sft_8gpu_h100_bf16_config --mode sft --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-4k-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678
Expected result
-

The immutable-revision, real image-text CORD v2 run completes exactly 100 full-SFT optimizer steps on eight H100 GPUs at TP2/PP1/CP1/EP8/ETP1 and GBS/MBS 64/4 with natural-routing HybridEP. Execution uses selective MoE/layernorm recompute, fused attention, TE cross entropy, fused weighted squared-ReLU, and router fusion without changing the image-text objective or routing policy. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.281308 to 0.03330838, the final ten steps average 3,603.67 ms and 197.10 TFLOP/s/GPU, sampled peak memory is 76,349 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 at 197.07 final-ten TFLOP/s/GPU with finite loss and zero skipped or NaN iterations. -

+

The immutable-revision, real image-text CORD v2 run completes exactly 100 full-SFT optimizer steps on eight H100 GPUs at TP2/PP1/CP1/EP8/ETP1 and GBS/MBS 64/4 with natural-routing HybridEP. Execution uses selective MoE/layernorm recompute, fused attention, TE cross entropy, fused weighted squared-ReLU, and router fusion without changing the image-text objective or routing policy. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.281308 to 0.03330838, the final ten steps average 3,603.67 ms and 197.10 TFLOP/s/GPU, sampled peak memory is 76,349 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 at 197.07 final-ten TFLOP/s/GPU with finite loss and zero skipped or NaN iterations.

@@ -363,13 +357,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_long_context_sft_8gpu_h100_bf16_config --mode sft --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-8k-packed-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678
+ 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_long_context_sft_8gpu_h100_bf16_config --mode sft --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/sft-8k-packed-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678
Expected result
-

The immutable-revision, real image-text CORD v2 run completes exactly 100 full-SFT optimizer steps on eight H100 GPUs at TP4/PP1/CP2/EP8/ETP1 and GBS/MBS 64/2 with 8K in-batch packing and the standard all-to-all dispatcher. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.251364 to 0.01020405, the final ten steps average 15,577.9 ms and 93.73 TFLOP/s/GPU, sampled peak memory is 75,197 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 with finite loss and zero skipped or NaN iterations. -

+

The immutable-revision, real image-text CORD v2 run completes exactly 100 full-SFT optimizer steps on eight H100 GPUs at TP4/PP1/CP2/EP8/ETP1 and GBS/MBS 64/2 with 8K in-batch packing and the standard all-to-all dispatcher. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.251364 to 0.01020405, the final ten steps average 15,577.9 ms and 93.73 TFLOP/s/GPU, sampled peak memory is 75,197 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 with finite loss and zero skipped or NaN iterations.

@@ -414,16 +407,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_peft_8gpu_h100_bf16_config --mode lora --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/peft-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678
+ 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_omni_cord_v2_peft_8gpu_h100_bf16_config --mode lora --step_func nemotron_omni_step --pretrained_checkpoint work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/gpu-megatron-clean/iter_0000000 --max_steps 100 --save_dir work/model-verification/nemotron-3-nano-omni-30b-a3b-reasoning/peft-checkpoints --save_interval 50 'dataset.source.load_kwargs={revision:"7f0115a4b758a71d6473b8d085751692da2fef98"}' dataset.do_validation=false dataset.do_test=false validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null logger.log_interval=1 logger.log_throughput=true rng.seed=5678
Expected result
-

The immutable-revision, real image-text CORD v2 run completes exactly 100 LoRA optimizer steps on eight H100 GPUs at TP2/PP1/CP1/EP8/ETP1 and GBS/MBS 64/1 with natural-routing HybridEP. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.282941 to 0.02634745, the final ten steps average 11,621.42 ms and 61.25 TFLOP/s/GPU, sampled peak memory is 44,873 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 with finite loss and zero skipped or NaN iterations. -

+

The immutable-revision, real image-text CORD v2 run completes exactly 100 LoRA optimizer steps on eight H100 GPUs at TP2/PP1/CP1/EP8/ETP1 and GBS/MBS 64/1 with natural-routing HybridEP. Model parameters, main gradients, and Adam moments use BF16; optimizer main parameters use scaled FP16 without parameter remainders. LM loss is finite from 1.282941 to 0.02634745, the final ten steps average 11,621.42 ms and 61.25 TFLOP/s/GPU, sampled peak memory is 44,873 MiB, and no iteration is skipped or NaN. Complete optimizer and RNG checkpoints are saved at steps 50 and 100. Direct step-50 resume loads the complete state and reaches step 100 with finite loss and zero skipped or NaN iterations.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano-4b.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano-4b.mdx index 7c73ffb46c..e0c5357fcb 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano-4b.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano-4b.mdx @@ -13,7 +13,7 @@ We use the following environment variables throughout this page - `MEGATRON_MODEL_PATH=/models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16` (feel free to set your own path) ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -131,8 +131,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully and creates iter_0000000. After the paired CPU export, all 263 tensors and 3,973,556,832 parameters match the pinned source exactly in name, shape, dtype, and value, with maximum difference zero. -

+

The command exits successfully and creates iter_0000000. After the paired CPU export, all 263 tensors and 3,973,556,832 parameters match the pinned source exactly in name, shape, dtype, and value, with maximum difference zero.

@@ -157,8 +156,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully and creates a reloadable iter_0000000. After the paired GPU export, all 263 tensors and 3,973,556,832 parameters match the pinned source exactly in name, shape, dtype, and value. -

+

The command exits successfully and creates a reloadable iter_0000000. After the paired GPU export, all 263 tensors and 3,973,556,832 parameters match the pinned source exactly in name, shape, dtype, and value.

@@ -183,8 +181,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict export exits successfully; all 263 BF16 tensors match the pinned source bitwise, the 171-byte generation configuration is preserved byte-for-byte, and Transformers reloads the output natively as NemotronHForCausalLM. -

+

Strict export exits successfully; all 263 BF16 tensors match the pinned source bitwise, the 171-byte generation configuration is preserved byte-for-byte, and Transformers reloads the output natively as NemotronHForCausalLM.

@@ -209,8 +206,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict export exits successfully; all 263 BF16 tensors match the pinned source bitwise, the generation configuration is exact, and Transformers reloads the output natively as NemotronHForCausalLM. A_log, D, dt_bias, and out_proj are exact for all 21 Mamba layers. -

+

Strict export exits successfully; all 263 BF16 tensors match the pinned source bitwise, the generation configuration is exact, and Transformers reloads the output natively as NemotronHForCausalLM. A_log, D, dt_bias, and out_proj are exact for all 21 Mamba layers.

@@ -260,8 +256,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The uninterrupted bounded random-initialization RP2 reference finishes all 100 steps at recipe-owned GBS/MBS 1024/1 with the five recorded metrics, finite loss, and no skipped or NaN iterations. It reaches peak learning rate 3e-4 at step 40 and completes cosine decay to 3e-5 at step 100. Both iter_0000050 and iter_0000100 contain all eight distributed shards plus metadata, optimizer/RNG train state, run config, and tokenizer; iter_0000100 reloads at step 100 without an extra optimizer step. -

+

The uninterrupted bounded random-initialization RP2 reference finishes all 100 steps at recipe-owned GBS/MBS 1024/1 with the five recorded metrics, finite loss, and no skipped or NaN iterations. It reaches peak learning rate 3e-4 at step 40 and completes cosine decay to 3e-5 at step 100. Both iter_0000050 and iter_0000100 contain all eight distributed shards plus metadata, optimizer/RNG train state, run config, and tokenizer; iter_0000100 reloads at step 100 without an extra optimizer step.

@@ -306,13 +301,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:1}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:1}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Pad-1 offline packing is 99.27% efficient. The TP1/DP8 run uses four microbatches per optimizer step and completes all 100 steps with the four recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, assistant-only loss masks contain 4,187,630 supervised tokens. The complete eight-shard iter_0000100 full-model checkpoint is saved and reloads for export. -

+

Pad-1 offline packing is 99.27% efficient. The TP1/DP8 run uses four microbatches per optimizer step and completes all 100 steps with the four recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, assistant-only loss masks contain 4,187,630 supervised tokens. The complete eight-shard iter_0000100 full-model checkpoint is saved and reloads for export.

@@ -357,13 +351,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_sft_8gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 32768 --tensor_model_parallel_size 2 --context_parallel_size 2 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-long-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.calculate_per_token_loss=true ddp.average_in_collective=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/long-sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_sft_8gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 32768 --tensor_model_parallel_size 2 --context_parallel_size 2 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-long-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.calculate_per_token_loss=true ddp.average_in_collective=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/long-sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Pad-4 offline packing is 99.35% efficient. The TP2/CP2/SP-on/DP2 run uses four microbatches per optimizer step and completes all 100 32K steps with the five recorded metrics, finite loss, and no skipped or NaN iterations. Across 26,214,400 token slots, assistant-only loss masks contain 16,734,347 supervised tokens. The complete eight-shard iter_0000100 checkpoint is saved and reloads at the same topology. -

+

Pad-4 offline packing is 99.35% efficient. The TP2/CP2/SP-on/DP2 run uses four microbatches per optimizer step and completes all 100 32K steps with the five recorded metrics, finite loss, and no skipped or NaN iterations. Across 26,214,400 token slots, assistant-only loss masks contain 16,734,347 supervised tokens. The complete eight-shard iter_0000100 checkpoint is saved and reloads at the same topology.

@@ -408,16 +401,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_peft_8gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:4}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --wait --nodes 1 --gpus-per-node 8 --recipe nemotron_3_nano_4b_peft_8gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-nano-4b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' dataset.hf_output_root=work/data/tulu3/nemotron-3-nano-4b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:4}' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-nano-4b/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Pad-4 offline packing is 99.36% efficient. The frozen-base LoRA run uses rank 8, alpha 16, zero dropout, and linear_qkv/linear_proj targets. Its TP1/DP8 layout uses four microbatches per optimizer step and completes all 100 steps with finite loss and no skipped or NaN iterations. Across 6,553,600 token slots, assistant-only loss masks contain 4,178,503 supervised tokens. The complete eight-shard iter_0000100 adapter checkpoint reloads over the pinned base model at step 100. -

+

Pad-4 offline packing is 99.36% efficient. The frozen-base LoRA run uses rank 8, alpha 16, zero dropout, and linear_qkv/linear_proj targets. Its TP1/DP8 layout uses four microbatches per optimizer step and completes all 100 steps with finite loss and no skipped or NaN iterations. Across 6,553,600 token slots, assistant-only loss masks contain 4,178,503 supervised tokens. The complete eight-shard iter_0000100 adapter checkpoint reloads over the pinned base model at step 100.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano.mdx index fdab91b128..5771a75ffc 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-nano.mdx @@ -13,7 +13,7 @@ We use the following environment variables throughout this page - `MEGATRON_MODEL_PATH=/models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16` (feel free to set your own path) ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -133,8 +133,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates. -

+

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.

@@ -159,8 +158,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned one-node 8-H100 import exits successfully at TP1/PP1/EP8/ETP1, persists iter_0000000, and reloads it for inference. A separate public GPU round-trip at the same topology exhaustively compares all 6,243 exported BF16 weights against the immutable source: 6,243 of 6,243 match with no skipped FP8 tensors or mismatches. -

+

The pinned one-node 8-H100 import exits successfully at TP1/PP1/EP8/ETP1, persists iter_0000000, and reloads it for inference. A separate public GPU round-trip at the same topology exhaustively compares all 6,243 exported BF16 weights against the immutable source: 6,243 of 6,243 match with no skipped FP8 tensors or mismatches.

@@ -185,8 +183,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The synchronous CPU export writes a complete indexed checkpoint. All 6,243 source keys and shapes match across 31,577,940,288 values. The 46 source FP32 tensors cast to BF16 are the 23 per-layer mixer A_log tensors and 23 per-layer mixer D tensors; each equals the export exactly after that declared cast, and every remaining tensor matches exactly at zero tolerance. Native loading succeeds as NemotronHForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export. -

+

The synchronous CPU export writes a complete indexed checkpoint. All 6,243 source keys and shapes match across 31,577,940,288 values. The 46 source FP32 tensors cast to BF16 are the 23 per-layer mixer A_log tensors and 23 per-layer mixer D tensors; each equals the export exactly after that declared cast, and every remaining tensor matches exactly at zero tolerance. Native loading succeeds as NemotronHForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export.

@@ -211,8 +208,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The synchronous TP1/PP1/EP8/ETP1 export writes a complete indexed checkpoint. All 6,243 source keys and shapes match across 31,577,940,288 values. The 69 source FP32 tensors cast to BF16 are the same 46 mixer A_log and D tensors plus 23 per-layer mixer gate e_score_correction_bias tensors. The legacy distributed GPU export model wrapper downcast those gate biases to BF16, while the CPU export preserved them in FP32; all 69 equal the export exactly after this declared cast, and every remaining tensor matches exactly at zero tolerance. Native loading succeeds as NemotronHForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export. -

+

The synchronous TP1/PP1/EP8/ETP1 export writes a complete indexed checkpoint. All 6,243 source keys and shapes match across 31,577,940,288 values. The 69 source FP32 tensors cast to BF16 are the same 46 mixer A_log and D tensors plus 23 per-layer mixer gate e_score_correction_bias tensors. The legacy distributed GPU export model wrapper downcast those gate biases to BF16, while the CPU export preserved them in FP32; all 69 equal the export exactly after this declared cast, and every remaining tensor matches exactly at zero tolerance. Native loading succeeds as NemotronHForCausalLM with no missing, unexpected, mismatched, or errored weights, and the tokenizer reloads from the export.

@@ -256,8 +252,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -301,8 +296,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -346,8 +340,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -391,8 +384,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -442,11 +434,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 8 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP8/ETP1, GBS/MBS 512/4, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.18025 to 0.004964447; the final ten steps average 12455.140 ms, 937.770 TFLOP/s/GPU, and 42094.107 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 8 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP8/ETP1, GBS/MBS 512/4, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.18025 to 0.004964447; the final ten steps average 12455.140 ms, 937.770 TFLOP/s/GPU, and 42094.107 tokens/s/GPU. The resolved configuration persists.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-super.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-super.mdx index b8809f09e7..1dbbc33703 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-super.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-super.mdx @@ -7,7 +7,7 @@ NeMo Megatron Bridge supports pretraining, full parameters finetuning, and LoRA Run all commands from `/opt/Megatron-Bridge` (e.g. `docker run -w /opt/Megatron-Bridge ...`) ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -176,8 +176,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned CPU import must exit successfully, create iter_0000000, and preserve the complete source checkpoint. A subsequent CPU export from that CPU-imported checkpoint must audit every supported tensor for exact keys, shapes, dtypes, and values; the currently verified CPU export uses the independently verified GPU-imported checkpoint instead. -

+

The pinned CPU import must exit successfully, create iter_0000000, and preserve the complete source checkpoint. A subsequent CPU export from that CPU-imported checkpoint must audit every supported tensor for exact keys, shapes, dtypes, and values; the currently verified CPU export uses the independently verified GPU-imported checkpoint instead.

@@ -202,8 +201,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned eight-GPU BF16 import exits successfully and creates a reloadable TP1/PP1/EP8/ETP1 iter_0000000 checkpoint. Its paired export exactly reproduces all 42,683 source tensors and 123,611,033,088 elements, including the 41 FP32 router correction-bias tensors. -

+

The pinned eight-GPU BF16 import exits successfully and creates a reloadable TP1/PP1/EP8/ETP1 iter_0000000 checkpoint. Its paired export exactly reproduces all 42,683 source tensors and 123,611,033,088 elements, including the 41 FP32 router correction-bias tensors.

@@ -228,8 +226,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The single-process CPU export exits successfully and writes 247,222,108,160 bytes across 42,683 tensors and 123,611,033,088 values. Every exported key, shape, dtype, and value is bitwise identical to the pinned Hugging Face BF16 source, with no dtype conversions, and Transformers strictly reloads all 763 state tensors natively as NemotronHForCausalLM. -

+

The single-process CPU export exits successfully and writes 247,222,108,160 bytes across 42,683 tensors and 123,611,033,088 values. Every exported key, shape, dtype, and value is bitwise identical to the pinned Hugging Face BF16 source, with no dtype conversions, and Transformers strictly reloads all 763 state tensors natively as NemotronHForCausalLM.

@@ -254,8 +251,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict distributed export exits successfully and writes 247,222,108,160 bytes across all 42,683 tensors. Keys, shapes, dtypes, values, and critical configuration match the pinned source with zero mismatches, including 41 FP32 router correction-bias tensors, and Transformers reloads the result natively as NemotronHForCausalLM with 88 layers. -

+

Strict distributed export exits successfully and writes 247,222,108,160 bytes across all 42,683 tensors. Keys, shapes, dtypes, values, and critical configuration match the pinned source with zero mismatches, including 41 FP32 router correction-bias tensors, and Transformers reloads the result natively as NemotronHForCausalLM with 88 layers.

@@ -305,8 +301,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 64 H100s, the BF16 support run completes 100 bounded RP2 optimizer steps at TP1/PP2/CP1/EP32/ETP1, GBS/MBS 1280/1, natural routing, HybridEP flex dispatch with 1.10x fixed expert capacity, eager execution, selective recompute of layernorm, moe_act, moe, and core_attn, no optimizer-state CPU offload, and ordered gradient reduction. Loss remains finite from 12.19922 to 5.706429 with zero skipped or NaN iterations. The final ten steps average 29368.690 ms and 236.330 TFLOP/s/GPU. Peak device memory is 58.079 GiB allocated and 70.336 GiB reserved. The resolved configuration persists, and complete iter_0000050 and iter_0000100 checkpoints each contain 64 distributed rank shards plus checkpoint metadata and training state. -

+

On exactly 64 H100s, the BF16 support run completes 100 bounded RP2 optimizer steps at TP1/PP2/CP1/EP32/ETP1, GBS/MBS 1280/1, natural routing, HybridEP flex dispatch with 1.10x fixed expert capacity, eager execution, selective recompute of layernorm, moe_act, moe, and core_attn, no optimizer-state CPU offload, and ordered gradient reduction. Loss remains finite from 12.19922 to 5.706429 with zero skipped or NaN iterations. The final ten steps average 29368.690 ms and 236.330 TFLOP/s/GPU. Peak device memory is 58.079 GiB allocated and 70.336 GiB reserved. The resolved configuration persists, and complete iter_0000050 and iter_0000100 checkpoints each contain 64 distributed rank shards plus checkpoint metadata and training state.

@@ -356,8 +351,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 16 GB200s, the BF16 support run must complete exactly 100 bounded RP2 optimizer steps at TP8/PP1/CP1/EP16/ETP1 and the recipe's cohort GBS/MBS 1024/1. Full activation recompute, 40% optimizer CPU offload, and eager execution keep the model and optimizer checkpoint within memory. Loss must stay finite with no skipped or NaN iteration, and complete iter_0000050 and iter_0000100 checkpoints must save. The recipe moved from GBS 16 to the cohort GBS 1024, so the earlier metrics no longer describe this command and it needs a rerun. -

+

On 16 GB200s, the BF16 support run must complete exactly 100 bounded RP2 optimizer steps at TP8/PP1/CP1/EP16/ETP1 and the recipe's cohort GBS/MBS 1024/1. Full activation recompute, 40% optimizer CPU offload, and eager execution keep the model and optimizer checkpoint within memory. Loss must stay finite with no skipped or NaN iteration, and complete iter_0000050 and iter_0000100 checkpoints must save. The recipe moved from GBS 16 to the cohort GBS 1024, so the earlier metrics no longer describe this command and it needs a rerun.

@@ -402,13 +396,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.75 optimizer.overlap_cpu_optimizer_d2h_h2d=true checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-sft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.75 optimizer.overlap_cpu_optimizer_d2h_h2d=true checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-sft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

The immutable-revision Tulu3 run must complete exactly 100 full-SFT steps on 16 H100s at sequence length 8192, TP8/EP16, the recipe's cohort GBS/MBS 32/1, and pad-8 offline packing. Loss must stay finite with no skipped or NaN iteration, and the step-100 checkpoint must save completely for export. The recipe moved from GBS 16 to the cohort GBS 32, so the earlier metrics no longer describe this command and it needs a rerun. -

+

The immutable-revision Tulu3 run must complete exactly 100 full-SFT steps on 16 H100s at sequence length 8192, TP8/EP16, the recipe's cohort GBS/MBS 32/1, and pad-8 offline packing. Loss must stay finite with no skipped or NaN iteration, and the step-100 checkpoint must save completely for export. The recipe moved from GBS 16 to the cohort GBS 32, so the earlier metrics no longer describe this command and it needs a rerun.

@@ -453,13 +446,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.75 optimizer.overlap_cpu_optimizer_d2h_h2d=true checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-sft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=0.75 optimizer.overlap_cpu_optimizer_d2h_h2d=true checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-sft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

The immutable-revision Tulu3 run must complete exactly 100 full-SFT steps on 16 GB200s at sequence length 8192, TP8/EP16, the recipe's cohort GBS/MBS 32/1, and pad-8 offline packing. Loss must stay finite with no skipped or NaN iteration, and the step-100 checkpoint must save completely. The recipe moved from GBS 16 to the cohort GBS 32, so the earlier metrics no longer describe this command and it needs a rerun. -

+

The immutable-revision Tulu3 run must complete exactly 100 full-SFT steps on 16 GB200s at sequence length 8192, TP8/EP16, the recipe's cohort GBS/MBS 32/1, and pad-8 offline packing. Loss must stay finite with no skipped or NaN iteration, and the step-100 checkpoint must save completely. The recipe moved from GBS 16 to the cohort GBS 32, so the earlier metrics no longer describe this command and it needs a rerun.

@@ -504,13 +496,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 10 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-long dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.attention_backend=auto model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=1.0 optimizer.overlap_cpu_optimizer_d2h_h2d=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-long-sft --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 10 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-long dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.attention_backend=auto model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 optimizer.optimizer_cpu_offload=true optimizer.optimizer_offload_fraction=1.0 optimizer.overlap_cpu_optimizer_d2h_h2d=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-long-sft --save_interval 10 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

On 16 H100s, the dedicated TP1/PP8/CP2/EP2 recipe completes exactly ten packed 32K SFT steps with pad-4 alignment, full recompute, and full optimizer CPU offload. Loss is finite from 1.168852 to 1.008992 with no skipped or NaN iterations. The ten-step metric window includes the first-step cold-start cost by policy, the resolved configuration is persisted, and the final model-only checkpoint saves successfully. -

+

On 16 H100s, the dedicated TP1/PP8/CP2/EP2 recipe completes exactly ten packed 32K SFT steps with pad-4 alignment, full recompute, and full optimizer CPU offload. Loss is finite from 1.168852 to 1.008992 with no skipped or NaN iterations. The ten-step metric window includes the first-step cold-start cost by policy, the resolved configuration is persisted, and the final model-only checkpoint saves successfully.

@@ -555,13 +546,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 10 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-long-sft --save_interval 10 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 10 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-long-sft --save_interval 10 logger.log_interval=1 logger.log_throughput=true
Expected result
-

A dedicated GB200 run must complete at least ten packed 32K SFT steps with CP2, finite loss, no skipped or NaN iterations, all five metrics, a persisted resolved configuration, and a reloadable final checkpoint. -

+

A dedicated GB200 run must complete at least ten packed 32K SFT steps with CP2, finite loss, no skipped or NaN iterations, all five metrics, a persisted resolved configuration, and a reloadable final checkpoint.

@@ -606,13 +596,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-peft-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=selective 'model.recompute_modules=[core_attn]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-peft --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-peft-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=selective 'model.recompute_modules=[core_attn]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/h100-peft --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

H100 training has completed 100 finite LoRA steps and saved an adapter, but this leaf remains unverified until the exported adapter reloads natively over the pinned Hugging Face base and produces a nonzero, finite logits effect. -

+

H100 training has completed 100 finite LoRA steps and saved an adapter, but this leaf remains unverified until the exported adapter reloads natively over the pinned Hugging Face base and produces a nonzero, finite logits effect.

@@ -657,13 +646,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-peft-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=selective 'model.recompute_modules=[core_attn]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-peft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/nemotron-3-super-120b-a12b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 8192 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="2d59de1cbd51c0adf384eb906b766d1aee0e0517"' dataset.hf_output_root=work/data/tulu3/nemotron-3-super-120b-a12b-peft-pad8 dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:8192,pad_seq_to_mult:8}' model.recompute_granularity=selective 'model.recompute_modules=[core_attn]' scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.ckpt_format=torch_dist checkpoint.async_save=false checkpoint.load=null --save_dir work/model-verification/nemotron-3-super-120b-a12b/gb200-peft --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

On 16 GB200s, the immutable-revision Tulu3 run completes exactly 100 pad-8 packed LoRA steps with finite loss from 1.23861 to 0.8878498, no skipped or NaN iterations, all five metrics, and a final adapter checkpoint. The adapter contains only 82,304 LoRA tensors targeting linear_qkv, linear_proj, linear_fc1, linear_fc2, in_proj, and out_proj. Native PEFT reload over the pinned base succeeds; all logits are finite and the adapter changes them by maximum absolute 3.1875 and mean absolute 0.472428. -

+

On 16 GB200s, the immutable-revision Tulu3 run completes exactly 100 pad-8 packed LoRA steps with finite loss from 1.23861 to 0.8878498, no skipped or NaN iterations, all five metrics, and a final adapter checkpoint. The adapter contains only 82,304 LoRA tensors targeting linear_qkv, linear_proj, linear_fc1, linear_fc2, in_proj, and out_proj. Native PEFT reload over the pinned base succeeds; all logits are finite and the adapter changes them by maximum absolute 3.1875 and mean absolute 0.472428.

@@ -713,8 +701,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 64 H100s, the canonical BF16 performance recipe must complete 50 mock-data steps at TP1/PP2/CP1/EP32/ETP1, GBS/MBS 1280/1, forced routing, HybridEP flex dispatch with 1.10x fixed expert capacity, eager execution, selective recompute of layernorm, moe_act, moe, and core_attn, no optimizer-state CPU offload, and ordered gradient reduction. All keyed rows have finite loss and throughput with zero skipped or NaN iterations. The final ten steps average 27588.580 ms and 248.610 TFLOP/s/GPU, passing gates of at most 28000 ms and at least 240 TFLOP/s/GPU. Peak device memory is 59.000 GiB allocated and 69.344 GiB reserved, and the resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence. -

+

On exactly 64 H100s, the canonical BF16 performance recipe must complete 50 mock-data steps at TP1/PP2/CP1/EP32/ETP1, GBS/MBS 1280/1, forced routing, HybridEP flex dispatch with 1.10x fixed expert capacity, eager execution, selective recompute of layernorm, moe_act, moe, and core_attn, no optimizer-state CPU offload, and ordered gradient reduction. All keyed rows have finite loss and throughput with zero skipped or NaN iterations. The final ten steps average 27588.580 ms and 248.610 TFLOP/s/GPU, passing gates of at most 28000 ms and at least 240 TFLOP/s/GPU. Peak device memory is 59.000 GiB allocated and 69.344 GiB reserved, and the resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence.

@@ -764,8 +751,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 64 GB200s, the canonical BF16 performance recipe completes 50 mock-data steps at TP2/PP1/CP1/EP64, GBS/MBS 512/1, HybridEP flex dispatch, and Transformer Engine CUDA graph scopes attn, mamba, moe_router, and moe_preprocess. All 50 keyed rows have finite loss and throughput with zero skipped or NaN iterations. The final ten steps average 9926.350 ms and 559.390 TFLOP/s/GPU, passing gates of at most 11000 ms and at least 500 TFLOP/s/GPU, and the resolved configuration is persisted. Mock data and forced routing make the loss a finiteness check rather than convergence evidence. These numbers are GB200-only evidence. -

+

On exactly 64 GB200s, the canonical BF16 performance recipe completes 50 mock-data steps at TP2/PP1/CP1/EP64, GBS/MBS 512/1, HybridEP flex dispatch, and Transformer Engine CUDA graph scopes attn, mamba, moe_router, and moe_preprocess. All 50 keyed rows have finite loss and throughput with zero skipped or NaN iterations. The final ten steps average 9926.350 ms and 559.390 TFLOP/s/GPU, passing gates of at most 11000 ms and at least 500 TFLOP/s/GPU, and the resolved configuration is persisted. Mock data and forced routing make the loss a finiteness check rather than convergence evidence. These numbers are GB200-only evidence.

@@ -815,11 +801,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 64 GB300s, the canonical NVFP4 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 512/1, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.17656 to 0.01398012; the final ten steps average 6357.940 ms, 873.330 TFLOP/s/GPU, and 10307.741 tokens/s/GPU. The resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence. -

+

On exactly 64 GB300s, the canonical NVFP4 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 512/1, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.17656 to 0.01398012; the final ten steps average 6357.940 ms, 873.330 TFLOP/s/GPU, and 10307.741 tokens/s/GPU. The resolved configuration persists. Mock data and forced routing make the loss a finiteness check rather than convergence evidence.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-ultra.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-ultra.mdx index ca59ad9dee..91ce1fb4a9 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-ultra.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3-ultra.mdx @@ -10,7 +10,7 @@ packed OpenMathInstruct-2 full SFT, and packed OpenMathInstruct-2 LoRA PEFT. Use the main example README for setup and scripts: [`examples/models/nemotron/nemotron_3/ultra/README.md`](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/models/nemotron/nemotron_3/ultra/README.md). - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -172,8 +172,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A machine with enough host memory must import the complete pinned BF16 checkpoint, persist iter_0000000, and reload it without missing, unexpected, shape-mismatched, dtype-mismatched, or value-mismatched tensors. -

+

A machine with enough host memory must import the complete pinned BF16 checkpoint, persist iter_0000000, and reload it without missing, unexpected, shape-mismatched, dtype-mismatched, or value-mismatched tensors.

@@ -198,8 +197,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exited successfully and persisted a reloadable iter_0000000 at TP1/PP3/EP8/ETP1. Exact distributed export back to the immutable source revision accounted for all 51,023 Nemotron-H, LatentMoE, Mamba, attention, and MTP tensors and 560,524,603,904 parameters with matching names, shapes, dtypes, and values. -

+

The command exited successfully and persisted a reloadable iter_0000000 at TP1/PP3/EP8/ETP1. Exact distributed export back to the immutable source revision accounted for all 51,023 Nemotron-H, LatentMoE, Mamba, attention, and MTP tensors and 560,524,603,904 parameters with matching names, shapes, dtypes, and values.

@@ -224,8 +222,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The single-process CPU export exits successfully and writes all 51,023 tensors, 560,524,603,904 values, and 1,121,049,257,984 tensor-payload bytes. Keys, shapes, dtypes, and values are bitwise identical to the pinned source with no dtype conversions, preserving its BF16 model weights and FP32 expert-score correction biases. Transformers strictly reloads the output natively as NemotronHForCausalLM with all 927 state tensors and no loading discrepancies. -

+

The single-process CPU export exits successfully and writes all 51,023 tensors, 560,524,603,904 values, and 1,121,049,257,984 tensor-payload bytes. Keys, shapes, dtypes, and values are bitwise identical to the pinned source with no dtype conversions, preserving its BF16 model weights and FP32 expert-score correction biases. Transformers strictly reloads the output natively as NemotronHForCausalLM with all 927 state tensors and no loading discrepancies.

@@ -250,8 +247,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exited successfully and wrote 225 source-native Hugging Face shards. All 51,023 tensors and 560,524,603,904 parameters matched the pinned source exactly in name, shape, dtype, and value, preserving BF16 model weights and FP32 expert-score correction biases. Semantic configuration, generation configuration, the 131,072-token vocabulary, special tokens, chat template, and representative encodings also matched, and the export reloaded natively with NemotronHConfig and architecture NemotronHForCausalLM. -

+

The command exited successfully and wrote 225 source-native Hugging Face shards. All 51,023 tensors and 560,524,603,904 parameters matched the pinned source exactly in name, shape, dtype, and value, preserving BF16 model weights and FP32 expert-score correction biases. Semantic configuration, generation configuration, the 131,072-token vocabulary, special tokens, chat template, and representative encodings also matched, and the export reloaded natively with NemotronHConfig and architecture NemotronHForCausalLM.

@@ -309,8 +305,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

In the card's current base container, the public BF16 library recipe must complete all 10 bounded RedPajama2 head_01 optimizer steps at TP4/PP1/CP1/EP64/ETP1, GBS/MBS 512/1, and sequence length 8192. Megatron-FSDP, natural expert routing, standard all-to-all dispatch, FP32 gradient communication, numerical checks, full-layer block recompute, and the library optimizer and constant-LR schedule must remain active. Every keyed step must have finite loss, throughput, and gradient norm with zero skipped and zero NaN iterations. -

+

In the card's current base container, the public BF16 library recipe must complete all 10 bounded RedPajama2 head_01 optimizer steps at TP4/PP1/CP1/EP64/ETP1, GBS/MBS 512/1, and sequence length 8192. Megatron-FSDP, natural expert routing, standard all-to-all dispatch, FP32 gradient communication, numerical checks, full-layer block recompute, and the library optimizer and constant-LR schedule must remain active. Every keyed step must have finite loss, throughput, and gradient norm with zero skipped and zero NaN iterations.

@@ -368,8 +363,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

In the card's current base container, the public checkpoint-initialized BF16 library recipe completed all 20 bounded RedPajama2 head_01 optimizer steps at TP4/PP4/CP1/EP16/ETP1, GBS/MBS 256/1, and sequence length 8192. Natural expert routing, non-FSDP distributed optimizer, HybridEP, targeted expert-activation offload, selective attention/layernorm recompute, numerical checks, and the library optimizer and constant-LR schedule remained active. Every keyed step had finite loss, throughput, and gradient norm with zero skipped and zero NaN iterations, and durable Weights & Biases telemetry was produced. -

+

In the card's current base container, the public checkpoint-initialized BF16 library recipe completed all 20 bounded RedPajama2 head_01 optimizer steps at TP4/PP4/CP1/EP16/ETP1, GBS/MBS 256/1, and sequence length 8192. Natural expert routing, non-FSDP distributed optimizer, HybridEP, targeted expert-activation offload, selective attention/layernorm recompute, numerical checks, and the library optimizer and constant-LR schedule remained active. Every keyed step had finite loss, throughput, and gradient norm with zero skipped and zero NaN iterations, and durable Weights & Biases telemetry was produced.

@@ -414,13 +408,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 24 --gpus-per-node 8 --recipe nemotron_3_ultra_sft_192gpu_h100_bf16_openmathinstruct2_packed_config --mode sft --wait --pretrained_checkpoint work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 scheduler.lr_decay_iters=100 'dataset.hf_dataset.split="train_1M[:10%]"' 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.hf_output_root=work/data/openmathinstruct2-ultra-4096-v1 dataset.seed=1234 dataset.offline_packing_specs.pad_seq_to_mult=2 validation.eval_iters=0 validation.eval_interval=0 --save_dir work/model-verification/nemotron-3-ultra-550b-a55b/sft-checkpoints --save_interval 100 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/sft-resolved.yaml logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 24 --gpus-per-node 8 --recipe nemotron_3_ultra_sft_192gpu_h100_bf16_openmathinstruct2_packed_config --mode sft --wait --pretrained_checkpoint work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 scheduler.lr_decay_iters=100 'dataset.hf_dataset.split="train_1M[:10%]"' 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.hf_output_root=work/data/openmathinstruct2-ultra-4096-v1 dataset.seed=1234 dataset.offline_packing_specs.pad_seq_to_mult=2 validation.eval_iters=0 validation.eval_interval=0 --save_dir work/model-verification/nemotron-3-ultra-550b-a55b/sft-checkpoints --save_interval 100 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/sft-resolved.yaml logger.tensorboard_dir=null
Expected result
-

The public 192-H100 packed OpenMathInstruct-2 full-SFT recipe must use the pinned dataset revision and bounded train_1M first-ten-percent selection, complete 100 optimizer steps at sequence length 4096 and GBS/MBS 128/1, and preserve pad_seq_to_mult 2. This published Ultra contract is support verification rather than the 8K/GBS8 qwen3_30b_a3b_convergence_v2 cohort. Loss must remain finite with no skipped or NaN iterations, all five metrics, a persisted post-setup configuration, audited supervised-token counts, and a complete reloadable final checkpoint. -

+

The public 192-H100 packed OpenMathInstruct-2 full-SFT recipe must use the pinned dataset revision and bounded train_1M first-ten-percent selection, complete 100 optimizer steps at sequence length 4096 and GBS/MBS 128/1, and preserve pad_seq_to_mult 2. This published Ultra contract is support verification rather than the 8K/GBS8 qwen3_30b_a3b_convergence_v2 cohort. Loss must remain finite with no skipped or NaN iterations, all five metrics, a persisted post-setup configuration, audited supervised-token counts, and a complete reloadable final checkpoint.

@@ -464,8 +457,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No GB200-specific Ultra full-SFT configuration is verified by this PR. The H100 library recipe must be adapted and validated independently before GB200 SFT support is claimed. -

+

No GB200-specific Ultra full-SFT configuration is verified by this PR. The H100 library recipe must be adapted and validated independently before GB200 SFT support is claimed.

@@ -509,8 +501,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Megatron Bridge does not publish a dedicated Ultra packed long-context recipe. The only published Ultra full-SFT recipe fixes sequence length 4096 and context parallelism one, so it cannot provide the required joint sequence-packing and CP-greater-than-one evidence and is not relabeled as long-context support. -

+

Megatron Bridge does not publish a dedicated Ultra packed long-context recipe. The only published Ultra full-SFT recipe fixes sequence length 4096 and context parallelism one, so it cannot provide the required joint sequence-packing and CP-greater-than-one evidence and is not relabeled as long-context support.

@@ -554,8 +545,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Megatron Bridge does not publish a dedicated Ultra packed long-context recipe for GB200. -

+

Megatron Bridge does not publish a dedicated Ultra packed long-context recipe for GB200.

@@ -600,13 +590,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe nemotron_3_ultra_peft_32gpu_h100_bf16_openmathinstruct2_packed_config --mode lora --wait --pretrained_checkpoint work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 scheduler.lr_decay_iters=100 'dataset.hf_dataset.split="train_1M[:10%]"' 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.hf_output_root=work/data/openmathinstruct2-ultra-4096-v1 dataset.seed=1234 dataset.offline_packing_specs.pad_seq_to_mult=2 validation.eval_iters=0 validation.eval_interval=0 --save_dir work/model-verification/nemotron-3-ultra-550b-a55b/peft-checkpoints --save_interval 100 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/peft-resolved.yaml logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe nemotron_3_ultra_peft_32gpu_h100_bf16_openmathinstruct2_packed_config --mode lora --wait --pretrained_checkpoint work/model-verification/nemotron-3-ultra-550b-a55b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 scheduler.lr_decay_iters=100 'dataset.hf_dataset.split="train_1M[:10%]"' 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.hf_output_root=work/data/openmathinstruct2-ultra-4096-v1 dataset.seed=1234 dataset.offline_packing_specs.pad_seq_to_mult=2 validation.eval_iters=0 validation.eval_interval=0 --save_dir work/model-verification/nemotron-3-ultra-550b-a55b/peft-checkpoints --save_interval 100 ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/nemotron-3-ultra-550b-a55b/peft-resolved.yaml logger.tensorboard_dir=null
Expected result
-

The public 32-H100 packed OpenMathInstruct-2 LoRA recipe must use the same pinned bounded dataset and pad_seq_to_mult 2 as full SFT, match all declared attention, MLP, and Mamba adapter targets, and complete 100 optimizer steps at sequence length 4096 and GBS/MBS 128/1. This published Ultra contract is support verification rather than the 8K/GBS8 qwen3_30b_a3b_convergence_v2 cohort. Loss must remain finite with no skipped or NaN iterations, all five metrics, audited supervised-token counts, a persisted post-setup configuration, and a reloadable adapter checkpoint. -

+

The public 32-H100 packed OpenMathInstruct-2 LoRA recipe must use the same pinned bounded dataset and pad_seq_to_mult 2 as full SFT, match all declared attention, MLP, and Mamba adapter targets, and complete 100 optimizer steps at sequence length 4096 and GBS/MBS 128/1. This published Ultra contract is support verification rather than the 8K/GBS8 qwen3_30b_a3b_convergence_v2 cohort. Loss must remain finite with no skipped or NaN iterations, all five metrics, audited supervised-token counts, a persisted post-setup configuration, and a reloadable adapter checkpoint.

@@ -650,8 +639,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No GB200-specific Ultra PEFT configuration is verified by this PR. -

+

No GB200-specific Ultra PEFT configuration is verified by this PR.

@@ -701,8 +689,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 256 GB200s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP2/PP1/CP1/EP64/ETP1, GBS/MBS 256/1, and sequence length 8192 using Megatron-FSDP HSDP. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.1668 to 6.738161; the final ten steps average 3213.970 ms, 917.750 TFLOP/s/GPU, and 2548.873 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 256 GB200s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP2/PP1/CP1/EP64/ETP1, GBS/MBS 256/1, and sequence length 8192 using Megatron-FSDP HSDP. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.1668 to 6.738161; the final ten steps average 3213.970 ms, 917.750 TFLOP/s/GPU, and 2548.873 tokens/s/GPU. The resolved configuration persists.

@@ -752,11 +739,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 256 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 256/1, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.20118 to 7.019335; the final ten steps average 2282.400 ms, 1306.850 TFLOP/s/GPU, and 3589.204 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 256 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP64/ETP1, GBS/MBS 256/1, and sequence length 8192. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.20118 to 7.019335; the final ten steps average 2282.400 ms, 1306.850 TFLOP/s/GPU, and 3589.204 tokens/s/GPU. The resolved configuration persists.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-lightning.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-lightning.mdx index 5afdb315a3..525a67e79c 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-lightning.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-lightning.mdx @@ -7,7 +7,7 @@ NeMo Megatron Bridge supports pretraining, full-parameter fine-tuning, and LoRA Run all commands from `/opt/Megatron-Bridge` (e.g. `docker run -w /opt/Megatron-Bridge ...`) ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -182,8 +182,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned checkpoint imports on CPU, creates a reloadable iter_0000000, and preserves every MTP tensor for the paired CPU export. After export, all 6,513 tensors and 32,913,266,240 parameters match the pinned source exactly in name, shape, dtype, and value. -

+

The pinned checkpoint imports on CPU, creates a reloadable iter_0000000, and preserves every MTP tensor for the paired CPU export. After export, all 6,513 tensors and 32,913,266,240 parameters match the pinned source exactly in name, shape, dtype, and value.

@@ -208,8 +207,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully and creates a reloadable iter_0000000. After the paired GPU export, all 6,513 tensors and 32,913,266,240 parameters match the pinned source exactly in name, shape, dtype, and value, including the MTP layers. -

+

The command exits successfully and creates a reloadable iter_0000000. After the paired GPU export, all 6,513 tensors and 32,913,266,240 parameters match the pinned source exactly in name, shape, dtype, and value, including the MTP layers.

@@ -234,8 +232,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict CPU export writes a native NemotronHForCausalLM checkpoint that reloads with Transformers. All 6,513 tensors and 32,913,266,240 parameters match the pinned source bitwise with maximum difference zero. -

+

Strict CPU export writes a native NemotronHForCausalLM checkpoint that reloads with Transformers. All 6,513 tensors and 32,913,266,240 parameters match the pinned source bitwise with maximum difference zero.

@@ -260,8 +257,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict export exits successfully; all 6,513 tensors match the pinned source bitwise with original dtypes preserved, maximum difference zero, and Transformers reloads the output natively as NemotronHForCausalLM. -

+

Strict export exits successfully; all 6,513 tensors match the pinned source bitwise with original dtypes preserved, maximum difference zero, and Transformers reloads the output natively as NemotronHForCausalLM.

@@ -319,8 +315,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two complete eight-H100 nodes, the real-data RP2 CP2/p2p run completes exactly 20 sequence-length-8,192, GBS/MBS 512/1 optimizer steps with natural routing and finite LM/MTP-1/MTP-2 losses. LM loss is 12.18927 -> 7.239111, steps 11-20 average 36,869.270 ms / 200.180 TFLOP/s/GPU, no iteration is skipped or NaN, and complete 20-file, 430-GiB checkpoints are written at steps 10 and 20. -

+

On two complete eight-H100 nodes, the real-data RP2 CP2/p2p run completes exactly 20 sequence-length-8,192, GBS/MBS 512/1 optimizer steps with natural routing and finite LM/MTP-1/MTP-2 losses. LM loss is 12.18927 -> 7.239111, steps 11-20 average 36,869.270 ms / 200.180 TFLOP/s/GPU, no iteration is skipped or NaN, and complete 20-file, 430-GiB checkpoints are written at steps 10 and 20.

@@ -378,8 +373,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192, GBS/MBS 512/2 real-data RP2 convergence-verification run completes exactly 100 finite-loss optimizer steps with natural routing, Transformer Engine fused cross entropy, CUDA graphs disabled, finite LM/MTP-1/MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.171970 -> 5.798433; real-data steps 91-100 average 21,483.630 ms / 686.920 TFLOP/s/GPU with 158.220 GiB peak allocated and 162.290 GiB peak reserved memory. Complete 15-file, 430-GiB checkpoints with distributed metadata are written at steps 50 and 100. -

+

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192, GBS/MBS 512/2 real-data RP2 convergence-verification run completes exactly 100 finite-loss optimizer steps with natural routing, Transformer Engine fused cross entropy, CUDA graphs disabled, finite LM/MTP-1/MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.171970 -> 5.798433; real-data steps 91-100 average 21,483.630 ms / 686.920 TFLOP/s/GPU with 158.220 GiB peak allocated and 162.290 GiB peak reserved memory. Complete 15-file, 430-GiB checkpoints with distributed metadata are written at steps 50 and 100.

@@ -437,8 +431,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192, GBS/MBS 512/2 BF16 real-data RP2 FSDP convergence-verification run completes exactly 100 finite-loss optimizer steps with natural routing, Transformer Engine fused cross entropy, active MTP-1/MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.171970 -> 5.797082; real-data steps 91-100 average 26,665.830 ms / 553.430 TFLOP/s/GPU with 160.760 GiB peak allocated and 166.230 GiB peak reserved memory. Complete 14-file, 369-GiB FSDP checkpoints with distributed metadata are written at steps 50 and 100. Do not use this run to claim convergence neutrality against ordinary BF16 pretraining: 188 of 300 shared LM/MTP loss values exceed the 1% loss gate, with a maximum relative difference of 12.95%, despite the similar final loss. -

+

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192, GBS/MBS 512/2 BF16 real-data RP2 FSDP convergence-verification run completes exactly 100 finite-loss optimizer steps with natural routing, Transformer Engine fused cross entropy, active MTP-1/MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.171970 -> 5.797082; real-data steps 91-100 average 26,665.830 ms / 553.430 TFLOP/s/GPU with 160.760 GiB peak allocated and 166.230 GiB peak reserved memory. Complete 14-file, 369-GiB FSDP checkpoints with distributed metadata are written at steps 50 and 100. Do not use this run to claim convergence neutrality against ordinary BF16 pretraining: 188 of 300 shared LM/MTP loss values exceed the 1% loss gate, with a maximum relative difference of 12.95%, despite the similar final loss.

@@ -496,8 +489,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192 MXFP8 FSDP recipe completes exactly 20 finite-loss optimizer steps with GBS/MBS 384/3, active MTP-1 and MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.19034 -> 3.913218, and steps 11-20 average 13,917.000 ms / 795.390 TFLOP/s/GPU with 169.54 GiB peak allocated and 173.86 GiB peak reserved memory. This run must not be compared with the BF16 real-data pretrain or BF16 FSDP runs: it uses MXFP8, mock data, forced routing, GBS/MBS 384/3, and no CUDA graphs. -

+

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, the sequence-length-8,192 MXFP8 FSDP recipe completes exactly 20 finite-loss optimizer steps with GBS/MBS 384/3, active MTP-1 and MTP-2 losses, and no skipped or NaN iterations. LM loss is 12.19034 -> 3.913218, and steps 11-20 average 13,917.000 ms / 795.390 TFLOP/s/GPU with 169.54 GiB peak allocated and 173.86 GiB peak reserved memory. This run must not be compared with the BF16 real-data pretrain or BF16 FSDP runs: it uses MXFP8, mock data, forced routing, GBS/MBS 384/3, and no CUDA graphs.

@@ -547,8 +539,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Packed OpenMathInstruct-2 full SFT completes 100 finite-loss steps with active MTP gradients, LM loss 0.4406566 -> 0.2757806, finite MTP-1 loss 0.7371141 -> 0.4133309, and no skipped or NaN iterations. Steps 91-100 average 9,887.800 ms / 79.240 TFLOP/s/GPU, and the complete reloadable full-model checkpoint is written at step 100. -

+

Packed OpenMathInstruct-2 full SFT completes 100 finite-loss steps with active MTP gradients, LM loss 0.4406566 -> 0.2757806, finite MTP-1 loss 0.7371141 -> 0.4133309, and no skipped or NaN iterations. Steps 91-100 average 9,887.800 ms / 79.240 TFLOP/s/GPU, and the complete reloadable full-model checkpoint is written at step 100.

@@ -606,8 +597,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, packed OpenMathInstruct-2 full SFT completes exactly 100 finite-loss optimizer steps with active MTP gradients. LM loss is 0.4407797 -> 0.2762039, MTP-1 loss is 0.7373838 -> 0.4146959, and MTP-2 loss is 0.8092746 -> 0.4744196. Steps 91-100 average 6,795.830 ms / 256.990 TFLOP/s/GPU with 101.510 GiB peak allocated and 103.960 GiB peak reserved memory. No iteration is skipped or NaN, and the complete 12-file reloadable full-model checkpoint is written at step 100. -

+

On two complete four-GPU GB200 nodes in one segment-2 NVLink domain, packed OpenMathInstruct-2 full SFT completes exactly 100 finite-loss optimizer steps with active MTP gradients. LM loss is 0.4407797 -> 0.2762039, MTP-1 loss is 0.7373838 -> 0.4146959, and MTP-2 loss is 0.8092746 -> 0.4744196. Steps 91-100 average 6,795.830 ms / 256.990 TFLOP/s/GPU with 101.510 GiB peak allocated and 103.960 GiB peak reserved memory. No iteration is skipped or NaN, and the complete 12-file reloadable full-model checkpoint is written at step 100.

@@ -652,13 +642,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_sft_config --mode sft --dataset openmathinstruct2 --max_steps 100 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --context_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.sequence_parallel=true model.cp_comm_type=p2p model.cross_entropy_loss_fusion=true model.calculate_per_token_loss=true model.recompute_granularity=full model.recompute_modules=null model.recompute_method=uniform model.recompute_num_layers=1 model.moe_router_force_load_balancing=false ddp.average_in_collective=false ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/long-sft-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe nemotron_3_5_lightning_sft_config --mode sft --dataset openmathinstruct2 --max_steps 100 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --context_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.sequence_parallel=true model.cp_comm_type=p2p model.cross_entropy_loss_fusion=true model.calculate_per_token_loss=true model.recompute_granularity=full model.recompute_modules=null model.recompute_method=uniform model.recompute_num_layers=1 model.moe_router_force_load_balancing=false ddp.average_in_collective=false ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/long-sft-checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Packed OpenMathInstruct-2 CP2 SFT completes exactly 100 finite-loss 32K steps with LM loss 0.4268321 -> 0.2650192 and MTP-1 loss 0.7322379 -> 0.4439273. All 100 steps have finite LM/MTP losses and no skipped or NaN iterations. Steps 91-100 average 40,863.010 ms / 188.950 TFLOP/s/GPU. Complete step-50 and step-100 checkpoints each contain 20 files, distributed metadata, saved run configuration and training state, with latest marker 100. -

+

Packed OpenMathInstruct-2 CP2 SFT completes exactly 100 finite-loss 32K steps with LM loss 0.4268321 -> 0.2650192 and MTP-1 loss 0.7322379 -> 0.4439273. All 100 steps have finite LM/MTP losses and no skipped or NaN iterations. Steps 91-100 average 40,863.010 ms / 188.950 TFLOP/s/GPU. Complete step-50 and step-100 checkpoints each contain 20 files, distributed metadata, saved run configuration and training state, with latest marker 100.

@@ -703,13 +692,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_sft_config --mode sft --dataset openmathinstruct2 --max_steps 100 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 --context_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 model.moe_flex_dispatcher_backend=hybridep 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.calculate_per_token_loss=true model.recompute_granularity=selective 'model.recompute_modules=[moe,layernorm,core_attn,mlp]' model.recompute_method=null model.recompute_num_layers=null model.moe_router_force_load_balancing=false ddp.average_in_collective=false ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/long-sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_sft_config --mode sft --dataset openmathinstruct2 --max_steps 100 --seq_length 32768 --lr 5e-6 --min_lr 0 --warmup_iters 10 --context_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 model.moe_flex_dispatcher_backend=hybridep 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:32768,pad_seq_to_mult:4}' model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false model.calculate_per_token_loss=true model.recompute_granularity=selective 'model.recompute_modules=[moe,layernorm,core_attn,mlp]' model.recompute_method=null model.recompute_num_layers=null model.moe_router_force_load_balancing=false ddp.average_in_collective=false ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save_optim=false checkpoint.save_rng=false checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/long-sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Packed OpenMathInstruct-2 CP2 SFT completes exactly 100 finite-loss 32K steps with LM loss 0.4268995 -> 0.2649925 and MTP-1 loss 0.7322624 -> 0.4439029. All 100 steps have finite LM/MTP losses and no skipped or NaN iterations. Steps 91-100 average 129,692.530 ms / 119.030 TFLOP/s/GPU. The complete step-100 checkpoint has distributed metadata, saved run configuration and training state, 12 files, and a latest marker of 100. -

+

Packed OpenMathInstruct-2 CP2 SFT completes exactly 100 finite-loss 32K steps with LM loss 0.4268995 -> 0.2649925 and MTP-1 loss 0.7322624 -> 0.4439029. All 100 steps have finite LM/MTP losses and no skipped or NaN iterations. Steps 91-100 average 129,692.530 ms / 119.030 TFLOP/s/GPU. The complete step-100 checkpoint has distributed metadata, saved run configuration and training state, 12 files, and a latest marker of 100.

@@ -754,13 +742,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe nemotron_3_5_lightning_peft_config --mode lora --dataset openmathinstruct2 --max_steps 100 --seq_length 4096 --lr 1e-4 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:4096,pad_seq_to_mult:2}' model.sequence_parallel=true ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe nemotron_3_5_lightning_peft_config --mode lora --dataset openmathinstruct2 --max_steps 100 --seq_length 4096 --lr 1e-4 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-h100/imported-megatron/iter_0000000 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:4096,pad_seq_to_mult:2}' model.sequence_parallel=true ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-h100/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Packed rank-32, alpha-32, zero-dropout LoRA completes 100 finite-loss steps with adapters on the MTP layers, LM loss 0.4406696 -> 0.2854756, finite MTP-1 loss 0.7370828 -> 0.4362708, and no skipped or NaN iterations. The complete adapter checkpoint is written at step 100 and reloads over the pinned base checkpoint at step 100. -

+

Packed rank-32, alpha-32, zero-dropout LoRA completes 100 finite-loss steps with adapters on the MTP layers, LM loss 0.4406696 -> 0.2854756, finite MTP-1 loss 0.7370828 -> 0.4362708, and no skipped or NaN iterations. The complete adapter checkpoint is written at step 100 and reloads over the pinned base checkpoint at step 100.

@@ -805,13 +792,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_peft_config --mode lora --dataset openmathinstruct2 --max_steps 100 --seq_length 4096 --lr 1e-4 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 model.moe_flex_dispatcher_backend=hybridep 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:4096,pad_seq_to_mult:2}' model.sequence_parallel=true ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe nemotron_3_5_lightning_peft_config --mode lora --dataset openmathinstruct2 --max_steps 100 --seq_length 4096 --lr 1e-4 --min_lr 0 --warmup_iters 10 --tensor_model_parallel_size 2 --expert_tensor_parallel_size 1 --expert_model_parallel_size 8 --pretrained_checkpoint work/model-verification/nemotron-3.5-lightning-gb200/imported-megatron/iter_0000000 model.moe_flex_dispatcher_backend=hybridep 'dataset.hf_dataset.load_kwargs={revision:"469216e3f46f4dacf476b382e192485ea51a143e"}' dataset.enable_offline_packing=true 'dataset.offline_packing_specs={packed_sequence_size:4096,pad_seq_to_mult:2}' model.sequence_parallel=true ddp.overlap_param_gather=false optimizer.overlap_param_gather=false scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.async_save=false --save_dir work/model-verification/nemotron-3.5-lightning-gb200/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

Packed rank-32, alpha-32, zero-dropout LoRA completes 100 finite-loss steps with 48 rank-local adapter attachment records covering MTP attention, expert, and shared-expert layers, LM loss 0.4406088 -> 0.2855029, finite MTP-1 loss 0.7371238 -> 0.4362065, and no skipped or NaN iterations. The complete adapter checkpoint is written at step 100 and reloads over the pinned base checkpoint at step 100. -

+

Packed rank-32, alpha-32, zero-dropout LoRA completes 100 finite-loss steps with 48 rank-local adapter attachment records covering MTP attention, expert, and shared-expert layers, LM loss 0.4406088 -> 0.2855029, finite MTP-1 loss 0.7371238 -> 0.4362065, and no skipped or NaN iterations. The complete adapter checkpoint is written at step 100 and reloads over the pinned base checkpoint at step 100.

@@ -869,8 +855,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 16x H100, the exact mock-data, force-balanced BF16 performance recipe completes exactly 50 steps with finite LM/MTP-1/MTP-2 losses and no skipped or NaN iterations. Transformer Engine CUDA graphs cover attention and Mamba scopes, while expert-FC1 activations are offloaded. Steps 41-50 average 18,467.280 ms and 399.580 model TFLOP/s/GPU; peak rank-0 allocated/reserved memory is 72.066/75.340 GiB. This benchmark-only run stores Adam moments in BF16, so its losses are not convergence-comparable with FP32 optimizer-state runs. -

+

On 16x H100, the exact mock-data, force-balanced BF16 performance recipe completes exactly 50 steps with finite LM/MTP-1/MTP-2 losses and no skipped or NaN iterations. Transformer Engine CUDA graphs cover attention and Mamba scopes, while expert-FC1 activations are offloaded. Steps 41-50 average 18,467.280 ms and 399.580 model TFLOP/s/GPU; peak rank-0 allocated/reserved memory is 72.066/75.340 GiB. This benchmark-only run stores Adam moments in BF16, so its losses are not convergence-comparable with FP32 optimizer-state runs.

@@ -928,11 +913,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 8x GB200, the exact mock-data BF16 performance recipe completes exactly 50 steps with finite LM/MTP-1/MTP-2 losses and no skipped or NaN iterations. Transformer Engine CUDA graphs cover attention, Mamba, MoE-router, and MoE-preprocess scopes. Steps 41-50 average 18,236.060 ms and 809.220 model TFLOP/s/GPU; peak rank-0 allocated/reserved memory is 156.850/161.260 GiB. -

+

On 8x GB200, the exact mock-data BF16 performance recipe completes exactly 50 steps with finite LM/MTP-1/MTP-2 losses and no skipped or NaN iterations. Transformer Engine CUDA graphs cover attention, Mamba, MoE-router, and MoE-preprocess scopes. Steps 41-50 average 18,236.060 ms and 809.220 model TFLOP/s/GPU; peak rank-0 allocated/reserved memory is 156.850/161.260 GiB.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl-text-only.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl-text-only.mdx index 8bfd546455..e09e7518ff 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl-text-only.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl-text-only.mdx @@ -6,7 +6,7 @@ This workflow reuses [Nemotron 3 Super](nemotron3-super.md) text recipes with th The public checkpoint revision remains unbound. CPU conversion, long-context training, and text-only pretrain/resume verification are deferred; their unverified status does not imply lack of implementation support. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -147,8 +147,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU conversion is deferred; no CPU-only verification is claimed. -

+

CPU conversion is deferred; no CPU-only verification is claimed.

@@ -173,8 +172,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed GPU import and paired export passed on H100 and GB200. All 42,683 language tensors, including 1,040 MTP tensors, matched in keys, shapes, dtypes and values across the saved round trip. Compared 247,222,108,160 source tensor bytes; no media tensors were exported. This establishes weight conversion, not forward or training parity. -

+

Distributed GPU import and paired export passed on H100 and GB200. All 42,683 language tensors, including 1,040 MTP tensors, matched in keys, shapes, dtypes and values across the saved round trip. Compared 247,222,108,160 source tensor bytes; no media tensors were exported. This establishes weight conversion, not forward or training parity.

@@ -193,8 +191,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU conversion is deferred; no CPU-only verification is claimed. -

+

CPU conversion is deferred; no CPU-only verification is claimed.

@@ -219,8 +216,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Exported a standalone native NemotronHForCausalLM checkpoint with unprefixed language and MTP weights and zero media tensors. All 42,683 tensors matched the source exactly, including 1,040 MTP tensors. The tokenizer was preserved, native HF config loading and normal text bridge dispatch passed, and the tensor-name/shape structure matched Nemotron 3 Super. This does not claim identical weights to that model. -

+

Exported a standalone native NemotronHForCausalLM checkpoint with unprefixed language and MTP weights and zero media tensors. All 42,683 tensors matched the source exactly, including 1,040 MTP tensors. The tokenizer was preserved, native HF config loading and normal text bridge dispatch passed, and the tensor-name/shape structure matched Nemotron 3 Super. This does not claim identical weights to that model.

@@ -264,8 +260,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Additional text-only pretrain and resume verification is deferred. Recipe construction alone does not establish training or resume correctness. -

+

Additional text-only pretrain and resume verification is deferred. Recipe construction alone does not establish training or resume correctness.

@@ -309,8 +304,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Additional text-only pretrain and resume verification is deferred. Recipe construction alone does not establish training or resume correctness. -

+

Additional text-only pretrain and resume verification is deferred. Recipe construction alone does not establish training or resume correctness.

@@ -355,13 +349,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --load_dir work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft checkpoint.ckpt_step=250 checkpoint.finetune=false --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft-recovery model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=64 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 mixed_precision.grad_reduce_in_fp32=true ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft-recovery-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft-recovery-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
+ 
./scripts/training/train.sh --wait --nodes 8 --gpus-per-node 8 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --load_dir work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft checkpoint.ckpt_step=250 checkpoint.finetune=false --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft-recovery model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=64 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 mixed_precision.grad_reduce_in_fp32=true ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft-recovery-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load_optim=true checkpoint.load_rng=true checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/h100-sft-recovery-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
Expected result
-

Completed a fresh-root continuation from the original complete step-250 checkpoint through step 500 after the reference run failed at final save. All 1250 loss comparisons across steps 251-500 passed the unchanged 1% plus 1e-6 bound. Metrics below describe only that continuation, not an uninterrupted 500-step execution. Uses the native Nemotron 3 Super SFT recipe on 64 H100 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP64/ETP1, sequence length 2048 and recipe GBS/MBS 32/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. Final export, native HF reload and one matched numerical fixture passed; those do not establish general probability equality. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison. -

+

Completed a fresh-root continuation from the original complete step-250 checkpoint through step 500 after the reference run failed at final save. All 1250 loss comparisons across steps 251-500 passed the unchanged 1% plus 1e-6 bound. Metrics below describe only that continuation, not an uninterrupted 500-step execution. Uses the native Nemotron 3 Super SFT recipe on 64 H100 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP64/ETP1, sequence length 2048 and recipe GBS/MBS 32/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. Final export, native HF reload and one matched numerical fixture passed; those do not establish general probability equality. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison.

@@ -406,13 +399,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 16 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --pretrained_checkpoint work/model-verification/nemotron-3.5-super-vl-text-only/megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/gb200-sft model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=64 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 mixed_precision.grad_reduce_in_fp32=true ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-sft-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-sft-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
+ 
./scripts/training/train.sh --wait --nodes 16 --gpus-per-node 4 --recipe nemotron_3_super_sft_16gpu_h100_bf16_config --mode sft --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --pretrained_checkpoint work/model-verification/nemotron-3.5-super-vl-text-only/megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/gb200-sft model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=64 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 mixed_precision.grad_reduce_in_fp32=true ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-sft-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-sft-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
Expected result
-

Completed 500 finite optimizer steps with zero skipped or NaN iterations and a complete final checkpoint. Uses the native Nemotron 3 Super SFT recipe on 64 GB200 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP64/ETP1, sequence length 2048 and recipe GBS/MBS 32/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. Final export, native HF reload and one matched numerical fixture passed; those do not establish general probability equality. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison. -

+

Completed 500 finite optimizer steps with zero skipped or NaN iterations and a complete final checkpoint. Uses the native Nemotron 3 Super SFT recipe on 64 GB200 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP64/ETP1, sequence length 2048 and recipe GBS/MBS 32/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. Final export, native HF reload and one matched numerical fixture passed; those do not establish general probability equality. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison.

@@ -456,8 +448,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Long-context text-only training is deferred; no long-context verification is claimed. -

+

Long-context text-only training is deferred; no long-context verification is claimed.

@@ -501,8 +492,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Long-context text-only training is deferred; no long-context verification is claimed. -

+

Long-context text-only training is deferred; no long-context verification is claimed.

@@ -547,13 +537,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --pretrained_checkpoint work/model-verification/nemotron-3.5-super-vl-text-only/megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/h100-peft model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=16 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/h100-peft-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/h100-peft-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
+ 
./scripts/training/train.sh --wait --nodes 2 --gpus-per-node 8 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --pretrained_checkpoint work/model-verification/nemotron-3.5-super-vl-text-only/megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/h100-peft model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=16 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/h100-peft-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/h100-peft-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
Expected result
-

Completed 500 finite optimizer steps with zero skipped or NaN iterations and a complete final checkpoint. Uses the native Nemotron 3 Super PEFT recipe on 16 H100 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP16/ETP1, sequence length 2048 and recipe GBS/MBS 16/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. LoRA rank/alpha 32/32 targets native language projections with zero dropout. Native adapter and trained router-bias buffers reloaded exactly; full merge, strict HF reload and one matched numerical fixture passed. Standalone HF adapter export does not preserve trained router-bias buffers. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison. -

+

Completed 500 finite optimizer steps with zero skipped or NaN iterations and a complete final checkpoint. Uses the native Nemotron 3 Super PEFT recipe on 16 H100 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP16/ETP1, sequence length 2048 and recipe GBS/MBS 16/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. LoRA rank/alpha 32/32 targets native language projections with zero dropout. Native adapter and trained router-bias buffers reloaded exactly; full merge, strict HF reload and one matched numerical fixture passed. Standalone HF adapter export does not preserve trained router-bias buffers. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison.

@@ -598,16 +587,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --pretrained_checkpoint work/model-verification/nemotron-3.5-super-vl-text-only/megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/gb200-peft model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=16 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-peft-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-peft-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
+ 
./scripts/training/train.sh --wait --nodes 4 --gpus-per-node 4 --recipe nemotron_3_super_peft_16gpu_h100_bf16_config --mode lora --dataset tulu3 --seq_length 2048 --max_steps 500 --save_interval 250 --pretrained_checkpoint work/model-verification/nemotron-3.5-super-vl-text-only/megatron/iter_0000000 --save_dir work/model-verification/nemotron-3.5-super-vl-text-only/gb200-peft model.hf_model_id=work/model-verification/nemotron-3.5-super-vl-text-only/hf tokenizer.tokenizer_model=work/model-verification/nemotron-3.5-super-vl-text-only/hf ++tokenizer.hf_tokenizer_kwargs.trust_remote_code=false model.expert_model_parallel_size=16 model.moe_token_dispatcher_type=flex model.moe_flex_dispatcher_backend=hybridep model.moe_shared_expert_overlap=false model.moe_router_force_load_balancing=false optimizer.use_precision_aware_optimizer=false optimizer.main_grads_dtype=float32 optimizer.main_params_dtype=float32 optimizer.exp_avg_dtype=float32 optimizer.exp_avg_sq_dtype=float32 ddp.grad_reduce_in_fp32=true dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/tulu3/jsonl dataset.do_validation=false dataset.do_test=false dataset.enable_offline_packing=true ++dataset.dataset_kwargs.pad_to_max_length=true ++dataset.dataset_kwargs.index_mapping_dir=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-peft-indices 'dataset.offline_packing_specs={packed_sequence_size:2048,pad_seq_to_mult:8,packed_train_data_path:work/data/tulu3/packed/training.idx.parquet,packed_val_data_path:work/data/tulu3/packed/validation.idx.parquet,packed_metadata_path:work/data/tulu3/packed/metadata.jsonl}' scheduler.lr_decay_iters=500 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false checkpoint.async_save=false checkpoint.exit_on_missing_checkpoint=true logger.log_interval=1 logger.log_throughput=true logger.log_device_memory_used=true logger.tensorboard_dir=null logger.save_config_filepath=work/model-verification/nemotron-3.5-super-vl-text-only/gb200-peft-config/ConfigContainer.yaml ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true
Expected result
-

Completed 500 finite optimizer steps with zero skipped or NaN iterations and a complete final checkpoint. Uses the native Nemotron 3 Super PEFT recipe on 16 GB200 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP16/ETP1, sequence length 2048 and recipe GBS/MBS 16/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. LoRA rank/alpha 32/32 targets native language projections with zero dropout. Native adapter and trained router-bias buffers reloaded exactly; full merge, strict HF reload and one matched numerical fixture passed. Standalone HF adapter export does not preserve trained router-bias buffers. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison. -

+

Completed 500 finite optimizer steps with zero skipped or NaN iterations and a complete final checkpoint. Uses the native Nemotron 3 Super PEFT recipe on 16 GB200 GPUs with this model's converted language weights and tokenizer, no media modules, TP8/PP1/EP16/ETP1, sequence length 2048 and recipe GBS/MBS 16/1. Tulu 3 offline-packed data uses assistant-only main and MTP loss masking, natural HybridEP routing and FP32 optimizer state. Both shared MTP prediction depths use loss scale 0.3. LoRA rank/alpha 32/32 targets native language projections with zero dropout. Native adapter and trained router-bias buffers reloaded exactly; full merge, strict HF reload and one matched numerical fixture passed. Standalone HF adapter export does not preserve trained router-bias buffers. Token rates are padded slots, not measured supervised-token counts. This is bounded support verification, not full convergence or a cross-model convergence comparison.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl.mdx b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl.mdx index cb06b1cad7..90abf92c7c 100644 --- a/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl.mdx +++ b/docs/fern/versions/nightly/pages/models/nemotron/nemotron3.5-super-vl.mdx @@ -10,7 +10,7 @@ The records below contain refreshed conversion, inference, and training verifica For language-only conversion and training without media components, see [the text-only companion guide](nemotron3.5-super-vl-text-only.md). - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -151,8 +151,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU-only import is not claimed in this card; the distributed GPU import below is the verified conversion path for this checkpoint. -

+

CPU-only import is not claimed in this card; the distributed GPU import below is the verified conversion path for this checkpoint.

@@ -177,8 +176,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed BF16 GPU import and paired export passed with all 43,079 source tensors exact in keys, shapes, dtypes and values. Compared 248,906,833,960 source bytes, including language, MTP, image encoder, projector, temporal embedder and the vision summary-index buffer. This establishes conversion, not training or numerical forward parity. -

+

Distributed BF16 GPU import and paired export passed with all 43,079 source tensors exact in keys, shapes, dtypes and values. Compared 248,906,833,960 source bytes, including language, MTP, image encoder, projector, temporal embedder and the vision summary-index buffer. This establishes conversion, not training or numerical forward parity.

@@ -197,8 +195,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU-only export is not claimed in this card; the distributed GPU export below is the verified conversion path for this checkpoint. -

+

CPU-only export is not claimed in this card; the distributed GPU export below is the verified conversion path for this checkpoint.

@@ -223,8 +220,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Saved round-trip comparison passed for all 43,079 tensors with no missing or extra keys and zero value differences. The exported package reloaded through native Transformers and performed bounded image generation. Exact conversion is separate from forward-logit correlation. -

+

Saved round-trip comparison passed for all 43,079 tensors with no missing or extra keys and zero value differences. The exported package reloaded through native Transformers and performed bounded image generation. Exact conversion is separate from forward-logit correlation.

@@ -274,8 +270,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Completed 200 finite optimizer steps on 64 H100s with zero skipped or NaN iterations, full checkpoints at steps 100 and 200, and a fresh final-checkpoint native inference reload. DataComp image-caption training warm-started from converted pretrained weights at sequence length 4096, GBS/MBS 1280/1, TP2/PP2/EP32/ETP1, natural HybridEP routing and FP32 optimizer state. Shared MTP uses two prediction depths and loss scale 0.3. Native finalization and update preflights passed for this workload. Rates are padded token slots per GPU, not actual supervised-token counts; production supervised totals were not recorded. This is bounded support verification, not full convergence. -

+

Completed 200 finite optimizer steps on 64 H100s with zero skipped or NaN iterations, full checkpoints at steps 100 and 200, and a fresh final-checkpoint native inference reload. DataComp image-caption training warm-started from converted pretrained weights at sequence length 4096, GBS/MBS 1280/1, TP2/PP2/EP32/ETP1, natural HybridEP routing and FP32 optimizer state. Shared MTP uses two prediction depths and loss scale 0.3. Native finalization and update preflights passed for this workload. Rates are padded token slots per GPU, not actual supervised-token counts; production supervised totals were not recorded. This is bounded support verification, not full convergence.

@@ -319,8 +314,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Refreshed DataComp pretraining used 64 GB200 GPUs, pretrained weights, natural HybridEP routing, FP32 optimizer state, TP2/PP1/EP64/ETP1, sequence length 8192, GBS/MBS 512/1 and selective moe_act recompute. The original 500-step schedule reached step 475 before its four-hour limit. A fresh-root continuation from step 250 reached step 500 and saved a complete final checkpoint, whose native model reload passed. This is not an uninterrupted 500-step reference. One of 1125 matched loss observations exceeded the unchanged resume tolerance; independent replay remains pending, so no verified pretrain/resume pair is claimed. -

+

Refreshed DataComp pretraining used 64 GB200 GPUs, pretrained weights, natural HybridEP routing, FP32 optimizer state, TP2/PP1/EP64/ETP1, sequence length 8192, GBS/MBS 512/1 and selective moe_act recompute. The original 500-step schedule reached step 475 before its four-hour limit. A fresh-root continuation from step 250 reached step 500 and saved a complete final checkpoint, whose native model reload passed. This is not an uninterrupted 500-step reference. One of 1125 matched loss observations exceeded the unchanged resume tolerance; independent replay remains pending, so no verified pretrain/resume pair is claimed.

@@ -370,8 +364,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Completed 200 finite optimizer steps on 64 H100 GPUs with zero skipped or NaN iterations and a complete final checkpoint. DataComp image-caption training uses sequence length 4096 and GBS/MBS 1280/1, natural HybridEP routing, FP32 optimizer state, and two shared MTP prediction depths at loss scale 0.3. The vision encoder is frozen; the language model and projection are trainable. Native finalization/update preflights passed. Final export and native HF reload are checked separately. Token rates are padded slots, not measured supervised-token totals. This verifies image-caption training, not video training or full convergence. -

+

Completed 200 finite optimizer steps on 64 H100 GPUs with zero skipped or NaN iterations and a complete final checkpoint. DataComp image-caption training uses sequence length 4096 and GBS/MBS 1280/1, natural HybridEP routing, FP32 optimizer state, and two shared MTP prediction depths at loss scale 0.3. The vision encoder is frozen; the language model and projection are trainable. Native finalization/update preflights passed. Final export and native HF reload are checked separately. Token rates are padded slots, not measured supervised-token totals. This verifies image-caption training, not video training or full convergence.

@@ -421,8 +414,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Completed 200 finite optimizer steps on 64 GB200 GPUs with zero skipped or NaN iterations and a complete final checkpoint. DataComp image-caption training uses sequence length 4096 and GBS/MBS 1280/1, natural HybridEP routing, FP32 optimizer state, and two shared MTP prediction depths at loss scale 0.3. The vision encoder is frozen; the language model and projection are trainable. Native finalization/update preflights passed. Final export and native HF reload are checked separately. Token rates are padded slots, not measured supervised-token totals. This verifies image-caption training, not video training or full convergence. -

+

Completed 200 finite optimizer steps on 64 GB200 GPUs with zero skipped or NaN iterations and a complete final checkpoint. DataComp image-caption training uses sequence length 4096 and GBS/MBS 1280/1, natural HybridEP routing, FP32 optimizer state, and two shared MTP prediction depths at loss scale 0.3. The vision encoder is frozen; the language model and projection are trainable. Native finalization/update preflights passed. Final export and native HF reload are checked separately. Token rates are padded slots, not measured supervised-token totals. This verifies image-caption training, not video training or full convergence.

@@ -466,8 +458,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No H100 long-context SFT result is claimed in this card. -

+

No H100 long-context SFT result is claimed in this card.

@@ -511,8 +502,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

No GB200 long-context SFT result is claimed in this card. -

+

No GB200 long-context SFT result is claimed in this card.

@@ -562,8 +552,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Completed 500 finite LoRA optimizer steps on 16 H100 GPUs with zero skipped or NaN iterations and a complete final native adapter checkpoint. DataComp image-caption training uses sequence length 4096, GBS/MBS 16/1, rank 32, alpha 32, zero adapter dropout and language-only targets. Native save/reload and merge preserve updated router expert-bias buffers as well as adapter tensors. Full merged HF reload and bounded image generation passed; stronger post-merge numerical diagnostics are not all passing. Standalone HF adapter export does not preserve these trained router buffers; use native reload and full-model merge/export. Token rates are padded slots, not measured supervised-token totals. -

+

Completed 500 finite LoRA optimizer steps on 16 H100 GPUs with zero skipped or NaN iterations and a complete final native adapter checkpoint. DataComp image-caption training uses sequence length 4096, GBS/MBS 16/1, rank 32, alpha 32, zero adapter dropout and language-only targets. Native save/reload and merge preserve updated router expert-bias buffers as well as adapter tensors. Full merged HF reload and bounded image generation passed; stronger post-merge numerical diagnostics are not all passing. Standalone HF adapter export does not preserve these trained router buffers; use native reload and full-model merge/export. Token rates are padded slots, not measured supervised-token totals.

@@ -613,11 +602,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Completed 500 finite LoRA optimizer steps on 16 GB200 GPUs with zero skipped or NaN iterations and a complete final native adapter checkpoint. DataComp image-caption training uses sequence length 4096, GBS/MBS 16/1, rank 32, alpha 32, zero adapter dropout and language-only targets. Native save/reload and merge preserve updated router expert-bias buffers as well as adapter tensors. Full merged HF reload and bounded image generation passed; stronger post-merge numerical diagnostics are not all passing. Standalone HF adapter export does not preserve these trained router buffers; use native reload and full-model merge/export. Token rates are padded slots, not measured supervised-token totals. -

+

Completed 500 finite LoRA optimizer steps on 16 GB200 GPUs with zero skipped or NaN iterations and a complete final native adapter checkpoint. DataComp image-caption training uses sequence length 4096, GBS/MBS 16/1, rank 32, alpha 32, zero adapter dropout and language-only targets. Native save/reload and merge preserve updated router expert-bias buffers as well as adapter tensors. Full merged HF reload and bounded image generation passed; stronger post-merge numerical diagnostics are not all passing. Standalone HF adapter export does not preserve these trained router buffers; use native reload and full-model merge/export. Token rates are padded slots, not measured supervised-token totals.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3-235b-a22b.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3-235b-a22b.mdx index ce9b51c3d2..731675627c 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3-235b-a22b.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3-235b-a22b.mdx @@ -4,7 +4,7 @@ Qwen family models are supported via the Bridge system with auto-detected configuration and weight mapping. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -132,8 +132,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates. -

+

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.

@@ -152,8 +151,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates. -

+

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.

@@ -172,8 +170,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates. -

+

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.

@@ -192,8 +189,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates. -

+

This workflow remains unverified and requires a public run against the pinned Hugging Face revision with all applicable verification gates.

@@ -237,8 +233,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -282,8 +277,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -327,8 +321,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -372,8 +365,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates. -

+

This workflow remains unverified and requires a bounded public run with the model's pinned revision and all applicable verification gates.

@@ -423,8 +415,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 256 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP4/CP1/EP32/ETP1, GBS/MBS 8192/2, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.75473 to 8.126335; the final ten steps average 14920.840 ms, 1300.440 TFLOP/s/GPU, and 8784.492 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 256 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP4/CP1/EP32/ETP1, GBS/MBS 8192/2, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.75473 to 8.126335; the final ten steps average 14920.840 ms, 1300.440 TFLOP/s/GPU, and 8784.492 tokens/s/GPU. The resolved configuration persists.

@@ -474,11 +465,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 256 GB200s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP8/CP1/EP32/ETP1, VPP3, GBS/MBS 8192/1, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.75152 to 8.126326; the final ten steps average 17905.340 ms, 1083.660 TFLOP/s/GPU, and 7320.274 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 256 GB200s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP8/CP1/EP32/ETP1, VPP3, GBS/MBS 8192/1, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.75152 to 8.126326; the final ten steps average 17905.340 ms, 1083.660 TFLOP/s/GPU, and 7320.274 tokens/s/GPU. The resolved configuration persists.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3-30b-a3b.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3-30b-a3b.mdx index 8845ed11bd..abb1a9a9be 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3-30b-a3b.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3-30b-a3b.mdx @@ -4,7 +4,7 @@ Qwen family models are supported via the Bridge system with auto-detected configuration and weight mapping. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -180,8 +180,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with all 18,867 BF16 tensors matching the recorded HF revision bitwise. -

+

The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with all 18,867 BF16 tensors matching the recorded HF revision bitwise.

@@ -206,8 +205,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, creates iter_0000000, and all 18,867 BF16 tensors reload with keys, shapes, dtypes, and values exactly matching the recorded HF revision. -

+

The command exits successfully, creates iter_0000000, and all 18,867 BF16 tensors reload with keys, shapes, dtypes, and values exactly matching the recorded HF revision.

@@ -232,8 +230,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully; all 18,867 exported BF16 tensors match the recorded HF revision bitwise, norm_topk_prob remains true, and the export reloads as Qwen3MoeForCausalLM without missing or unexpected keys. -

+

The command exits successfully; all 18,867 exported BF16 tensors match the recorded HF revision bitwise, norm_topk_prob remains true, and the export reloads as Qwen3MoeForCausalLM without missing or unexpected keys.

@@ -258,8 +255,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict export exits successfully, all 18,867 written BF16 tensors match the recorded HF revision bitwise, norm_topk_prob remains true, and Transformers reloads the output as Qwen3MoeForCausalLM without missing or unexpected keys. -

+

Strict export exits successfully, all 18,867 written BF16 tensors match the recorded HF revision bitwise, norm_topk_prob remains true, and Transformers reloads the output as Qwen3MoeForCausalLM without missing or unexpected keys.

@@ -309,8 +305,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 16x H100, the public alias resolves to the 16-GPU recipe and completes exactly 100 bounded RP2 optimizer steps with TP1/PP1/CP1/EP16/ETP1, DP16, SP off, GBS/MBS 1024/1, and 64-way gradient accumulation. Natural routing, HybridEP, and Transformer Engine CUDA graphs for moe_router and moe_preprocess remain active. Loss is finite from 12.41145 to 6.139116 with no skipped or NaN iterations, all five metrics are recorded, and complete iter_0000050 and iter_0000100 checkpoints are saved. -

+

On 16x H100, the public alias resolves to the 16-GPU recipe and completes exactly 100 bounded RP2 optimizer steps with TP1/PP1/CP1/EP16/ETP1, DP16, SP off, GBS/MBS 1024/1, and 64-way gradient accumulation. Natural routing, HybridEP, and Transformer Engine CUDA graphs for moe_router and moe_preprocess remain active. Loss is finite from 12.41145 to 6.139116 with no skipped or NaN iterations, all five metrics are recorded, and complete iter_0000050 and iter_0000100 checkpoints are saved.

@@ -360,8 +355,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 8x GB200, this support-verification workload completes exactly 100 bounded RP2 optimizer steps with TP1/PP1/CP1/EP8/ETP1, DP8, SP off, GBS/MBS 512/4, and 16-way gradient accumulation. MXFP8 compute, natural routing, HybridEP, Transformer Engine CUDA graphs for moe_router and moe_preprocess, selective MoE activation recompute, communication overlap, functional safety checks, and MXFP8 parameter all-gather remain active. Loss is finite from 10.70991 to 6.046449 with no skipped or NaN iterations, all five metrics are recorded, and complete eight-shard iter_0000050 and iter_0000100 checkpoints are saved. Timing and throughput are support sanity checks, not cross-model convergence or tuned performance claims. -

+

On 8x GB200, this support-verification workload completes exactly 100 bounded RP2 optimizer steps with TP1/PP1/CP1/EP8/ETP1, DP8, SP off, GBS/MBS 512/4, and 16-way gradient accumulation. MXFP8 compute, natural routing, HybridEP, Transformer Engine CUDA graphs for moe_router and moe_preprocess, selective MoE activation recompute, communication overlap, functional safety checks, and MXFP8 parameter all-gather remain active. Loss is finite from 10.70991 to 6.046449 with no skipped or NaN iterations, all five metrics are recorded, and complete eight-shard iter_0000050 and iter_0000100 checkpoints are saved. Timing and throughput are support sanity checks, not cross-model convergence or tuned performance claims.

@@ -406,13 +400,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_sft_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="ad44e777bcd18fa416d9da3bd8f70d33ebb85d39"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --save_dir work/model-verification/qwen3-30b-a3b/sft-convergence-v1-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_sft_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="ad44e777bcd18fa416d9da3bd8f70d33ebb85d39"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --save_dir work/model-verification/qwen3-30b-a3b/sft-convergence-v1-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.tensorboard_dir=null
Expected result
-

The immutable-revision 16-GPU run completes exactly 100 full-SFT steps at TP1/PP1/CP1/EP16/ETP1, DP16, SP off, GBS/MBS 32/1, and two-way gradient accumulation with natural routing. Pad-1 offline packing is 99.30% efficient, and the sampled 6,553,600 token slots contain 4,350,004 supervised tokens after label masking. LM loss is finite from 1.704380 to 0.9030643 with no skipped or NaN iterations, all five metrics are recorded, and the complete sixteen-shard iter_0000100 full-model checkpoint reloads successfully. -

+

The immutable-revision 16-GPU run completes exactly 100 full-SFT steps at TP1/PP1/CP1/EP16/ETP1, DP16, SP off, GBS/MBS 32/1, and two-way gradient accumulation with natural routing. Pad-1 offline packing is 99.30% efficient, and the sampled 6,553,600 token slots contain 4,350,004 supervised tokens after label masking. LM loss is finite from 1.704380 to 0.9030643 with no skipped or NaN iterations, all five metrics are recorded, and the complete sixteen-shard iter_0000100 full-model checkpoint reloads successfully.

@@ -457,13 +450,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --context_parallel_size 2 -tp 8 -pp 1 -ep 8 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="ad44e777bcd18fa416d9da3bd8f70d33ebb85d39"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=16 model.cp_comm_type=p2p model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen3_30b_a3b_sft_8gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --context_parallel_size 2 -tp 8 -pp 1 -ep 8 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="ad44e777bcd18fa416d9da3bd8f70d33ebb85d39"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=16 model.cp_comm_type=p2p model.cross_entropy_loss_fusion=false model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true
Expected result
-

The immutable-revision 16-GPU run completes exactly 20 Tulu3 SFT steps at sequence length 32768 with TP8/PP1/CP2/EP8/SP-on, DeepEP, and explicit pad-16 offline packing. LM loss is 1.645009 to 1.468103; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.28%, and the sampled training window contains 13,573,663 actual supervised tokens. PP=1 keeps tokens, labels, loss masks, and packed-sequence boundaries on one pipeline stage. -

+

The immutable-revision 16-GPU run completes exactly 20 Tulu3 SFT steps at sequence length 32768 with TP8/PP1/CP2/EP8/SP-on, DeepEP, and explicit pad-16 offline packing. LM loss is 1.645009 to 1.468103; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.28%, and the sampled training window contains 13,573,663 actual supervised tokens. PP=1 keeps tokens, labels, loss masks, and packed-sequence boundaries on one pipeline stage.

@@ -508,13 +500,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen3_30b_a3b_peft_4gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="ad44e777bcd18fa416d9da3bd8f70d33ebb85d39"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --save_dir work/model-verification/qwen3-30b-a3b/peft-tp4-ep4-pad4-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen3_30b_a3b_peft_4gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-30b-a3b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="ad44e777bcd18fa416d9da3bd8f70d33ebb85d39"' dataset.hf_output_root=work/data/tulu3/qwen3-30b-a3b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 model.moe_router_force_load_balancing=false checkpoint.load=null ddp.check_for_nan_in_grad=true ddp.check_for_large_grads=true rerun_state_machine.check_for_nan_in_loss=true --save_dir work/model-verification/qwen3-30b-a3b/peft-tp4-ep4-pad4-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

The immutable-revision 4-GPU run completes exactly 100 PEFT steps at TP4/PP1/CP1/EP4/ETP1, DP1, SP on, GBS/MBS 32/1, and 32-way gradient accumulation with natural routing and DeepEP. Pad-4 offline packing is 99.39% efficient, and the sampled 6,553,600 token slots contain 4,332,480 supervised tokens after label masking. Only rank-8, alpha-16, zero-dropout LoRA on linear_qkv and linear_proj is trainable. LM loss is finite from 1.575987 to 1.113119 with no skipped or NaN iterations, all five metrics are recorded, and the complete four-shard iter_0000100 adapter checkpoint covers all 192 expected adapter entries. -

+

The immutable-revision 4-GPU run completes exactly 100 PEFT steps at TP4/PP1/CP1/EP4/ETP1, DP1, SP on, GBS/MBS 32/1, and 32-way gradient accumulation with natural routing and DeepEP. Pad-4 offline packing is 99.39% efficient, and the sampled 6,553,600 token slots contain 4,332,480 supervised tokens after label masking. Only rank-8, alpha-16, zero-dropout LoRA on linear_qkv and linear_proj is trainable. LM loss is finite from 1.575987 to 1.113119 with no skipped or NaN iterations, all five metrics are recorded, and the complete four-shard iter_0000100 adapter checkpoint covers all 192 expected adapter entries.

@@ -564,8 +555,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two nodes with 16x H100, the exact mock-data performance recipe completes exactly 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. HybridEP permute fusion uses 32 SMs and 64-token combine chunks, expert-parallel communication overlap is active with delayed weight-gradient compute disabled, and Transformer Engine CUDA graph capture completes for all 48 graphable layers with moe_router and moe_preprocess scopes. Over steps 41-50, the run averages at most 20.50 seconds per step and at least 295 model TFLOP/s/GPU, while peak allocated memory remains below 65 GiB/GPU. -

+

On two nodes with 16x H100, the exact mock-data performance recipe completes exactly 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. HybridEP permute fusion uses 32 SMs and 64-token combine chunks, expert-parallel communication overlap is active with delayed weight-gradient compute disabled, and Transformer Engine CUDA graph capture completes for all 48 graphable layers with moe_router and moe_preprocess scopes. Over steps 41-50, the run averages at most 20.50 seconds per step and at least 295 model TFLOP/s/GPU, while peak allocated memory remains below 65 GiB/GPU.

@@ -615,8 +605,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two nodes with 8x GB200, the exact mock-data performance recipe completes exactly 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. The benchmark uses forced load balancing, HybridEP, MXFP8, and full-iteration CUDA graphs. Over steps 41-50, the run averages at most 7 seconds per step and at least 900 model TFLOP/s/GPU. -

+

On two nodes with 8x GB200, the exact mock-data performance recipe completes exactly 50 steps with finite losses, no skipped or NaN iterations, and all five metrics recorded. The benchmark uses forced load balancing, HybridEP, MXFP8, and full-iteration CUDA graphs. Over steps 41-50, the run averages at most 7 seconds per step and at least 900 model TFLOP/s/GPU.

@@ -666,8 +655,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On exactly 8 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP8/ETP1, GBS/MBS 512/8, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.34753 to 8.125566; the final ten steps average 5854.910 ms, 1030.070 TFLOP/s/GPU, and 44773.361 tokens/s/GPU. The resolved configuration persists. -

+

On exactly 8 GB300s, the canonical MXFP8 mock-data recipe completes exactly 50 optimizer steps at TP1/PP1/CP1/EP8/ETP1, GBS/MBS 512/8, and sequence length 4096. All 50 keyed rows have finite loss with zero skipped or NaN iterations. Loss moves from 12.34753 to 8.125566; the final ten steps average 5854.910 ms, 1030.070 TFLOP/s/GPU, and 44773.361 tokens/s/GPU. The resolved configuration persists.

@@ -717,8 +705,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows. -

+

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.

@@ -768,8 +755,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows. -

+

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.

@@ -819,8 +805,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows. -

+

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.

@@ -870,11 +855,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows. -

+

On GB300, the exact mock-data MXFP8 recipe completes 50 steps at each of 8, 32, 128, and 256 GPUs with TP1/PP1/CP1/EP8/ETP1, MBS8, sequence length 4096, and GBS proportional to GPU count. Every point has finite losses and performance values, zero skipped or NaN iterations, a persisted post-setup runtime config, and all five metrics recorded from complete keyed optimizer-step rows.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3-8b.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3-8b.mdx index b747f24470..d9ef0751fa 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3-8b.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3-8b.mdx @@ -4,7 +4,7 @@ Qwen family models are supported via the Bridge system with auto-detected configuration and weight mapping. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -122,8 +122,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with all 399 HF tensors matching the recorded HF revision exactly in keys, shapes, dtypes, and values. -

+

The command exits successfully, creates iter_0000000, and the checkpoint round-trips through CPU export with all 399 HF tensors matching the recorded HF revision exactly in keys, shapes, dtypes, and values.

@@ -148,8 +147,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully, creates iter_0000000, and the checkpoint reloads at TP=4 with weights exactly matching the recorded HF revision. -

+

The command exits successfully, creates iter_0000000, and the checkpoint reloads at TP=4 with weights exactly matching the recorded HF revision.

@@ -174,8 +172,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The command exits successfully; all 399 exported tensors match the recorded HF revision exactly, and the export reloads on CPU as Qwen3ForCausalLM. -

+

The command exits successfully; all 399 exported tensors match the recorded HF revision exactly, and the export reloads on CPU as Qwen3ForCausalLM.

@@ -200,8 +197,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Strict export exits successfully and the Hugging Face output reloads with AutoModelForCausalLM as Qwen3ForCausalLM. -

+

Strict export exits successfully and the Hugging Face output reloads with AutoModelForCausalLM as Qwen3ForCausalLM.

@@ -251,8 +247,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The uninterrupted 16-GPU bounded RP2 run completes exactly 100 steps at TP1/PP1/CP1, DP16, GBS/MBS 1024/1, and 64-way gradient accumulation. It reaches peak learning rate at step 40 and completes cosine decay at step 100. LM loss is finite from 12.73617 to 6.190218 with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved. -

+

The uninterrupted 16-GPU bounded RP2 run completes exactly 100 steps at TP1/PP1/CP1, DP16, GBS/MBS 1024/1, and 64-way gradient accumulation. It reaches peak learning rate at step 40 and completes cosine decay at step 100. LM loss is finite from 12.73617 to 6.190218 with no skipped or NaN iterations, all five metrics are recorded, the post-setup configuration persists, and complete 16-shard iter_0000050 and iter_0000100 checkpoints are saved.

@@ -297,13 +292,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen3_8b_sft_4gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b968826d9c46dd6066d109eabc6255188de91218"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/qwen3-8b/sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen3_8b_sft_4gpu_h100_bf16_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 5e-6 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b968826d9c46dd6066d109eabc6255188de91218"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-sft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=1 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/qwen3-8b/sft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Immutable Tulu3 pad-1 offline packing is 99.30% efficient. Full SFT uses DP=1 with 32 gradient-accumulation steps and reaches step 100 with the four recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, the sampled assistant-only loss masks contain 4,350,004 supervised tokens. The complete four-shard iter_0000100 full-model checkpoint is saved. -

+

Immutable Tulu3 pad-1 offline packing is 99.30% efficient. Full SFT uses DP=1 with 32 gradient-accumulation steps and reaches step 100 with the four recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, the sampled assistant-only loss masks contain 4,350,004 supervised tokens. The complete four-shard iter_0000100 full-model checkpoint is saved.

@@ -348,13 +342,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe qwen3_8b_sft_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b968826d9c46dd6066d109eabc6255188de91218"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=8 model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 8 --recipe qwen3_8b_sft_32k_config --mode sft --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 20 --seq_length 32768 --context_parallel_size 2 --lr 1e-6 --min_lr 0 --warmup_iters 2 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b968826d9c46dd6066d109eabc6255188de91218"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-long-context-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=8 model.sequence_parallel=true model.cp_comm_type=a2a model.cross_entropy_loss_fusion=false scheduler.lr_decay_iters=20 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null checkpoint.save=null logger.log_interval=1 logger.log_throughput=true
Expected result
-

The immutable-revision 8-GPU run completes exactly 20 Tulu3 SFT steps at sequence length 32768 with recipe-owned TP4/PP1/CP2/SP-on, GBS/MBS 8/1, and explicit pad-8 offline packing. LM loss is 1.611622 to 1.490618; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.97%, and the sampled training window contains 3,444,917 actual supervised tokens. -

+

The immutable-revision 8-GPU run completes exactly 20 Tulu3 SFT steps at sequence length 32768 with recipe-owned TP4/PP1/CP2/SP-on, GBS/MBS 8/1, and explicit pad-8 offline packing. LM loss is 1.611622 to 1.490618; skipped/NaN totals are 0/0. The persisted post-setup runtime config matches the command, packing is 99.97%, and the sampled training window contains 3,444,917 actual supervised tokens.

@@ -399,16 +392,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe qwen3_8b_peft_1gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b968826d9c46dd6066d109eabc6255188de91218"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/qwen3-8b/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 1 --recipe qwen3_8b_peft_1gpu_h100_bf16_config --mode lora --dataset tulu3 --pretrained_checkpoint work/model-verification/qwen3-8b/imported-megatron/iter_0000000 --max_steps 100 --seq_length 2048 --lr 1e-4 --min_lr 0 --warmup_iters 10 'dataset.hf_dataset.split="train[:10000]"' 'dataset.hf_dataset.load_kwargs={revision:"b14afda60f1bbebe55d5d2fa1e4df5042f97f8be"}' '++tokenizer.hf_tokenizer_kwargs.revision="b968826d9c46dd6066d109eabc6255188de91218"' dataset.hf_output_root=work/data/tulu3/qwen3-8b-peft-b14afda60f1b dataset.hf_rewrite=true dataset.seed=1234 rng.seed=5678 dataset.do_validation=false dataset.hf_validation_proportion=null dataset.enable_offline_packing=true +dataset.offline_packing_specs.pad_seq_to_mult=4 scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 checkpoint.load=null --save_dir work/model-verification/qwen3-8b/peft-checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true
Expected result
-

Immutable Tulu3 pad-4 offline packing is 99.39% efficient. The 100 LoRA steps use DP=1 with 32 gradient-accumulation steps and complete with the five recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, the sampled assistant-only loss masks contain 4,332,480 supervised tokens. The complete single-shard iter_0000100 adapter checkpoint is saved. -

+

Immutable Tulu3 pad-4 offline packing is 99.39% efficient. The 100 LoRA steps use DP=1 with 32 gradient-accumulation steps and complete with the five recorded metrics, finite loss, and no skipped or NaN iterations. Across 6,553,600 token slots, the sampled assistant-only loss masks contain 4,332,480 supervised tokens. The complete single-shard iter_0000100 adapter checkpoint is saved.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx index f1f0c4560d..3f47c21fd8 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx @@ -12,7 +12,7 @@ Qwen 3.5/3.6 models are supported via Megatron Bridge with auto-detected configu Use `transformers` >= 5.2.0 for Qwen3.5 and >= 5.8.1 for Qwen3.6. ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -145,8 +145,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned-revision import exited successfully in 14m39s and created iter_0000000. Model weights remained on CPU; exactly one shared runtime GPU was required because Transformer Engine constructs the Qwen vision modules through CUDA even for CPU-weight conversion. The resulting checkpoint was reloadable by the subsequent verified CPU export. -

+

The pinned-revision import exited successfully in 14m39s and created iter_0000000. Model weights remained on CPU; exactly one shared runtime GPU was required because Transformer Engine constructs the Qwen vision modules through CUDA even for CPU-weight conversion. The resulting checkpoint was reloadable by the subsequent verified CPU export.

@@ -171,8 +170,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 16-H100 pinned-revision import exited successfully at TP2/PP1/EP8/ETP1 and created a 16-shard iter_0000000 distributed checkpoint. Subsequent verified GPU export and inference workloads reloaded this checkpoint successfully. -

+

The 16-H100 pinned-revision import exited successfully at TP2/PP1/EP8/ETP1 and created a 16-shard iter_0000000 distributed checkpoint. Subsequent verified GPU export and inference workloads reloaded this checkpoint successfully.

@@ -197,8 +195,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Export exited successfully in 6m42s with weights on CPU and exactly one shared runtime GPU for Transformer Engine vision-module construction. Transformers strictly reloaded all 1,045 BF16 tensors in 26 indexed shards as Qwen3_5MoeForConditionalGeneration. The exported config preserves 40 language layers, 256 experts, top-8 routing, 27 vision layers, and one MTP layer with dedicated embeddings disabled. -

+

Export exited successfully in 6m42s with weights on CPU and exactly one shared runtime GPU for Transformer Engine vision-module construction. Transformers strictly reloaded all 1,045 BF16 tensors in 26 indexed shards as Qwen3_5MoeForConditionalGeneration. The exported config preserves 40 language layers, 256 experts, top-8 routing, 27 vision layers, and one MTP layer with dedicated embeddings disabled.

@@ -223,8 +220,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 16-H100 distributed export exited successfully and wrote 26 indexed shards containing 1,045 BF16 tensors and 71,903,645,408 serialized tensor bytes. Transformers independently reloaded the persisted export as Qwen3_5MoeForConditionalGeneration with empty missing, unexpected, mismatched, and error sets. -

+

The 16-H100 distributed export exited successfully and wrote 26 indexed shards containing 1,045 BF16 tensors and 71,903,645,408 serialized tensor bytes. Transformers independently reloaded the persisted export as Qwen3_5MoeForConditionalGeneration with empty missing, unexpected, mismatched, and error sets.

@@ -274,8 +270,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 16-H100 command completed all 50 full-model DataComp Energon steps at TP1/PP2/CP1/EP8/ETP1 and MBS1/GBS512 after warm-starting the immutable imported checkpoint for natural routing. Loss remained finite from 3.604611 to 3.458849 with zero skipped or NaN iterations. The 519,827/5,173 train/validation split was prepared with the official DataComp downloader at commit 4a8df1992566, DataComp-1B metadata revision 086ebeee20d4, img2dataset 1.40.0, and preparation manifest 6e273a96a756. This verifies bounded full-model training, not canonical DataComp/CLIP convergence or checkpoint resume. -

+

The 16-H100 command completed all 50 full-model DataComp Energon steps at TP1/PP2/CP1/EP8/ETP1 and MBS1/GBS512 after warm-starting the immutable imported checkpoint for natural routing. Loss remained finite from 3.604611 to 3.458849 with zero skipped or NaN iterations. The 519,827/5,173 train/validation split was prepared with the official DataComp downloader at commit 4a8df1992566, DataComp-1B metadata revision 086ebeee20d4, img2dataset 1.40.0, and preparation manifest 6e273a96a756. This verifies bounded full-model training, not canonical DataComp/CLIP convergence or checkpoint resume.

@@ -320,13 +315,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_sft_16gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.6-35b-a3b/sft-support-v11/checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/sft-support-v11/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_sft_16gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.6-35b-a3b/sft-support-v11/checkpoints --save_interval 50 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/sft-support-v11/resolved-config.yaml
Expected result
-

The exact 16-H100 command completed all 100 MedPix SFT steps at TP1/PP2/CP1/EP8/ETP1, MBS1/GBS32, natural expert routing, HybridEP, one-layer full-uniform recompute, graph-free language and vision stacks, FP32 optimizer state, and Transformer Engine fused cross entropy. Loss was finite from 2.211317 to 1.006886 with zero skipped or NaN iterations. Steps 91-100 averaged 3,268.140 ms and 57.690 model TFLOP/s/GPU. Step-50 and step-100 saves both completed; the final checkpoint contains 20 nonempty files, including 16 distributed shards, totaling 509,636,716,209 bytes, and the tracker selects iteration 100. -

+

The exact 16-H100 command completed all 100 MedPix SFT steps at TP1/PP2/CP1/EP8/ETP1, MBS1/GBS32, natural expert routing, HybridEP, one-layer full-uniform recompute, graph-free language and vision stacks, FP32 optimizer state, and Transformer Engine fused cross entropy. Loss was finite from 2.211317 to 1.006886 with zero skipped or NaN iterations. Steps 91-100 averaged 3,268.140 ms and 57.690 model TFLOP/s/GPU. Step-50 and step-100 saves both completed; the final checkpoint contains 20 nonempty files, including 16 distributed shards, totaling 509,636,716,209 bytes, and the tracker selects iteration 100.

@@ -371,13 +365,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_sft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-sft/resolved-config.yaml
Expected result
-

On 8x GB200, the shared Qwen3.5/Qwen3.6-VL functional recipe completed exactly 100 MedPix SFT steps at TP1/PP1/CP1/EP8/ETP1, MBS1/GBS32, natural routing, HybridEP, no activation recompute, and BF16 gradient reduction. Loss was finite from 2.212866 to 0.9978729 with zero skipped or NaN iterations. Steps 91-100 averaged 9,899.640 ms and 38.180 model TFLOP/s/GPU. The recipe intentionally disables checkpoint output for this bounded functional run; reload/export coverage remains represented by the H100 SFT and sft_export_inference items. -

+

On 8x GB200, the shared Qwen3.5/Qwen3.6-VL functional recipe completed exactly 100 MedPix SFT steps at TP1/PP1/CP1/EP8/ETP1, MBS1/GBS32, natural routing, HybridEP, no activation recompute, and BF16 gradient reduction. Loss was finite from 2.212866 to 0.9978729 with zero skipped or NaN iterations. Steps 91-100 averaged 9,899.640 ms and 38.180 model TFLOP/s/GPU. The recipe intentionally disables checkpoint output for this bounded functional run; reload/export coverage remains represented by the H100 SFT and sft_export_inference items.

@@ -422,13 +415,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_sft_long_context_32gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false scheduler.lr_decay_iters=20 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/medpix-long-context/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 4 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_sft_long_context_32gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false scheduler.lr_decay_iters=20 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/medpix-long-context/resolved-config.yaml
Expected result
-

The pinned 32-H100 run completed exactly 20 full-SFT steps at TP1/PP4/CP2/EP8/ETP1, dense DP4 and expert DP1, MBS2/GBS512, and 64-way gradient accumulation. It used 8192-token processor-native deferred in-batch packing with pad-to-multiple-of-4, fixed 448-by-448 MedPix images, per-token loss, and one-layer full-uniform activation recompute. The persisted post-setup config confirms every setting. Loss was finite from 1.997653 to 0.997894 with zero skipped or NaN iterations. After the compile-heavy first step, steps 11-20 averaged 36,847.110 ms and 84.830 TFLOP/s/GPU; the process exited successfully. -

+

The pinned 32-H100 run completed exactly 20 full-SFT steps at TP1/PP4/CP2/EP8/ETP1, dense DP4 and expert DP1, MBS2/GBS512, and 64-way gradient accumulation. It used 8192-token processor-native deferred in-batch packing with pad-to-multiple-of-4, fixed 448-by-448 MedPix images, per-token loss, and one-layer full-uniform activation recompute. The persisted post-setup config confirms every setting. Loss was finite from 1.997653 to 0.997894 with zero skipped or NaN iterations. After the compile-heavy first step, steps 11-20 averaged 36,847.110 ms and 84.830 TFLOP/s/GPU; the process exited successfully.

@@ -473,13 +465,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_peft_16gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.6-35b-a3b/peft-support-v5/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/peft-support-v5/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 8 --recipe qwen35_vl_35b_a3b_peft_16gpu_h100_bf16_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/imported-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' dataset.do_validation=false dataset.pad_to_max_length=true scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.6-35b-a3b/peft-support-v5/checkpoints --save_interval 100 logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/peft-support-v5/resolved-config.yaml
Expected result
-

The exact 16-H100 command completed all 100 MedPix LoRA steps at TP1/PP2/CP1/EP8/ETP1, MBS1/GBS32, natural expert routing, HybridEP, scoped Transformer Engine CUDA graphs, and no activation recompute. The recipe disables dispatch-backward/expert-wgrad overlap because LoRA-wrapped expert linears do not expose Megatron Core's backward_dw hook. Loss was finite from 2.212696 to 1.270382 with zero skipped or NaN iterations. Steps 91-100 averaged 1,714.300 ms and 109.940 model TFLOP/s/GPU. The step-100 checkpoint contains 20 nonempty files, including 16 distributed shards, totaling 5,004,722,270 bytes. -

+

The exact 16-H100 command completed all 100 MedPix LoRA steps at TP1/PP2/CP1/EP8/ETP1, MBS1/GBS32, natural expert routing, HybridEP, scoped Transformer Engine CUDA graphs, and no activation recompute. The recipe disables dispatch-backward/expert-wgrad overlap because LoRA-wrapped expert linears do not expose Megatron Core's backward_dw hook. Loss was finite from 2.212696 to 1.270382 with zero skipped or NaN iterations. Steps 91-100 averaged 1,714.300 ms and 109.940 model TFLOP/s/GPU. The step-100 checkpoint contains 20 nonempty files, including 16 distributed shards, totaling 5,004,722,270 bytes.

@@ -524,16 +515,15 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 2 --gpus-per-node 4 --recipe qwen35_vl_35b_a3b_peft_8gpu_gb200_bf16_functional_config --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.6-35b-a3b/gb200-imported/iter_0000000 --max_steps 100 --warmup_iters 10 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=Qwen/Qwen3.6-35B-A3B 'dataset.hf_processor_kwargs={revision:995ad96eacd98c81ed38be0c5b274b04031597b0}' scheduler.lr_decay_iters=100 model.hf_model_id=Qwen/Qwen3.6-35B-A3B model.bos_token_id=248044 logger.save_config_filepath=work/model-verification/qwen3.6-35b-a3b/gb200-peft/resolved-config.yaml
Expected result
-

On the same 8x GB200 topology and MedPix schedule as the controlled SFT item, LoRA completed exactly 100 finite steps with zero skipped or NaN iterations. The persisted configs differ only in the evidence-output path, the intended PEFT LR/min-LR, and the LoRA block; model, data, topology, batch, schedule, and safety settings match. Step-1 loss 2.214827 differs from SFT's 2.212866 by only 0.001961, or 0.088618%, ruling out an initial-forward configuration mismatch. The later loss difference reflects LoRA's trainable parameter set and learning rate. Steps 91-100 averaged 5,511.530 ms and 68.410 model TFLOP/s/GPU. -

+

On the same 8x GB200 topology and MedPix schedule as the controlled SFT item, LoRA completed exactly 100 finite steps with zero skipped or NaN iterations. The persisted configs differ only in the evidence-output path, the intended PEFT LR/min-LR, and the LoRA block; model, data, topology, batch, schedule, and safety settings match. Step-1 loss 2.214827 differs from SFT's 2.212866 by only 0.001961, or 0.088618%, ruling out an initial-forward configuration mismatch. The later loss difference reflects LoRA's trainable parameter set and learning rate. Steps 91-100 averaged 5,511.530 ms and 68.410 model TFLOP/s/GPU.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3.8-27b.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3.8-27b.mdx index d48c1ddc57..0bc023e643 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3.8-27b.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3.8-27b.mdx @@ -12,7 +12,7 @@ Qwen 3.5/3.6 models are supported via Megatron Bridge with auto-detected configu Use `transformers` >= 5.2.0 for Qwen3.5 and >= 5.8.1 for Qwen3.6. ``` - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -145,8 +145,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned-revision CPU import completed in 3m11s and created a reloadable iter_0000000 containing 8 nonempty files totaling 55,586,366,001 bytes. The converter selected Qwen35VLBridge, completed all 956 mappings, kept model weights on CPU, and reported 27,781,427,952 parameters. One shared runtime GB200 was used for Transformer Engine vision-module construction. -

+

The pinned-revision CPU import completed in 3m11s and created a reloadable iter_0000000 containing 8 nonempty files totaling 55,586,366,001 bytes. The converter selected Qwen35VLBridge, completed all 956 mappings, kept model weights on CPU, and reported 27,781,427,952 parameters. One shared runtime GB200 was used for Transformer Engine vision-module construction.

@@ -171,8 +170,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The pinned-revision four-GB200 import completed in 2m01s at TP4/PP1 and created an 11-file iter_0000000 totaling 55,592,033,255 bytes, including four distributed checkpoint shards. All 956 conversion mappings completed, and subsequent export and two independent inference runs reloaded it. -

+

The pinned-revision four-GB200 import completed in 2m01s at TP4/PP1 and created an 11-file iter_0000000 totaling 55,592,033,255 bytes, including four distributed checkpoint shards. All 956 conversion mappings completed, and subsequent export and two independent inference runs reloaded it.

@@ -197,8 +195,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU export completed in 2m16s, wrote every source tensor, and produced 26 files totaling 55,583,128,197 bytes. The index preserves the original 1,199-key set and exact 18-shard placement with 55,562,855,904 BF16 tensor bytes. Transformers strictly reloaded the export as Qwen3_5ForConditionalGeneration with empty missing, unexpected, mismatched, and error sets, then completed processor-native GB200 inference. Export normalizes vision_config.model_type from qwen3_5 to qwen3_5_vision and updates producer-version metadata without changing the architecture. -

+

CPU export completed in 2m16s, wrote every source tensor, and produced 26 files totaling 55,583,128,197 bytes. The index preserves the original 1,199-key set and exact 18-shard placement with 55,562,855,904 BF16 tensor bytes. Transformers strictly reloaded the export as Qwen3_5ForConditionalGeneration with empty missing, unexpected, mismatched, and error sets, then completed processor-native GB200 inference. Export normalizes vision_config.model_type from qwen3_5 to qwen3_5_vision and updates producer-version metadata without changing the architecture.

@@ -223,8 +220,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The four-GB200 distributed export completed in 1m55s, wrote every source tensor, and produced 26 files totaling 55,583,128,222 bytes. Its index has the same 1,199 keys, 18-shard placement, and 55,562,855,904 BF16 tensor bytes as the pinned source and CPU export. Transformers strictly reloaded it with empty missing, unexpected, mismatched, and error sets and completed processor-native inference. Relative to CPU export, config.json only adds the metadata field vision_config.dtype=bfloat16. -

+

The four-GB200 distributed export completed in 1m55s, wrote every source tensor, and produced 26 files totaling 55,583,128,222 bytes. Its index has the same 1,199 keys, 18-shard placement, and 55,562,855,904 BF16 tensor bytes as the pinned source and CPU export. Transformers strictly reloaded it with empty missing, unexpected, mismatched, and error sets and completed processor-native inference. Relative to CPU export, config.json only adds the metadata field vision_config.dtype=bfloat16.

@@ -274,8 +270,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The 16-GB200 MockVLM pretrain warm-started from the imported checkpoint and completed exactly 20 steps at TP4/PP4, sequence length 4096, and MBS2/GBS32. The language model and multimodal vision merger trained while the vision encoder remained frozen. Deterministic mode is omitted: it replaces the fused Gated DeltaNet kernels with a torch-native reference path and is rejected alongside the recipe's pre-gated-delta-rule fusion. Loss remained finite from 6.852052 to 3.831639 with finite gradients and zero skipped or NaN iterations. Steps 11-20 averaged 3,551.780 ms, 390.360 model TFLOP/s/GPU, and 2,306.449 token slots/s/GPU; step 11, the first step after the iteration-10 save, took 4,593.5 ms. Checkpoints at iterations 10 and 20 each contain 20 files totaling 399,255,348,187 bytes, and the tracker records iteration 20. The built-in 32-iteration validation and test passes completed at loss 3.824842 and 3.824848. The process exited successfully in 16m03s. -

+

The 16-GB200 MockVLM pretrain warm-started from the imported checkpoint and completed exactly 20 steps at TP4/PP4, sequence length 4096, and MBS2/GBS32. The language model and multimodal vision merger trained while the vision encoder remained frozen. Deterministic mode is omitted: it replaces the fused Gated DeltaNet kernels with a torch-native reference path and is rejected alongside the recipe's pre-gated-delta-rule fusion. Loss remained finite from 6.852052 to 3.831639 with finite gradients and zero skipped or NaN iterations. Steps 11-20 averaged 3,551.780 ms, 390.360 model TFLOP/s/GPU, and 2,306.449 token slots/s/GPU; step 11, the first step after the iteration-10 save, took 4,593.5 ms. Checkpoints at iterations 10 and 20 each contain 20 files totaling 399,255,348,187 bytes, and the tracker records iteration 20. The built-in 32-iteration validation and test passes completed at loss 3.824842 and 3.824848. The process exited successfully in 16m03s.

@@ -320,13 +315,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft --dataset medpix --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 --save_interval 50 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.pad_to_max_length=true tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 model.recompute_modules=null scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.8-27b/sft-recompute/checkpoints logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-recompute/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft --dataset medpix --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 --save_interval 50 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.pad_to_max_length=true tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 model.recompute_modules=null scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.8-27b/sft-recompute/checkpoints logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-recompute/resolved-config.yaml
Expected result
-

The 16-GB200 run completed exactly 100 full-SFT MedPix steps at TP4/PP4, MBS4/GBS32, deterministic mode, and one-layer full-uniform activation recompute. Loss remained finite from 2.116404 to 1.429567 with zero skipped or NaN iterations. Steps 91-100 averaged 24,160.490 ms and 58.010 model TFLOP/s/GPU. Step-50 and step-100 saves each contain 20 files totaling 404,422,605,133 bytes, the tracker selects iteration 100, and the process exited successfully in 45m42s. The unchanged stock TP4/PP4 recipe without activation recompute reached the training loop but OOMed near the approximately 184.3-GiB per-GPU device limit; the successful command records the required config-only recompute overrides. -

+

The 16-GB200 run completed exactly 100 full-SFT MedPix steps at TP4/PP4, MBS4/GBS32, deterministic mode, and one-layer full-uniform activation recompute. Loss remained finite from 2.116404 to 1.429567 with zero skipped or NaN iterations. Steps 91-100 averaged 24,160.490 ms and 58.010 model TFLOP/s/GPU. Step-50 and step-100 saves each contain 20 files totaling 404,422,605,133 bytes, the tracker selects iteration 100, and the process exited successfully in 45m42s. The unchanged stock TP4/PP4 recipe without activation recompute reached the training loop but OOMed near the approximately 184.3-GiB per-GPU device limit; the successful command records the required config-only recompute overrides.

@@ -371,13 +365,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 --seq_length 8192 --pipeline_model_parallel_size 2 --context_parallel_size 2 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.enable_in_batch_packing=true dataset.defer_in_batch_packing_to_step=true dataset.in_batch_packing_pad_to_multiple_of=4 tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.calculate_per_token_loss=true model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 ddp.average_in_collective=false scheduler.lr_decay_iters=20 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-long/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_vl_27b_sft_16gpu_h100_bf16_config --mode sft --dataset medpix --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 20 --warmup_iters 2 --seq_length 8192 --pipeline_model_parallel_size 2 --context_parallel_size 2 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.enable_in_batch_packing=true dataset.defer_in_batch_packing_to_step=true dataset.in_batch_packing_pad_to_multiple_of=4 tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.calculate_per_token_loss=true model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 ddp.average_in_collective=false scheduler.lr_decay_iters=20 checkpoint.load=null checkpoint.save=null validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/sft-long/resolved-config.yaml
Expected result
-

The 16-GB200 run completed exactly 20 full-SFT steps at TP4/PP2/CP2, MBS4/GBS32, 8192 tokens, deferred processor-native in-batch packing with pad-to-multiple-of-4, per-token loss, and one-layer full-uniform activation recompute. Loss remained finite from 2.343622 to 1.378064 with zero skipped or NaN iterations. Steps 11-20 averaged 8,164.790 ms and 362.530 model TFLOP/s/GPU. The process exited successfully without checkpoint output. An earlier otherwise identical attempt with --deterministic reached the training loop but triggered the explicit guard that packed sequence does not support deterministic mode; the successful command omits that incompatible flag, as does the Qwen 3.5 MoE reference card. -

+

The 16-GB200 run completed exactly 20 full-SFT steps at TP4/PP2/CP2, MBS4/GBS32, 8192 tokens, deferred processor-native in-batch packing with pad-to-multiple-of-4, per-token loss, and one-layer full-uniform activation recompute. Loss remained finite from 2.343622 to 1.378064 with zero skipped or NaN iterations. Steps 11-20 averaged 8,164.790 ms and 362.530 model TFLOP/s/GPU. The process exited successfully without checkpoint output. An earlier otherwise identical attempt with --deterministic reached the training loop but triggered the explicit guard that packed sequence does not support deterministic mode; the successful command omits that incompatible flag, as does the Qwen 3.5 MoE reference card.

@@ -422,13 +415,12 @@ Choose a workflow, precision, and exact recorded combination. The command and ex Command - 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen35_vl_27b_peft_2gpu_h100_bf16_config --mode lora --tensor_model_parallel_size 4 --dataset medpix --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 --save_interval 50 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.pad_to_max_length=true tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 model.recompute_modules=null scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.8-27b/peft-tp4-recompute/checkpoints logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/peft-tp4-recompute/resolved-config.yaml
+ 
./scripts/training/train.sh --nodes 1 --gpus-per-node 4 --recipe qwen35_vl_27b_peft_2gpu_h100_bf16_config --mode lora --tensor_model_parallel_size 4 --dataset medpix --deterministic --pretrained_checkpoint work/model-verification/qwen3.8-27b/gpu-megatron/iter_0000000 --max_steps 100 --warmup_iters 10 --save_interval 50 'dataset.source.load_kwargs={revision:69eb219d477ab8699296da1fd6b89beb768042d3}' 'dataset.source.adapter_kwargs={resized_height:448,resized_width:448}' dataset.hf_processor_path=work/model-verification/qwen3.8-27b/gpu-hf-export dataset.do_validation=false dataset.pad_to_max_length=true tokenizer.tokenizer_model=work/model-verification/qwen3.8-27b/gpu-hf-export model.hf_model_id=Qwen/Qwen3.8-27B model.bos_token_id=248044 model.recompute_granularity=full model.recompute_method=uniform model.recompute_num_layers=1 model.recompute_modules=null scheduler.lr_decay_iters=100 validation.eval_iters=0 validation.eval_interval=0 ddp.check_for_large_grads=true --save_dir work/model-verification/qwen3.8-27b/peft-tp4-recompute/checkpoints logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.8-27b/peft-tp4-recompute/resolved-config.yaml
Expected result
-

The four-GB200 run completed exactly 100 deterministic MedPix LoRA steps at TP4/PP1, MBS4/GBS32, and one-layer full-uniform activation recompute. Loss remained finite from 2.115661 to 1.558339 with zero skipped or NaN iterations. Steps 91-100 averaged 74,048.550 ms and 75.790 model TFLOP/s/GPU. Step-50 and step-100 adapter checkpoints each contain 8 files totaling 2,294,105,245 bytes, the tracker selects iteration 100, and the process exited successfully in 2h01m43s. The unchanged stock TP2 two-GPU recipe without activation recompute OOMed; a TP4 retry without recompute also OOMed near the approximately 184.3-GiB per-GPU device limit. The successful command records only the required topology and recompute config overrides, with no code change. -

+

The four-GB200 run completed exactly 100 deterministic MedPix LoRA steps at TP4/PP1, MBS4/GBS32, and one-layer full-uniform activation recompute. Loss remained finite from 2.115661 to 1.558339 with zero skipped or NaN iterations. Steps 91-100 averaged 74,048.550 ms and 75.790 model TFLOP/s/GPU. Step-50 and step-100 adapter checkpoints each contain 8 files totaling 2,294,105,245 bytes, the tracker selects iteration 100, and the process exited successfully in 2h01m43s. The unchanged stock TP2 two-GPU recipe without activation recompute OOMed; a TP4 retry without recompute also OOMed near the approximately 184.3-GiB per-GPU device limit. The successful command records only the required topology and recompute config overrides, with no code change.

@@ -482,8 +474,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On 16 GB200 GPUs, the exact canonical BF16 language-and-projector MockVLM recipe completes 20 steps at TP2/PP1/CP1, DP8, sequence length 4096, and MBS2/GBS32. The language model and multimodal vision merger train while the vision encoder remains frozen. The recipe uses fused Gated DeltaNet execution without activation recompute and disables evaluation and checkpoint output for the bounded run. Loss remains finite from 6.852387 to 3.884668 with finite gradients and zero skipped or NaN iterations. Excluding first-iteration kernel compilation, steps 11-20 average 1,679.460 ms, 819.190 corrected model TFLOP/s/GPU, and 4,877.758 token slots/s/GPU; peak allocated memory is 174.120 GiB. The process exits successfully. This mock-data result is throughput evidence, not convergence evidence. -

+

On 16 GB200 GPUs, the exact canonical BF16 language-and-projector MockVLM recipe completes 20 steps at TP2/PP1/CP1, DP8, sequence length 4096, and MBS2/GBS32. The language model and multimodal vision merger train while the vision encoder remains frozen. The recipe uses fused Gated DeltaNet execution without activation recompute and disables evaluation and checkpoint output for the bounded run. Loss remains finite from 6.852387 to 3.884668 with finite gradients and zero skipped or NaN iterations. Excluding first-iteration kernel compilation, steps 11-20 average 1,679.460 ms, 819.190 corrected model TFLOP/s/GPU, and 4,877.758 token slots/s/GPU; peak allocated memory is 174.120 GiB. The process exits successfully. This mock-data result is throughput evidence, not convergence evidence.

@@ -533,11 +524,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

On two nodes with 16x H100, the BF16 language-and-projector MockVLM recipe warm-starts from the card's imported GPU Megatron checkpoint and completes 20 steps at TP4/PP4/CP1, DP1, sequence length 4096, and MBS2/GBS32. Sequence parallelism is enabled by the explicit model.sequence_parallel=true override; the recipe default leaves it disabled, which exhausts H100 memory on the first pipeline stage in bounded runs. The language model, including its single MTP layer, and the multimodal vision merger train while the vision encoder remains frozen. Gated DeltaNet runs fused, without activation recompute or CUDA graphs, and evaluation and checkpoint output are disabled. Loss remains finite from 6.860321 to 4.485981 with zero skipped or NaN iterations. Excluding first-iteration Triton compilation and autotuning, steps 11-20 average 4,591.290 ms, 299.620 model TFLOP/s/GPU, and 1,784.248 token slots/s/GPU. The run used a NeMo 26.10 release-candidate base container with the pinned Megatron-Core submodule, and the process exits successfully. This mock-data result is throughput evidence, not convergence evidence. -

+

On two nodes with 16x H100, the BF16 language-and-projector MockVLM recipe warm-starts from the card's imported GPU Megatron checkpoint and completes 20 steps at TP4/PP4/CP1, DP1, sequence length 4096, and MBS2/GBS32. Sequence parallelism is enabled by the explicit model.sequence_parallel=true override; the recipe default leaves it disabled, which exhausts H100 memory on the first pipeline stage in bounded runs. The language model, including its single MTP layer, and the multimodal vision merger train while the vision encoder remains frozen. Gated DeltaNet runs fused, without activation recompute or CUDA graphs, and evaluation and checkpoint output are disabled. Loss remains finite from 6.860321 to 4.485981 with zero skipped or NaN iterations. Excluding first-iteration Triton compilation and autotuning, steps 11-20 average 4,591.290 ms, 299.620 model TFLOP/s/GPU, and 1,784.248 token slots/s/GPU. The run used a NeMo 26.10 release-candidate base container with the pinned Megatron-Core submodule, and the process exits successfully. This mock-data result is throughput evidence, not convergence evidence.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/fern/versions/nightly/pages/models/stepfun/step35.mdx b/docs/fern/versions/nightly/pages/models/stepfun/step35.mdx index d19238818d..71931ddf11 100644 --- a/docs/fern/versions/nightly/pages/models/stepfun/step35.mdx +++ b/docs/fern/versions/nightly/pages/models/stepfun/step35.mdx @@ -2,7 +2,7 @@ [Step-3.5-Flash](https://huggingface.co/stepfun-ai/Step-3.5-Flash) is a MoE language model from StepFun. Megatron Bridge supports checkpoint conversion, inference, and continued pretraining through a dedicated Step-3.5 bridge, provider, and recipe. - +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} ## Verified configurations @@ -120,8 +120,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU import completed all mappings for the immutable 44-shard source, constructed 199,997,706,240 parameters, and created iter_0000000. The checkpoint reloaded successfully for CPU export; its subsequent strict audit covered all 804 source tensors. -

+

CPU import completed all mappings for the immutable 44-shard source, constructed 199,997,706,240 parameters, and created iter_0000000. The checkpoint reloaded successfully for CPU export; its subsequent strict audit covered all 804 source tensors.

@@ -146,8 +145,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed import completed at TP1/PP1/EP8/ETP1, constructed 33,525,780,480 parameters per rank, and created iter_0000000. The persisted checkpoint then reloaded successfully for both distributed export and checkpoint-backed generation. -

+

Distributed import completed at TP1/PP1/EP8/ETP1, constructed 33,525,780,480 parameters per rank, and created iter_0000000. The persisted checkpoint then reloaded successfully for both distributed export and checkpoint-backed generation.

@@ -172,8 +170,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

CPU export reloaded the imported checkpoint and wrote 44 indexed BF16 shards with all 804 source keys, shapes, and dtypes. Of 398,768,626,944 serialized bytes, 395,600,878,848 bytes across 801 tensors matched the immutable source bitwise. The three published MTP output entries occupy 3,167,748,096 bytes and were intentionally materialized from Megatron's shared lm_head representation; each exported entry matched lm_head bitwise and used independent safetensors storage. A Transformers reload through the pinned checkpoint's metadata/RoPE compatibility view ignored only the three published inference-unused MTP layers, predicted the same next token as the source (' Paris'), and reached cosine similarity 0.997260 with maximum/mean absolute logit differences 1.320312/0.209649. -

+

CPU export reloaded the imported checkpoint and wrote 44 indexed BF16 shards with all 804 source keys, shapes, and dtypes. Of 398,768,626,944 serialized bytes, 395,600,878,848 bytes across 801 tensors matched the immutable source bitwise. The three published MTP output entries occupy 3,167,748,096 bytes and were intentionally materialized from Megatron's shared lm_head representation; each exported entry matched lm_head bitwise and used independent safetensors storage. A Transformers reload through the pinned checkpoint's metadata/RoPE compatibility view ignored only the three published inference-unused MTP layers, predicted the same next token as the source (' Paris'), and reached cosine similarity 0.997260 with maximum/mean absolute logit differences 1.320312/0.209649.

@@ -198,8 +195,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

Distributed export reloaded the TP1/PP1/EP8/ETP1 checkpoint and wrote 44 indexed BF16 shards with all 804 source keys, shapes, and dtypes. The exact audit found 801 tensors and 395,600,878,848 bytes bit-identical to the source. Megatron-Core shares the output head with MTP, so the three source MTP output tensors were explicitly normalized to lm_head.weight; all three exported copies matched that tensor bitwise and had independent storage. The generic grouped-expert export path recovered from CUDA stack pressure by moving only the affected merge to CPU. A Transformers reload through the pinned checkpoint's metadata/RoPE compatibility view ignored only the three published inference-unused MTP layers, predicted the same next token as the source (' Paris'), and reached cosine similarity 0.997260 with maximum/mean absolute logit differences 1.320312/0.209649. -

+

Distributed export reloaded the TP1/PP1/EP8/ETP1 checkpoint and wrote 44 indexed BF16 shards with all 804 source keys, shapes, and dtypes. The exact audit found 801 tensors and 395,600,878,848 bytes bit-identical to the source. Megatron-Core shares the output head with MTP, so the three source MTP output tensors were explicitly normalized to lm_head.weight; all three exported copies matched that tensor bitwise and had independent storage. The generic grouped-expert export path recovered from CUDA stack pressure by moving only the affected merge to CPU. A Transformers reload through the pinned checkpoint's metadata/RoPE compatibility view ignored only the three published inference-unused MTP layers, predicted the same next token as the source (' Paris'), and reached cosine similarity 0.997260 with maximum/mean absolute logit differences 1.320312/0.209649.

@@ -243,8 +239,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

The public Step-3.5-Flash H100 recipe must complete a bounded 100-step run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card. -

+

The public Step-3.5-Flash H100 recipe must complete a bounded 100-step run with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.

@@ -288,8 +283,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A pinned-data 100-step full-SFT run must finish with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card. -

+

A pinned-data 100-step full-SFT run must finish with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable final checkpoint. Training is deferred by this card.

@@ -333,8 +327,7 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A dedicated packed long-context SFT run must complete 100 steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable checkpoint. Training is deferred by this card. -

+

A dedicated packed long-context SFT run must complete 100 steps with finite loss, no skipped or NaN iterations, all five metrics, and a reloadable checkpoint. Training is deferred by this card.

@@ -378,11 +371,10 @@ Choose a workflow, precision, and exact recorded combination. The command and ex
Expected result
-

A Step-3.5-Flash PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, all five metrics, and a reloadable adapter checkpoint. Training is deferred by this card. -

+

A Step-3.5-Flash PEFT recipe with an audited adapter target set must complete 100 steps with finite loss, all five metrics, and a reloadable adapter checkpoint. Training is deferred by this card.

- +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/scripts/docs/generate_model_verification_catalog.py b/scripts/docs/generate_model_verification_catalog.py index 258a18946c..30cfb0836f 100644 --- a/scripts/docs/generate_model_verification_catalog.py +++ b/scripts/docs/generate_model_verification_catalog.py @@ -38,6 +38,9 @@ NAV_END = "# END GENERATED MODEL VERIFICATION CATALOG" MODEL_SECTION_START = "" MODEL_SECTION_END = "" +FERN_GENERATED_NOTICE = "{/* Generated by scripts/docs/generate_model_verification_catalog.py. Do not edit. */}" +FERN_MODEL_SECTION_START = "{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */}" +FERN_MODEL_SECTION_END = "{/* END GENERATED VERIFIED CONFIGURATIONS */}" VALID_STATUSES = {"verified", "unverified", "unsupported", "not_applicable"} STATUS_LABELS = { "verified": "✅ verified", @@ -547,9 +550,13 @@ def _model_directory(catalog: dict[str, object], repo_root: Path, *, fern: bool) href = f"{relative_model_doc}{suffix}#verified-{model['slug']}" entries = [_mapping(entry, f"{model['slug']} entry") for entry in model["entries"]] status = _directory_status(entries) - allowlist_suffix = ( - " " if model_name in DETECT_SECRETS_ALLOWLIST_MODEL_NAMES else "" - ) + allowlist_suffix = "" + if model_name in DETECT_SECRETS_ALLOWLIST_MODEL_NAMES: + # detect-secrets requires the comment terminator at end of line. + # Keep the MDX closing brace on the next line; Sphinx is unchanged. + allowlist_suffix = ( + " {/* pragma: allowlist secret */\n }" if fern else " " + ) lines.extend( [ f' ', @@ -656,7 +663,7 @@ def _recorded_metrics(entry: dict[str, object], *, heading_level: int) -> list[s return lines -def _model_combination_detail(entry: dict[str, object], *, heading_level: int) -> list[str]: +def _model_combination_detail(entry: dict[str, object], *, heading_level: int, fern: bool = False) -> list[str]: status = str(entry["status"]).replace("_", "-") precision = str(entry.get("precision") or "not specified") precision_label = PRECISION_LABELS.get(precision, precision.upper()) @@ -685,6 +692,9 @@ def _model_combination_detail(entry: dict[str, object], *, heading_level: int) - if isinstance(commands, list) and commands: for index, command in enumerate(commands, start=1): command_label = f"Command {index}" if len(commands) > 1 else "Command" + escaped_command = html.escape(str(command)) + if fern: + escaped_command = escaped_command.replace("{", "{").replace("}", "}") lines.extend( [ '
', @@ -692,13 +702,16 @@ def _model_combination_detail(entry: dict[str, object], *, heading_level: int) - f" {command_label}", ' ', "
", - f' 
{html.escape(str(command))}
', + f' 
{escaped_command}
', " ", ] ) else: lines.append("

No runnable command is recorded for this status.

") expected_result = str(entry.get("expected_result") or "None recorded.") + if fern: + # YAML block scalars may end in a newline, which breaks MDX paragraph parsing. + expected_result = " ".join(expected_result.split()) lines.extend( [ " ", @@ -712,7 +725,7 @@ def _model_combination_detail(entry: dict[str, object], *, heading_level: int) - return lines -def _model_explorer(entries: list[dict[str, object]], *, heading_level: int = 2) -> list[str]: +def _model_explorer(entries: list[dict[str, object]], *, heading_level: int = 2, fern: bool = False) -> list[str]: focused_entries = [entry for entry in entries if _entry_capability(entry) is not None] lines = [ f"{'#' * heading_level} Run a configuration", @@ -774,7 +787,9 @@ def _model_explorer(entries: list[dict[str, object]], *, heading_level: int = 2) lines.extend(" " + line for line in _model_combination_button(entry)) lines.extend([" ", '
']) for entry in focused_entries: - lines.extend(" " + line for line in _model_combination_detail(entry, heading_level=heading_level + 1)) + lines.extend( + " " + line for line in _model_combination_detail(entry, heading_level=heading_level + 1, fern=fern) + ) lines.extend(["
", "", ""]) return lines @@ -784,7 +799,7 @@ def render_supported_models_page(catalog: dict[str, object], repo_root: Path, *, models = [_mapping(model, "catalog model") for model in catalog["models"]] entry_count = sum(len(model["entries"]) for model in models) lines = [ - GENERATED_NOTICE, + FERN_GENERATED_NOTICE if fern else GENERATED_NOTICE, "", "# Supported Models", "", @@ -830,10 +845,10 @@ def render_supported_models_page(catalog: dict[str, object], repo_root: Path, *, return "\n".join(lines).rstrip() + "\n" -def render_model_section(models: list[dict[str, object]]) -> str: +def render_model_section(models: list[dict[str, object]], *, fern: bool = False) -> str: """Render generated verification selectors for one canonical model guide.""" lines = [ - MODEL_SECTION_START, + FERN_MODEL_SECTION_START if fern else MODEL_SECTION_START, "", "## Verified configurations", "", @@ -848,8 +863,8 @@ def render_model_section(models: list[dict[str, object]]) -> str: hf_id = str(model["hf_id"]) model_label = hf_id.partition("/")[2] or hf_id lines.extend([f"### {_md_cell(model_label)}", ""]) - lines.extend(_model_explorer(entries, heading_level=4 if multiple_models else 3)) - lines.extend([MODEL_SECTION_END, ""]) + lines.extend(_model_explorer(entries, heading_level=4 if multiple_models else 3, fern=fern)) + lines.extend([FERN_MODEL_SECTION_END if fern else MODEL_SECTION_END, ""]) return "\n".join(lines).rstrip() + "\n" @@ -919,7 +934,9 @@ def expected_outputs(repo_root: Path, catalog: dict[str, object]) -> dict[Path, title = MODEL_PAGE_TITLES.get(str(model["slug"])) merged_page = _merge_model_page(source_path.read_text(encoding="utf-8"), section, title=title) outputs[sphinx_path] = merged_page - outputs[fern_path] = merged_page + outputs[fern_path] = _merge_model_page( + source_path.read_text(encoding="utf-8"), render_model_section(models, fern=True), title=title + ) nav_path = repo_root / "docs/fern/versions/nightly.yml" outputs[nav_path] = _update_nav(nav_path.read_text(encoding="utf-8")) return outputs diff --git a/skills/nemo-mbridge-perf-activation-recompute/SKILL.md b/skills/nemo-mbridge-perf-activation-recompute/SKILL.md index c1a76e48e7..7fb474535e 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/SKILL.md +++ b/skills/nemo-mbridge-perf-activation-recompute/SKILL.md @@ -14,8 +14,6 @@ license: Apache-2.0 Stable docs: @docs/training/activation-recomputation.md Card: @skills/nemo-mbridge-perf-activation-recompute/card.yaml - - Activation recompute (activation checkpointing) trades additional forward work during backward for lower retained-activation memory. The useful checkpoint boundary depends on the model architecture, attention backend, parallelism, and the tensor that actually drives the per-rank peak. ## Quick Decision Guide @@ -278,4 +276,4 @@ Limitations of this evidence: - `skills/nemo-mbridge-perf-memory-tuning/SKILL.md` - `skills/nemo-mbridge-perf-cuda-graphs/SKILL.md` - `skills/nemo-mbridge-perf-cpu-offloading/SKILL.md` -- Megatron Core activation recomputation guide: +- [Megatron Core activation recomputation guide](https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/index.html) diff --git a/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md b/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md index e872b4ddd8..6ac46f8c46 100644 --- a/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md +++ b/skills/nemo-mbridge-perf-cuda-graphs/SKILL.md @@ -10,8 +10,6 @@ when_to_use: Reducing host-driver overhead via CUDA graphs, or tracing a crash o Stable documentation: @docs/training/cuda-graphs.md Card: @skills/nemo-mbridge-perf-cuda-graphs/card.yaml - - ## What It Is CUDA graphs capture GPU operations once and replay them with minimal diff --git a/tests/unit_tests/doc_consistency/test_model_verification_catalog.py b/tests/unit_tests/doc_consistency/test_model_verification_catalog.py index 4ee5bb8ad0..a08fc81a5f 100644 --- a/tests/unit_tests/doc_consistency/test_model_verification_catalog.py +++ b/tests/unit_tests/doc_consistency/test_model_verification_catalog.py @@ -352,3 +352,58 @@ def test_normalized_commands_equal_card_scalars(catalog: dict[str, object]) -> N if source_commands is None: source_commands = [] assert entry["commands"] == [command.strip() for command in source_commands] + + +@pytest.mark.parametrize("fern", [False, True]) +def test_generated_comments_match_output_format(generator: ModuleType, catalog: dict[str, object], fern: bool) -> None: + directory = generator.render_supported_models_page(catalog, REPO_ROOT, fern=fern) + section = generator.render_model_section([_models(catalog)["qwen3-30b-a3b"]], fern=fern) + if fern: + assert directory.startswith(generator.FERN_GENERATED_NOTICE) + assert "{/* pragma: allowlist secret */\n }" in directory + assert section.startswith(generator.FERN_MODEL_SECTION_START) + assert section.rstrip().endswith(generator.FERN_MODEL_SECTION_END) + assert "" in directory + assert section.startswith(generator.MODEL_SECTION_START) + assert section.rstrip().endswith(generator.MODEL_SECTION_END) + assert "{/*" not in directory + section + + +@pytest.mark.parametrize("fern", [False, True]) +def test_commands_and_paragraphs_match_output_format( + generator: ModuleType, catalog: dict[str, object], fern: bool +) -> None: + command = 'echo "${MODEL}" && printf "{a,b}\\n"\necho done' + expected_result = "First line.\nSecond line with .\n" + entry = dict(_models(catalog)["qwen3-30b-a3b"]["entries"][0]) + entry.update(commands=[command], expected_result=expected_result) + page = "\n".join(generator._model_combination_detail(entry, heading_level=4, fern=fern)) + escaped_command = html.escape(command) + if fern: + escaped_command = escaped_command.replace("{", "{").replace("}", "}") + assert "

First line. Second line with <value>.

" in page + assert "\n

" not in page + else: + assert f"

{html.escape(expected_result)}

" in page + assert f'{escaped_command}' in page + assert html.unescape(escaped_command) == command + + +def test_all_fern_model_outputs_use_mdx_markers(generator: ModuleType, catalog: dict[str, object]) -> None: + outputs = generator.expected_outputs(REPO_ROOT, catalog) + fern_pages = {path: page for path, page in outputs.items() if path.suffix == ".mdx"} + assert len(fern_pages) == len(catalog["models"]) + 1 + for path, page in fern_pages.items(): + if path.name == "README.mdx": + assert page.startswith(generator.FERN_GENERATED_NOTICE) + else: + assert page.count(generator.FERN_MODEL_SECTION_START) == 1 + assert page.count(generator.FERN_MODEL_SECTION_END) == 1 + assert "", + ) + text = re.sub(r"

(.*?)

", lambda match: "

" + " ".join(match[1].split()) + "

", text, flags=re.DOTALL) + return re.sub( + r'()(.*?)()', + lambda match: match[1] + match[2].replace("{", "{").replace("}", "}") + match[3], + text, + flags=re.DOTALL, + ) + + +def test_model_guide_format_normalization_preserves_content() -> None: + """Format parity must still detect changed prose, commands, and code spacing.""" + sphinx = ( + "\n" + "

Expected result\n

\n" + 'echo ${MODEL}\n --flag\n' + "\n" + ) + fern = ( + "{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */}\n" + "

Expected result

\n" + 'echo ${MODEL}\n --flag\n' + "{/* END GENERATED VERIFIED CONFIGURATIONS */}\n" + ) + expected = _normalize_model_guide_format(sphinx) + assert expected == _normalize_model_guide_format(fern) + for old, new in (("Expected result", "Different result"), ("MODEL", "OTHER"), (" --flag", " --flag")): + assert expected != _normalize_model_guide_format(fern.replace(old, new)) + assert _normalize_model_guide_format("Prose {value}") == "Prose {value}" + + def test_qwen3_model_guides_match_fern_and_reference_exported_recipes(): """Split Qwen3 guides must match Fern and reference exported recipes.""" defined_recipes = _defined_recipe_names() for sphinx_path, fern_path in QWEN3_DOC_PAIRS: sphinx_text = _read(sphinx_path) - assert sphinx_text == _read(fern_path) + assert _normalize_model_guide_format(sphinx_text) == _normalize_model_guide_format(_read(fern_path)) documented_recipes = set(re.findall(r"(qwen[0-9A-Za-z_]+_config)", sphinx_text)) assert documented_recipes, f"{sphinx_path.relative_to(REPO_ROOT)} documents no Qwen recipes"