Skip to content

Commit 8edbc78

Browse files
yaoyu-33yaoyu-33
committed
Merge branch 'main' into codex/issue5434-gtp-stage1
Signed-off-by: yaoyu-33 <yaoyu.094@gmail.com>
2 parents 40b50b4 + 092d343 commit 8edbc78

148 files changed

Lines changed: 5305 additions & 689 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.dev.commit‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
12eed0a5ac3f1ced6a3a4f76727db530bdbf7391
1+
b3b1e2d793c596c2fa61076d001ab93f294e4e87

‎.github/scripts/test_docker_dependency_layers.sh‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ set -euo pipefail
33

44
dockerfile="${1:-docker/Dockerfile.ci}"
55
workflow="${2:-.github/workflows/cicd-main.yml}"
6+
fw_final_dockerfile="${3:-docker/Dockerfile.fw_final}"
67

78
baseline_arg_line=$(grep -n '^ARG BASELINE_MCORE_REF$' "$dockerfile" | cut -d: -f1)
89
baseline_clone_line=$(grep -n 'git clone --filter=blob:none --no-checkout' "$dockerfile" | cut -d: -f1)
@@ -24,6 +25,15 @@ trap 'rm -rf "$temporary_dir"' EXIT
2425
((baseline_line < dispatched_copy_line))
2526
((dispatched_copy_line < delta_line))
2627

28+
fw_final_from_line=$(grep -n '^FROM ${NEMO_FW_FINAL_BASE_IMAGE} AS nemo_fw_final$' "$fw_final_dockerfile" | cut -d: -f1)
29+
fw_final_root_line=$(grep -n '^USER root$' "$fw_final_dockerfile" | cut -d: -f1)
30+
fw_final_clone_line=$(grep -n '^RUN git clone --depth 1 https://github.com/NVIDIA-NeMo/NeMo.git' "$fw_final_dockerfile" | cut -d: -f1)
31+
[[ -n "$fw_final_from_line" ]]
32+
[[ -n "$fw_final_root_line" ]]
33+
[[ -n "$fw_final_clone_line" ]]
34+
((fw_final_from_line < fw_final_root_line))
35+
((fw_final_root_line < fw_final_clone_line))
36+
2737
grep -q -- '--mount=type=cache,target=/root/.cache/uv' "$dockerfile"
2838
if grep -q -- '--mount=type=secret,id=GH_TOKEN' "$dockerfile"; then
2939
echo "Baseline MCore clone must not require a GitHub token" >&2
@@ -236,7 +246,12 @@ if ! grep -A1 '^te = \[$' pyproject.toml | grep -Fxq ' "megatron-core[te]",';
236246
echo "Bridge's TE extra must enable the selected MCore ref's TE extra" >&2
237247
exit 1
238248
fi
239-
grep -Fq '{ name = "megatron-core", extra = ["te"] }' uv.lock
249+
if ! grep -Fq \
250+
'{ name = "megatron-core", extras = ["te"], marker = "extra == '\''te'\''", editable = "3rdparty/Megatron-LM" }' \
251+
uv.lock; then
252+
echo "Bridge's lock metadata must preserve the MCore TE extra" >&2
253+
exit 1
254+
fi
240255
if grep -q 'transformer-engine @ git+https://github.com/NVIDIA/TransformerEngine.git@' pyproject.toml || \
241256
grep -q '^name = "transformer-engine"$' pyproject.toml; then
242257
echo "Bridge must inherit the TransformerEngine source and metadata from the selected MCore ref" >&2

‎.github/workflows/cicd-main.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ env:
6767

6868
jobs:
6969
pre-flight:
70-
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@c1a0837f362a1a696e647238ab1cf916b2a7cf4a # v1.8.6
70+
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@cf5acebace78c7c339bc1f804357a991336f1c4d # v1.8.10
7171
with:
7272
default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }}
7373
non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }}
@@ -693,7 +693,7 @@ jobs:
693693
local gpu_count
694694
gpu_count=$(grep -m1 '^# GPU_COUNT=' "$f" | cut -d= -f2)
695695
if [ -n "$gpu_count" ]; then
696-
echo "${RUNNER_PREFIX}" | sed "s/gpu-x[0-9]*/gpu-${gpu_count}/"
696+
echo "${RUNNER_PREFIX}" | sed "s/gpu-x[0-9]*/gpu-x${gpu_count}/"
697697
else
698698
echo "${RUNNER_PREFIX}"
699699
fi

‎.github/workflows/config/.secrets.baseline‎

Lines changed: 17 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.github/workflows/copyright-check.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ on:
2323

2424
jobs:
2525
pre-flight:
26-
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@c1a0837f362a1a696e647238ab1cf916b2a7cf4a # v1.8.6
26+
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@cf5acebace78c7c339bc1f804357a991336f1c4d # v1.8.10
2727

2828
copyright-check:
2929
needs: [pre-flight]

‎.github/workflows/install-test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ concurrency:
5757

5858
jobs:
5959
pre-flight:
60-
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@c1a0837f362a1a696e647238ab1cf916b2a7cf4a # v1.8.6
60+
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@cf5acebace78c7c339bc1f804357a991336f1c4d # v1.8.10
6161
with:
6262
default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }}
6363
non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }}

‎.github/workflows/release.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ concurrency:
7070

7171
jobs:
7272
pre-flight:
73-
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@c1a0837f362a1a696e647238ab1cf916b2a7cf4a # v1.8.6
73+
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@cf5acebace78c7c339bc1f804357a991336f1c4d # v1.8.10
7474
with:
7575
default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }}
7676
non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }}

‎.main.commit‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
14346b65a2d0790e451919858f7771078105c5f0
1+
2a75ac12c54ba6a42b34024756820bf819a2e25f

‎3rdparty/Megatron-LM‎

Submodule Megatron-LM updated 38 files

‎docker/Dockerfile.fw_final‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
ARG NEMO_FW_FINAL_BASE_IMAGE
1616
FROM ${NEMO_FW_FINAL_BASE_IMAGE} AS nemo_fw_final
1717

18+
# The CI base image runs as UID 1000, while framework assembly writes to root-owned system paths.
19+
USER root
20+
1821
WORKDIR /opt
1922

2023
##############################################################################

0 commit comments

Comments
 (0)