Skip to content

Commit 6e8600f

Browse files
svcnemo-autobotsvcnemo-autobot
committed
Merge commit '538f38e1eb59c4d8179a1838e7285b1eab1eea58' into chore/bump-mcore-dev-260812
2 parents 6696b2e + 538f38e commit 6e8600f

39 files changed

Lines changed: 1746 additions & 173 deletions

File tree

‎.github/CODEOWNERS‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@ pyproject.toml @nvidia-nemo/automation
55
ruff.toml @nvidia-nemo/automation
66
.pre-commit-config.yaml
77

8-
scripts/performance/argument_parser.py @nvidia-nemo/automation @malay-nagda @erhoo82
9-
scripts/performance/setup_experiment.py @nvidia-nemo/automation @malay-nagda @erhoo82
10-
scripts/performance/bootstrap.py @nvidia-nemo/automation @malay-nagda @erhoo82
11-
scripts/performance/run_script.py @nvidia-nemo/automation @malay-nagda @erhoo82
12-
scripts/performance/run_recipe.py @nvidia-nemo/automation @malay-nagda @erhoo82
8+
scripts/performance @malay-nagda @dingqingy-nv
9+
src/megatron/bridge/perf_recipes @malay-nagda @dingqingy-nv
10+
11+
scripts/performance/argument_parser.py @nvidia-nemo/automation
12+
scripts/performance/setup_experiment.py @nvidia-nemo/automation
13+
scripts/performance/bootstrap.py @nvidia-nemo/automation
14+
scripts/performance/run_script.py @nvidia-nemo/automation
15+
scripts/performance/run_recipe.py @nvidia-nemo/automation

‎.github/actions/test-template/action.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ inputs:
4343
description: "Has Azure credentials"
4444
required: false
4545
default: "false"
46-
PAT:
47-
description: "GitHub Personal Access Token"
46+
github-token:
47+
description: "GitHub token"
4848
required: true
4949
container-image:
5050
description: "Container image to use for test"
@@ -159,7 +159,7 @@ runs:
159159
160160
docker exec -t nemo_container_${{ github.run_id }} bash -c '
161161
set -e
162-
export GH_TOKEN=${{ inputs.PAT }}
162+
export GH_TOKEN=${{ inputs.github-token }}
163163
164164
timeout $(( ${{ inputs.timeout }} * 60 ))s bash tests/${{ inputs.is_unit_test == 'true' && 'unit_tests' || format('functional_tests/launch_scripts/{0}', inputs.script_dir) }}/${{ inputs.script }}.sh || EXIT_CODE=$?
165165

‎.github/workflows/cicd-main.yml‎

Lines changed: 102 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ jobs:
9090
expect_l1: ${{ steps.configure.outputs.expect_l1 }}
9191
expect_l2: ${{ steps.configure.outputs.expect_l2 }}
9292
perf_scripts_only: ${{ steps.configure.outputs.perf_scripts_only }}
93+
skip_queue: ${{ steps.configure.outputs.skip_queue }}
9394
merge_sha: ${{ steps.merge-sha.outputs.merge_sha }}
9495
steps:
9596
- name: Get PR info
@@ -122,6 +123,7 @@ jobs:
122123
EVENT_NAME: ${{ github.event_name }}
123124
REF: ${{ github.ref }}
124125
TEST_SUITE: ${{ github.event.inputs.test_suite }}
126+
PR_HEAD_LABEL: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').head.label || '' }}
125127
run: |
126128
PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }}
127129
@@ -131,6 +133,9 @@ jobs:
131133
NEEDS_MORE_TESTS=$(echo "$LABELS" | jq 'any(. == "needs-more-tests")')
132134
FULL_TEST_SUITE=$(echo "$LABELS" | jq 'any(. == "full-test-suite")')
133135
136+
SKIP_QUEUE=false
137+
[[ "$PR_HEAD_LABEL" == svcnemo-autobot:chore/bump-mcore-* ]] && SKIP_QUEUE=true
138+
134139
# Detect if every changed file lives under scripts/performance/
135140
PERF_SCRIPTS_ONLY=false
136141
if [[ -n "$PR_NUMBER" ]]; then
@@ -173,6 +178,7 @@ jobs:
173178
echo "docs_only=$DOCS_ONLY" | tee -a "$GITHUB_OUTPUT"
174179
echo "needs_more_tests=$NEEDS_MORE_TESTS" | tee -a "$GITHUB_OUTPUT"
175180
echo "full_test_suite=$FULL_TEST_SUITE" | tee -a "$GITHUB_OUTPUT"
181+
echo "skip_queue=$SKIP_QUEUE" | tee -a "$GITHUB_OUTPUT"
176182
echo "expect_l0=$EXPECT_L0" | tee -a "$GITHUB_OUTPUT"
177183
echo "expect_l1=$EXPECT_L1" | tee -a "$GITHUB_OUTPUT"
178184
echo "expect_l2=$EXPECT_L2" | tee -a "$GITHUB_OUTPUT"
@@ -296,7 +302,8 @@ jobs:
296302
!(needs.pre-flight.outputs.is_ci_workload == 'true'
297303
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
298304
|| needs.configure.outputs.docs_only == 'true'
299-
|| needs.configure.outputs.perf_scripts_only == 'true')
305+
|| needs.configure.outputs.perf_scripts_only == 'true'
306+
|| needs.configure.outputs.skip_queue == 'true')
300307
&& github.event_name != 'merge_group'
301308
steps:
302309
- name: Running CI tests
@@ -310,6 +317,7 @@ jobs:
310317
(
311318
success()
312319
|| needs.pre-flight.outputs.is_ci_workload == 'true'
320+
|| needs.configure.outputs.skip_queue == 'true'
313321
|| needs.pre-flight.outputs.force_run_all == 'true'
314322
|| github.event_name == 'merge_group'
315323
)
@@ -340,7 +348,7 @@ jobs:
340348
echo "matrix=$MATRIX" | tee -a "$GITHUB_OUTPUT"
341349
342350
cicd-container-build:
343-
needs: [pre-flight, configure, cicd-wait-in-queue, cicd-compute-build-matrix]
351+
needs: [pre-flight, configure, cicd-compute-build-matrix]
344352
strategy:
345353
fail-fast: false
346354
matrix: ${{ fromJson(needs.cicd-compute-build-matrix.outputs.matrix) }}
@@ -523,8 +531,6 @@ jobs:
523531
tags: |
524532
${{ matrix.registry }}/megatron-bridge:${{ steps.cache_keys.outputs.key }}
525533
${{ matrix.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
526-
secrets: |
527-
GH_TOKEN=${{ secrets.PAT }}
528534
529535
cicd-import-check:
530536
if: |
@@ -536,7 +542,7 @@ jobs:
536542
)
537543
&& !cancelled()
538544
&& needs.configure.outputs.perf_scripts_only != 'true'
539-
needs: [pre-flight, configure, cicd-wait-in-queue, cicd-container-build]
545+
needs: [pre-flight, configure, cicd-container-build]
540546
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
541547
name: Launch_Import_Check
542548
steps:
@@ -567,7 +573,7 @@ jobs:
567573
)
568574
&& !cancelled()
569575
&& (github.event.inputs.test_suite == '' || github.event.inputs.test_suite == 'all' || github.event.inputs.test_suite == 'unit-only' || contains('L0 L1 L2', github.event.inputs.test_suite))
570-
needs: [pre-flight, configure, cicd-wait-in-queue, cicd-container-build]
576+
needs: [pre-flight, configure, cicd-container-build]
571577
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
572578
name: Launch_Unit_Tests_Core
573579
env:
@@ -580,13 +586,20 @@ jobs:
580586
submodules: recursive
581587
ref: ${{ needs.configure.outputs.merge_sha }}
582588

589+
- name: Generate GitHub App token
590+
id: app-token
591+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
592+
with:
593+
app-id: ${{ vars.BOT_ID }}
594+
private-key: ${{ secrets.BOT_KEY }}
595+
583596
- name: main
584597
uses: ./.github/actions/test-template
585598
with:
586599
script: Launch_Unit_Tests_Core
587-
timeout: 18
600+
timeout: 30
588601
is_unit_test: "true"
589-
PAT: ${{ secrets.PAT }}
602+
github-token: ${{ steps.app-token.outputs.token }}
590603
container-image: ${{ needs.pre-flight.outputs.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
591604
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
592605
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
@@ -601,7 +614,7 @@ jobs:
601614
)
602615
&& !cancelled()
603616
&& (github.event.inputs.test_suite == '' || github.event.inputs.test_suite == 'all' || github.event.inputs.test_suite == 'unit-only' || contains('L0 L1 L2', github.event.inputs.test_suite))
604-
needs: [pre-flight, configure, cicd-wait-in-queue, cicd-container-build]
617+
needs: [pre-flight, configure, cicd-container-build]
605618
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
606619
name: Launch_Unit_Tests_Diffusion
607620
env:
@@ -614,13 +627,20 @@ jobs:
614627
submodules: recursive
615628
ref: ${{ needs.configure.outputs.merge_sha }}
616629

630+
- name: Generate GitHub App token
631+
id: app-token
632+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
633+
with:
634+
app-id: ${{ vars.BOT_ID }}
635+
private-key: ${{ secrets.BOT_KEY }}
636+
617637
- name: main
618638
uses: ./.github/actions/test-template
619639
with:
620640
script: Launch_Unit_Tests_Diffusion
621641
timeout: 18
622642
is_unit_test: "true"
623-
PAT: ${{ secrets.PAT }}
643+
github-token: ${{ steps.app-token.outputs.token }}
624644
container-image: ${{ needs.pre-flight.outputs.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
625645
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
626646
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
@@ -716,13 +736,20 @@ jobs:
716736
submodules: recursive
717737
ref: ${{ needs.configure.outputs.merge_sha }}
718738

739+
- name: Generate GitHub App token
740+
id: app-token
741+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
742+
with:
743+
app-id: ${{ vars.BOT_ID }}
744+
private-key: ${{ secrets.BOT_KEY }}
745+
719746
- name: main
720747
uses: ./.github/actions/test-template
721748
with:
722749
script: ${{ matrix.script }}
723750
timeout: ${{ fromJSON(matrix.timeout || '30') }}
724751
is_unit_test: "false"
725-
PAT: ${{ secrets.PAT }}
752+
github-token: ${{ steps.app-token.outputs.token }}
726753
container-image: ${{ needs.pre-flight.outputs.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
727754
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
728755
runner: ${{ matrix.runner }}
@@ -757,13 +784,20 @@ jobs:
757784
submodules: recursive
758785
ref: ${{ needs.configure.outputs.merge_sha }}
759786

787+
- name: Generate GitHub App token
788+
id: app-token
789+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
790+
with:
791+
app-id: ${{ vars.BOT_ID }}
792+
private-key: ${{ secrets.BOT_KEY }}
793+
760794
- name: main
761795
uses: ./.github/actions/test-template
762796
with:
763797
script: ${{ matrix.script }}
764798
timeout: ${{ fromJSON(matrix.timeout || '30') }}
765799
is_unit_test: "false"
766-
PAT: ${{ secrets.PAT }}
800+
github-token: ${{ steps.app-token.outputs.token }}
767801
container-image: ${{ needs.pre-flight.outputs.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
768802
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
769803
runner: ${{ matrix.runner }}
@@ -798,13 +832,20 @@ jobs:
798832
submodules: recursive
799833
ref: ${{ needs.configure.outputs.merge_sha }}
800834

835+
- name: Generate GitHub App token
836+
id: app-token
837+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
838+
with:
839+
app-id: ${{ vars.BOT_ID }}
840+
private-key: ${{ secrets.BOT_KEY }}
841+
801842
- name: main
802843
uses: ./.github/actions/test-template
803844
with:
804845
script: ${{ matrix.script }}
805846
timeout: ${{ fromJSON(matrix.timeout || '30') }}
806847
is_unit_test: "false"
807-
PAT: ${{ secrets.PAT }}
848+
github-token: ${{ steps.app-token.outputs.token }}
808849
container-image: ${{ needs.pre-flight.outputs.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
809850
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
810851
runner: ${{ matrix.runner }}
@@ -829,14 +870,21 @@ jobs:
829870
submodules: recursive
830871
ref: ${{ needs.configure.outputs.merge_sha }}
831872

873+
- name: Generate GitHub App token
874+
id: app-token
875+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
876+
with:
877+
app-id: ${{ vars.BOT_ID }}
878+
private-key: ${{ secrets.BOT_KEY }}
879+
832880
- name: main
833881
uses: ./.github/actions/test-template
834882
with:
835883
script_dir: h100/flaky
836884
script: ${{ matrix.script }}
837885
timeout: ${{ fromJSON(matrix.timeout || '30') }}
838886
is_unit_test: "false"
839-
PAT: ${{ secrets.PAT }}
887+
github-token: ${{ steps.app-token.outputs.token }}
840888
container-image: ${{ needs.pre-flight.outputs.registry }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
841889
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
842890
runner: ${{ matrix.runner }}
@@ -935,6 +983,13 @@ jobs:
935983
submodules: recursive
936984
ref: ${{ needs.configure.outputs.merge_sha }}
937985

986+
- name: Generate GitHub App token
987+
id: app-token
988+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
989+
with:
990+
app-id: ${{ vars.BOT_ID }}
991+
private-key: ${{ secrets.BOT_KEY }}
992+
938993
- name: main
939994
uses: ./.github/actions/test-template
940995
with:
@@ -946,7 +1001,7 @@ jobs:
9461001
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
9471002
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
9481003
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
949-
PAT: ${{ secrets.PAT }}
1004+
github-token: ${{ steps.app-token.outputs.token }}
9501005
container-image: ${{ env.container-registry-gb200 }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
9511006
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
9521007
runner: ${{ matrix.runner }}
@@ -984,6 +1039,13 @@ jobs:
9841039
submodules: recursive
9851040
ref: ${{ needs.configure.outputs.merge_sha }}
9861041

1042+
- name: Generate GitHub App token
1043+
id: app-token
1044+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
1045+
with:
1046+
app-id: ${{ vars.BOT_ID }}
1047+
private-key: ${{ secrets.BOT_KEY }}
1048+
9871049
- name: main
9881050
uses: ./.github/actions/test-template
9891051
with:
@@ -995,7 +1057,7 @@ jobs:
9951057
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
9961058
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
9971059
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
998-
PAT: ${{ secrets.PAT }}
1060+
github-token: ${{ steps.app-token.outputs.token }}
9991061
container-image: ${{ env.container-registry-gb200 }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
10001062
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
10011063
runner: ${{ matrix.runner }}
@@ -1033,6 +1095,13 @@ jobs:
10331095
submodules: recursive
10341096
ref: ${{ needs.configure.outputs.merge_sha }}
10351097

1098+
- name: Generate GitHub App token
1099+
id: app-token
1100+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
1101+
with:
1102+
app-id: ${{ vars.BOT_ID }}
1103+
private-key: ${{ secrets.BOT_KEY }}
1104+
10361105
- name: main
10371106
uses: ./.github/actions/test-template
10381107
with:
@@ -1044,7 +1113,7 @@ jobs:
10441113
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
10451114
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
10461115
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
1047-
PAT: ${{ secrets.PAT }}
1116+
github-token: ${{ steps.app-token.outputs.token }}
10481117
container-image: ${{ env.container-registry-gb200 }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
10491118
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
10501119
runner: ${{ matrix.runner }}
@@ -1070,6 +1139,13 @@ jobs:
10701139
submodules: recursive
10711140
ref: ${{ needs.configure.outputs.merge_sha }}
10721141

1142+
- name: Generate GitHub App token
1143+
id: app-token
1144+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
1145+
with:
1146+
app-id: ${{ vars.BOT_ID }}
1147+
private-key: ${{ secrets.BOT_KEY }}
1148+
10731149
- name: main
10741150
uses: ./.github/actions/test-template
10751151
with:
@@ -1081,7 +1157,7 @@ jobs:
10811157
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
10821158
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
10831159
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
1084-
PAT: ${{ secrets.PAT }}
1160+
github-token: ${{ steps.app-token.outputs.token }}
10851161
container-image: ${{ env.container-registry-gb200 }}/megatron-bridge:${{ needs.configure.outputs.merge_sha }}
10861162
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
10871163
runner: ${{ matrix.runner }}
@@ -1176,10 +1252,17 @@ jobs:
11761252
|| needs.configure.outputs.perf_scripts_only == 'true'
11771253
)
11781254
steps:
1255+
- name: Generate GitHub App token
1256+
id: app-token
1257+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
1258+
with:
1259+
app-id: ${{ vars.BOT_ID }}
1260+
private-key: ${{ secrets.BOT_KEY }}
1261+
11791262
- name: Generate fake coverage report
11801263
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
11811264
with:
1182-
github-token: ${{ secrets.PAT }}
1265+
github-token: ${{ steps.app-token.outputs.token }}
11831266
script: |
11841267
await github.rest.repos.createCommitStatus({
11851268
owner: context.repo.owner,

0 commit comments

Comments
 (0)