88permissions :
99 id-token : write # This is required for requesting the JWT
1010
11+ # Credentials are requested for 6h (the GithubActionsRole max session duration)
12+ # so the aws-codebuild-run-build step can keep polling for the whole build. The
13+ # canary CodeBuild timeout is set just under this (5.5h in CDK) so a hung build
14+ # is recorded by CodeBuild as TIMED_OUT (counted in FailedBuilds) instead of the
15+ # GitHub job being cancelled first.
1116jobs :
1217 unit-test-v3 :
1318 runs-on : ubuntu-latest
2126 with :
2227 role-to-assume : ${{ secrets.CI_AWS_ROLE_ARN }}
2328 aws-region : us-west-2
24- role-duration-seconds : 10800
29+ role-duration-seconds : 21600
2530 - name : Run Unit Tests V3 for ${{ matrix.submodule }}
2631 uses : aws-actions/aws-codebuild-run-build@v1
2732 with :
3136 SUBMODULE
3237 env :
3338 SUBMODULE : ${{ matrix.submodule }}
39+ # Per-submodule integ canaries. These also cover the us-west-2 gpu_intensive
40+ # tests (train, serve) and, in sagemaker-serve, the Bedrock import_model tests
41+ # (run in a dedicated serial step inside the canary buildspec because the
42+ # "Concurrent model import jobs" quota is fixed at 1).
3443 canaries-v3-master :
3544 runs-on : ubuntu-latest
45+ timeout-minutes : 360
3646 strategy :
3747 fail-fast : false
3848 matrix :
4353 with :
4454 role-to-assume : ${{ secrets.CI_AWS_ROLE_ARN }}
4555 aws-region : us-west-2
46- role-duration-seconds : 10800
56+ role-duration-seconds : 21600
4757 - name : Run Canaries V3 master for ${{ matrix.submodule }}
4858 uses : aws-actions/aws-codebuild-run-build@v1
4959 with :
5363 SUBMODULE
5464 env :
5565 SUBMODULE : ${{ matrix.submodule }}
56- # GPU integ tests against the master source. Two region jobs trigger the same
57- # CodeBuild project name (deployed in both regions); only the region/creds and
58- # the region-override buildspec differ. Their results are AND-ed with
59- # import-model in report-result into a single per-run pass/fail metric.
60- gpu-integ-tests-master :
61- runs-on : ubuntu-latest
62- steps :
63- - name : Configure AWS Credentials
64- uses : aws-actions/configure-aws-credentials@v4
65- with :
66- role-to-assume : ${{ secrets.CI_AWS_ROLE_ARN }}
67- aws-region : us-west-2
68- role-duration-seconds : 10800
69- - name : Run GPU Integ Tests (master)
70- uses : aws-actions/aws-codebuild-run-build@v1
71- with :
72- project-name : sagemaker-python-sdk-ci-health-gpu-integ-tests-master
73- source-version : refs/heads/master
66+ # us_east_1-marked GPU integ tests against the master source. These run in the
67+ # separate us-east-1 (Nova) account, so they cannot be part of the us-west-2
68+ # canaries. Their result is reported to us-west-2 by report-result.
7469 gpu-integ-tests-master-us-east-1 :
7570 runs-on : ubuntu-latest
7671 steps :
@@ -79,39 +74,21 @@ jobs:
7974 with :
8075 role-to-assume : ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }}
8176 aws-region : us-east-1
82- role-duration-seconds : 10800
77+ role-duration-seconds : 21600
8378 - name : Run GPU Integ Tests (master, us-east-1)
8479 uses : aws-actions/aws-codebuild-run-build@v1
8580 with :
8681 project-name : sagemaker-python-sdk-ci-health-gpu-integ-tests-master
8782 source-version : refs/heads/master
88- # Bedrock model-import integ tests. Run serially (concurrency 1) in their own
89- # CodeBuild project because the "Concurrent model import jobs" Bedrock quota is
90- # fixed at 1 and not raisable; running them in parallel (as PR checks did)
91- # makes them collide and flake. us-west-2 only (no us_east_1-marked tests).
92- # Folded into the master run-level pass/fail metric alongside the GPU jobs.
93- import-model-integ-tests :
94- runs-on : ubuntu-latest
95- steps :
96- - name : Configure AWS Credentials
97- uses : aws-actions/configure-aws-credentials@v4
98- with :
99- role-to-assume : ${{ secrets.CI_AWS_ROLE_ARN }}
100- aws-region : us-west-2
101- role-duration-seconds : 10800
102- - name : Run Bedrock Model-Import Integ Tests
103- uses : aws-actions/aws-codebuild-run-build@v1
104- with :
105- project-name : sagemaker-python-sdk-ci-health-import-model-integ-tests
106- source-version : refs/heads/master
107- # Run-level result: the master GPU run is successful only if the us-west-2,
108- # us-east-1, and import-model jobs all succeeded. Emits GpuIntegMasterRunFailure
109- # = 1 (failed) / 0 (succeeded) to CloudWatch in us-west-2. The CDK alarm
83+ # Run-level result for the us-east-1 GPU job. Its CodeBuild FailedBuilds metric
84+ # lives in the us-east-1 account, which the us-west-2 CIHealthCompositeAlarm-V3
85+ # cannot see, so the result is re-emitted to us-west-2 as
86+ # GpuIntegMasterRunFailure = 1 (failed) / 0 (succeeded). The CDK alarm
11087 # (GpuIntegMasterRunAlarm) sums this over a UTC day; it is folded into
11188 # CIHealthCompositeAlarm-V3, which owns ticketing. Only scheduled runs emit the
11289 # metric (manual workflow_dispatch runs are excluded).
11390 report-result :
114- needs : [gpu-integ-tests-master, gpu-integ-tests-master- us-east-1, import-model-integ-tests ]
91+ needs : [gpu-integ-tests-master- us-east-1]
11592 if : always()
11693 runs-on : ubuntu-latest
11794 steps :
@@ -129,14 +106,12 @@ jobs:
129106 echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission."
130107 exit 0
131108 fi
132- if [ "${{ needs.gpu-integ-tests-master.result }}" == "success" ] && \
133- [ "${{ needs.gpu-integ-tests-master-us-east-1.result }}" == "success" ] && \
134- [ "${{ needs.import-model-integ-tests.result }}" == "success" ]; then
109+ if [ "${{ needs.gpu-integ-tests-master-us-east-1.result }}" == "success" ]; then
135110 value=0
136- echo "All master GPU/import jobs succeeded; emitting GpuIntegMasterRunFailure=0"
111+ echo "us-east-1 master GPU job succeeded; emitting GpuIntegMasterRunFailure=0"
137112 else
138113 value=1
139- echo "At least one master GPU/import job did not succeed; emitting GpuIntegMasterRunFailure=1"
114+ echo "us-east-1 master GPU job did not succeed; emitting GpuIntegMasterRunFailure=1"
140115 fi
141116 aws cloudwatch put-metric-data \
142117 --namespace GpuIntegRunMetrics \
0 commit comments