Skip to content

Commit 9564a60

Browse files
Merge branch 'master' into fix-s3-sse-kms-validate-path
2 parents 74d1742 + 1a3cecf commit 9564a60

1,068 files changed

Lines changed: 47134 additions & 21850 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci-health-v3-master.yml‎

Lines changed: 24 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ on:
88
permissions:
99
id-token: write # This is required for requesting the JWT
1010

11+
# Credentials are requested for 6h (the GithubActionsRole max session duration)
12+
# so the aws-codebuild-run-build step can keep polling for the whole build. The
13+
# canary CodeBuild timeout is set just under this (5.5h in CDK) so a hung build
14+
# is recorded by CodeBuild as TIMED_OUT (counted in FailedBuilds) instead of the
15+
# GitHub job being cancelled first.
1116
jobs:
1217
unit-test-v3:
1318
runs-on: ubuntu-latest
@@ -21,7 +26,7 @@ jobs:
2126
with:
2227
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
2328
aws-region: us-west-2
24-
role-duration-seconds: 10800
29+
role-duration-seconds: 21600
2530
- name: Run Unit Tests V3 for ${{ matrix.submodule }}
2631
uses: aws-actions/aws-codebuild-run-build@v1
2732
with:
@@ -31,8 +36,13 @@ jobs:
3136
SUBMODULE
3237
env:
3338
SUBMODULE: ${{ matrix.submodule }}
39+
# Per-submodule integ canaries. These also cover the us-west-2 gpu_intensive
40+
# tests (train, serve) and, in sagemaker-serve, the Bedrock import_model tests
41+
# (run in a dedicated serial step inside the canary buildspec because the
42+
# "Concurrent model import jobs" quota is fixed at 1).
3443
canaries-v3-master:
3544
runs-on: ubuntu-latest
45+
timeout-minutes: 360
3646
strategy:
3747
fail-fast: false
3848
matrix:
@@ -43,7 +53,7 @@ jobs:
4353
with:
4454
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
4555
aws-region: us-west-2
46-
role-duration-seconds: 10800
56+
role-duration-seconds: 21600
4757
- name: Run Canaries V3 master for ${{ matrix.submodule }}
4858
uses: aws-actions/aws-codebuild-run-build@v1
4959
with:
@@ -53,24 +63,9 @@ jobs:
5363
SUBMODULE
5464
env:
5565
SUBMODULE: ${{ matrix.submodule }}
56-
# GPU integ tests against the master source. Two region jobs trigger the same
57-
# CodeBuild project name (deployed in both regions); only the region/creds and
58-
# the region-override buildspec differ. Their results are AND-ed with
59-
# import-model in report-result into a single per-run pass/fail metric.
60-
gpu-integ-tests-master:
61-
runs-on: ubuntu-latest
62-
steps:
63-
- name: Configure AWS Credentials
64-
uses: aws-actions/configure-aws-credentials@v4
65-
with:
66-
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
67-
aws-region: us-west-2
68-
role-duration-seconds: 10800
69-
- name: Run GPU Integ Tests (master)
70-
uses: aws-actions/aws-codebuild-run-build@v1
71-
with:
72-
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-master
73-
source-version: refs/heads/master
66+
# us_east_1-marked GPU integ tests against the master source. These run in the
67+
# separate us-east-1 (Nova) account, so they cannot be part of the us-west-2
68+
# canaries. Their result is reported to us-west-2 by report-result.
7469
gpu-integ-tests-master-us-east-1:
7570
runs-on: ubuntu-latest
7671
steps:
@@ -79,39 +74,21 @@ jobs:
7974
with:
8075
role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }}
8176
aws-region: us-east-1
82-
role-duration-seconds: 10800
77+
role-duration-seconds: 21600
8378
- name: Run GPU Integ Tests (master, us-east-1)
8479
uses: aws-actions/aws-codebuild-run-build@v1
8580
with:
8681
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-master
8782
source-version: refs/heads/master
88-
# Bedrock model-import integ tests. Run serially (concurrency 1) in their own
89-
# CodeBuild project because the "Concurrent model import jobs" Bedrock quota is
90-
# fixed at 1 and not raisable; running them in parallel (as PR checks did)
91-
# makes them collide and flake. us-west-2 only (no us_east_1-marked tests).
92-
# Folded into the master run-level pass/fail metric alongside the GPU jobs.
93-
import-model-integ-tests:
94-
runs-on: ubuntu-latest
95-
steps:
96-
- name: Configure AWS Credentials
97-
uses: aws-actions/configure-aws-credentials@v4
98-
with:
99-
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
100-
aws-region: us-west-2
101-
role-duration-seconds: 10800
102-
- name: Run Bedrock Model-Import Integ Tests
103-
uses: aws-actions/aws-codebuild-run-build@v1
104-
with:
105-
project-name: sagemaker-python-sdk-ci-health-import-model-integ-tests
106-
source-version: refs/heads/master
107-
# Run-level result: the master GPU run is successful only if the us-west-2,
108-
# us-east-1, and import-model jobs all succeeded. Emits GpuIntegMasterRunFailure
109-
# = 1 (failed) / 0 (succeeded) to CloudWatch in us-west-2. The CDK alarm
83+
# Run-level result for the us-east-1 GPU job. Its CodeBuild FailedBuilds metric
84+
# lives in the us-east-1 account, which the us-west-2 CIHealthCompositeAlarm-V3
85+
# cannot see, so the result is re-emitted to us-west-2 as
86+
# GpuIntegMasterRunFailure = 1 (failed) / 0 (succeeded). The CDK alarm
11087
# (GpuIntegMasterRunAlarm) sums this over a UTC day; it is folded into
11188
# CIHealthCompositeAlarm-V3, which owns ticketing. Only scheduled runs emit the
11289
# metric (manual workflow_dispatch runs are excluded).
11390
report-result:
114-
needs: [gpu-integ-tests-master, gpu-integ-tests-master-us-east-1, import-model-integ-tests]
91+
needs: [gpu-integ-tests-master-us-east-1]
11592
if: always()
11693
runs-on: ubuntu-latest
11794
steps:
@@ -129,14 +106,12 @@ jobs:
129106
echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission."
130107
exit 0
131108
fi
132-
if [ "${{ needs.gpu-integ-tests-master.result }}" == "success" ] && \
133-
[ "${{ needs.gpu-integ-tests-master-us-east-1.result }}" == "success" ] && \
134-
[ "${{ needs.import-model-integ-tests.result }}" == "success" ]; then
109+
if [ "${{ needs.gpu-integ-tests-master-us-east-1.result }}" == "success" ]; then
135110
value=0
136-
echo "All master GPU/import jobs succeeded; emitting GpuIntegMasterRunFailure=0"
111+
echo "us-east-1 master GPU job succeeded; emitting GpuIntegMasterRunFailure=0"
137112
else
138113
value=1
139-
echo "At least one master GPU/import job did not succeed; emitting GpuIntegMasterRunFailure=1"
114+
echo "us-east-1 master GPU job did not succeed; emitting GpuIntegMasterRunFailure=1"
140115
fi
141116
aws cloudwatch put-metric-data \
142117
--namespace GpuIntegRunMetrics \

‎.github/workflows/ci-health-v3-release.yml‎

Lines changed: 24 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,17 @@ on:
88
permissions:
99
id-token: write # This is required for requesting the JWT
1010

11+
# Credentials are requested for 6h (the GithubActionsRole max session duration)
12+
# so the aws-codebuild-run-build step can keep polling for the whole build. The
13+
# canary CodeBuild timeout is set just under this (5.5h in CDK) so a hung build
14+
# is recorded by CodeBuild as TIMED_OUT (counted in FailedBuilds) instead of the
15+
# GitHub job being cancelled first.
1116
jobs:
17+
# Per-submodule integ canaries against the published PyPI release. These also
18+
# cover the us-west-2 gpu_intensive tests (train, serve).
1219
canaries-v3-release:
1320
runs-on: ubuntu-latest
21+
timeout-minutes: 360
1422
strategy:
1523
fail-fast: false
1624
matrix:
@@ -21,7 +29,7 @@ jobs:
2129
with:
2230
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
2331
aws-region: us-west-2
24-
role-duration-seconds: 10800
32+
role-duration-seconds: 21600
2533
- name: Run Canaries V3 release for ${{ matrix.submodule }}
2634
uses: aws-actions/aws-codebuild-run-build@v1
2735
with:
@@ -31,24 +39,9 @@ jobs:
3139
SUBMODULE
3240
env:
3341
SUBMODULE: ${{ matrix.submodule }}
34-
# GPU integ tests against the published PyPI release. Two region jobs trigger
35-
# the same CodeBuild project name (deployed in both regions); only the
36-
# region/creds and the region-override buildspec differ. Their results are
37-
# AND-ed in report-result into a single per-run pass/fail metric.
38-
gpu-integ-tests-release:
39-
runs-on: ubuntu-latest
40-
steps:
41-
- name: Configure AWS Credentials
42-
uses: aws-actions/configure-aws-credentials@v4
43-
with:
44-
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
45-
aws-region: us-west-2
46-
role-duration-seconds: 10800
47-
- name: Run GPU Integ Tests (release)
48-
uses: aws-actions/aws-codebuild-run-build@v1
49-
with:
50-
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-release
51-
source-version: refs/heads/master
42+
# us_east_1-marked GPU integ tests against the published PyPI release. These
43+
# run in the separate us-east-1 (Nova) account, so they cannot be part of the
44+
# us-west-2 canaries. Their result is reported to us-west-2 by report-result.
5245
gpu-integ-tests-release-us-east-1:
5346
runs-on: ubuntu-latest
5447
steps:
@@ -57,20 +50,21 @@ jobs:
5750
with:
5851
role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }}
5952
aws-region: us-east-1
60-
role-duration-seconds: 10800
53+
role-duration-seconds: 21600
6154
- name: Run GPU Integ Tests (release, us-east-1)
6255
uses: aws-actions/aws-codebuild-run-build@v1
6356
with:
6457
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-release
6558
source-version: refs/heads/master
66-
# Run-level result: the release GPU run is successful only if both region jobs
67-
# succeeded. Emits GpuIntegReleaseRunFailure = 1 (failed) / 0 (succeeded) to
68-
# CloudWatch in us-west-2. The CDK alarm (GpuIntegReleaseRunAlarm) sums this
69-
# over a UTC day; it is folded into CIHealthCompositeAlarm-V3, which owns
70-
# ticketing. Only scheduled runs emit the metric (manual workflow_dispatch runs
71-
# are excluded).
59+
# Run-level result for the us-east-1 GPU job. Its CodeBuild FailedBuilds metric
60+
# lives in the us-east-1 account, which the us-west-2 CIHealthCompositeAlarm-V3
61+
# cannot see, so the result is re-emitted to us-west-2 as
62+
# GpuIntegReleaseRunFailure = 1 (failed) / 0 (succeeded). The CDK alarm
63+
# (GpuIntegReleaseRunAlarm) sums this over a UTC day; it is folded into
64+
# CIHealthCompositeAlarm-V3, which owns ticketing. Only scheduled runs emit the
65+
# metric (manual workflow_dispatch runs are excluded).
7266
report-result:
73-
needs: [gpu-integ-tests-release, gpu-integ-tests-release-us-east-1]
67+
needs: [gpu-integ-tests-release-us-east-1]
7468
if: always()
7569
runs-on: ubuntu-latest
7670
steps:
@@ -88,13 +82,12 @@ jobs:
8882
echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission."
8983
exit 0
9084
fi
91-
if [ "${{ needs.gpu-integ-tests-release.result }}" == "success" ] && \
92-
[ "${{ needs.gpu-integ-tests-release-us-east-1.result }}" == "success" ]; then
85+
if [ "${{ needs.gpu-integ-tests-release-us-east-1.result }}" == "success" ]; then
9386
value=0
94-
echo "All release GPU jobs succeeded; emitting GpuIntegReleaseRunFailure=0"
87+
echo "us-east-1 release GPU job succeeded; emitting GpuIntegReleaseRunFailure=0"
9588
else
9689
value=1
97-
echo "At least one release GPU job did not succeed; emitting GpuIntegReleaseRunFailure=1"
90+
echo "us-east-1 release GPU job did not succeed; emitting GpuIntegReleaseRunFailure=1"
9891
fi
9992
aws cloudwatch put-metric-data \
10093
--namespace GpuIntegRunMetrics \
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: SageMaker Core - Daily Sync with Botocore
2+
3+
# Scheduled trigger that kicks off the CodeBuild project which fetches the
4+
# latest service-2.json models from boto3/botocore, regenerates the
5+
# sagemaker-core resource/shape classes, and opens a "Daily Sync with Botocore"
6+
# PR as the sagemaker-bot user.
7+
#
8+
# NOTE: The CodeBuild project `sagemaker-core-botocore-sync` is provisioned
9+
# separately (deferred account-side setup). It reuses the repo's existing
10+
# `CI_AWS_ROLE_ARN` OIDC role (same role every other CI workflow uses); that
11+
# role must be granted codebuild:StartBuild on this project. Until the project
12+
# exists, this workflow will fail on the Run CodeBuild step.
13+
14+
on:
15+
schedule:
16+
# Every Monday to Friday at 10:00 UTC (3:00 PDT)
17+
- cron: "00 10 * * 1-5"
18+
# Allow manual runs for testing once CodeBuild is wired up.
19+
workflow_dispatch:
20+
21+
permissions:
22+
id-token: write # Required for requesting the OIDC JWT
23+
24+
jobs:
25+
sync-with-botocore:
26+
runs-on: ubuntu-latest
27+
steps:
28+
- name: Configure AWS Credentials
29+
uses: aws-actions/configure-aws-credentials@v4
30+
with:
31+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
32+
role-duration-seconds: 10800
33+
aws-region: us-west-2
34+
35+
- name: Run CodeBuild
36+
uses: aws-actions/aws-codebuild-run-build@v1
37+
with:
38+
project-name: sagemaker-core-botocore-sync

‎.pydocstylerc‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# pydocstyle configuration shared by all four submodules. Each submodule's tox
2+
# ``docstyle`` env runs ``pydocstyle src/sagemaker`` from the submodule root;
3+
# pydocstyle walks up from the checked files and picks this file up.
4+
#
5+
# Restored from the v2 (master-v2) branch, where it lived at the repo root and
6+
# was lost in the v3 monorepo split. The generated API surface
7+
# (``sagemaker/core/resources.py`` and ``sagemaker/core/shapes/shapes.py``)
8+
# is excluded: it is produced by the code generator, not hand-written.
9+
[pydocstyle]
10+
inherit = false
11+
ignore = D104,D107,D202,D203,D213,D214,D400,D401,D404,D406,D407,D411,D413,D414,D415,D417
12+
match = (?!record_pb2)(?!resources\.py$)(?!shapes\.py$).*\.py
13+
match-dir = (?!.*test).*

‎.pylintrc‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,16 @@ ignore=CVS,tensorflow_serving
2525
ignore-patterns=
2626
.*_pb2.py, # Ignore all files generated by the protocol buffer compiler
2727

28+
# Add files or directories matching the regex patterns to the ignore list. The
29+
# regex matches against paths and can be comma(and newline)-separated. Use
30+
# forward slashes and ``[.]`` rather than ``\.``: pylint rewrites backslashes to
31+
# build the Windows variant of each pattern. The sagemaker-core API surface
32+
# below is emitted by the code generator and is not hand-maintained, so it is
33+
# not linted.
34+
ignore-paths=
35+
^.*/sagemaker/core/resources[.]py$,
36+
^.*/sagemaker/core/shapes/shapes[.]py$
37+
2838
# Pickle collected data for later comparisons.
2939
persistent=yes
3040

@@ -109,9 +119,12 @@ disable=
109119
W0237, # Argument renamed in override
110120
W0613, # Unused argument
111121
W0621, # Redefining name from outer scope
112-
W0719
122+
W0718, # Broad exception caught: the SDK deliberately catches Exception at many boundaries
123+
W0719, # Broad exception raised
124+
W1203, # Logging f-string interpolation: the v3 code base uses f-strings in log calls throughout
113125
W1404, # Implicit string concatenation
114126
W1514, # `open()` used without encoding
127+
R0801, # Duplicate code: the v3 split intentionally mirrors helpers across submodules
115128

116129
[REPORTS]
117130
# Set the output format. Available formats are text, parseable, colorized, msvs
@@ -245,8 +258,10 @@ bad-functions=
245258
max-nested-blocks=5
246259

247260
[FORMAT]
248-
# Maximum number of characters on a single line.
249-
max-line-length=100
261+
# Maximum number of characters on a single line. Black formats to 100 but will
262+
# not split long strings and comments; flake8 (tox.ini) accepts 120 for the same
263+
# reason, so pylint is aligned with it rather than flagging black's output.
264+
max-line-length=120
250265

251266
# Regexp for a line that is allowed to be longer than the limit. Can only be a single regex.
252267
# The following matches any semblance of a url of any sort.

0 commit comments

Comments
 (0)