@@ -246,18 +246,36 @@ jobs:
246246 # Additive: runs the shallow (submit-then-stop) suite for sagemaker-train
247247 # alongside the existing integ-tests job above, which is unchanged.
248248 #
249- # Why a separate job rather than folding this into the CodeBuild suite: this
250- # job's selection is reviewable in the PR that changes it, whereas the
251- # sagemaker-train CodeBuild buildspec is CDK-managed outside this repo. It also
252- # reports as its own check, so a shallow failure is distinguishable at a glance
253- # from a deep-suite failure, and it finishes in minutes -- fast feedback that
254- # does not wait on the 2XLARGE CodeBuild container.
249+ # Runs in CodeBuild, not on the runner. It began as a runner job, but
250+ # actions/checkout refuses to place a fork's head commit in a
251+ # pull_request_target job -- correctly, because the runner also holds the base
252+ # repo's GITHUB_TOKEN and assumes CI_AWS_ROLE_ARN, so a fork could edit
253+ # conftest.py and read those credentials out. That is the "pwn request" shape,
254+ # and on a public repo it is a live credential-exfiltration path, so the fix is
255+ # to move the execution rather than override the refusal with
256+ # allow-unsafe-pr-checkout. Nearly every PR here comes from a fork, so a
257+ # same-repo guard would have left the suite with almost no gate coverage.
258+ # source-version-override is how the three jobs above already run PR code: the
259+ # build never sees the runner's token, secrets or default-branch cache.
255260 #
256- # What runs here: only tests/integ/train/shallow. The client-side tests
257- # (recipe resolution, data utils, dry-run, log streaming) are deliberately NOT
258- # repeated -- the CodeBuild suite already runs the whole tests/integ tree, so
259- # widening this job's scope would duplicate them and double the job creation
260- # the shallow suite performs.
261+ # Why its own project rather than folding this into the sagemaker-train
262+ # integ-tests project: it reports as its own check, so a shallow failure is
263+ # distinguishable at a glance from a deep-suite failure, and it runs
264+ # concurrently with the deep suite instead of queueing behind it.
265+ #
266+ # Tradeoff of moving off the runner: the pytest selection now lives in the
267+ # CDK's buildspecs.ts (createCIShallowIntegBuildSpec) instead of this file, so
268+ # changing which tests run is no longer reviewable in a PR to this repo. That
269+ # is the price of executing fork code safely, and it is the same place the
270+ # other three test jobs' selections already live.
271+ #
272+ # What runs there: only tests/integ/train/shallow, deselecting gpu_intensive
273+ # (the CPT and MTRL classes, which need a pre-provisioned HyperPod cluster and
274+ # an agent runtime plus an MLflow app) and us_east_1 (Nova cases, which run in
275+ # the integ-tests-us-east-1 project against the Nova account). The client-side
276+ # tests are deliberately not repeated -- the deep suite already runs the whole
277+ # tests/integ tree, so widening scope would duplicate them and double the job
278+ # creation the shallow suite performs.
261279 #
262280 # Why submit-then-stop is worth gating on: CreateTrainingJob returns a
263281 # TrainingJobArn only after the request has cleared public-model validation,
@@ -272,88 +290,22 @@ jobs:
272290 fast-integ-tests :
273291 runs-on : ubuntu-latest
274292 needs : [detect-changes]
293+ # No same-repo guard: nothing here checks out PR code, so fork PRs are gated
294+ # too. The suite only runs when sagemaker-train is in the change set.
275295 if : contains(fromJson(needs.detect-changes.outputs.submodules), 'sagemaker-train')
276296 steps :
277- - uses : actions/checkout@v3
278- with :
279- # pull_request_target checks out the base ref by default; these tests
280- # must run against the PR's code.
281- ref : ' refs/pull/${{ github.event.pull_request.number }}/head'
282-
283- - name : Set up Python
284- uses : actions/setup-python@v5
285- with :
286- python-version : ' 3.12'
287-
288297 - name : Configure AWS Credentials
289298 uses : aws-actions/configure-aws-credentials@v4
290299 with :
291300 role-to-assume : ${{ secrets.CI_AWS_ROLE_ARN }}
292301 aws-region : us-west-2
293302 role-duration-seconds : 10800
294303
295- - name : Install sagemaker-train and test dependencies
296- run : |
297- python -m pip install --upgrade pip
298- pip install ./sagemaker-core
299- pip install ./sagemaker-train
300- pip install -r requirements/extras/test_requirements.txt
301-
302304 - name : Run shallow sagemaker-train integ tests
303- working-directory : sagemaker-train
304- env :
305- AWS_DEFAULT_REGION : us-west-2
306- # Role resolution goes through iam:SimulatePrincipalPolicy, which is
307- # low-TPS; adaptive retries keep parallel workers from throttling each
308- # other.
309- AWS_RETRY_MODE : adaptive
310- AWS_MAX_ATTEMPTS : ' 10'
311- # Cap the training jobs the *service* counts against the concurrency
312- # quota, across all xdist workers, so the suite stays inside the
313- # "concurrent model customization serverless jobs per Region" quota
314- # (20) with room for the deep integ-tests suite running the same
315- # account concurrently. The harness holds each slot until the job is
316- # terminal, not until stop() returns -- the service counts a job for
317- # ~1-3 min after the stop -- so 10 means "at most 10 jobs counted at
318- # once", the batches-of-10 behaviour, not "10 stops in flight".
319- #
320- # Not redundant with -n 8. -n caps worker processes; this caps what
321- # the service counts, and with the slot held to terminal those diverge
322- # sharply (each drain outlives the worker's stop() by minutes). It is
323- # also what keeps the ceiling stable if -n is raised. A serverful job
324- # counts one slot per instance.
325- SHALLOW_MAX_CONCURRENT_JOBS : ' 10'
326- run : |
327- # Scoped to shallow/ only -- see the comment above this job for why the
328- # rest of tests/integ/train is not repeated here.
329- #
330- # 84 of the suite's 100 tests run; the 16 deselected are:
331- # us_east_1 (5) -- Nova cases; this job holds us-west-2
332- # credentials only, so they run in the
333- # integ-tests-us-east-1 job instead.
334- # gpu_intensive (11) -- the CPT and MTRL classes. These are written in
335- # the shallow style but cannot be made
336- # self-contained: CPT submits only via HyperPod
337- # (a pre-provisioned cluster, not
338- # CreateTrainingJob) and MTRL needs an agent
339- # runtime plus an MLflow app. Both become
340- # gate-eligible by dropping one marker once those
341- # prerequisites exist in the PR account.
342- python -m pytest tests/integ/train/shallow \
343- -m "not gpu_intensive and not us_east_1" \
344- -n 8 \
345- --dist loadfile \
346- -v \
347- --durations=15 \
348- --junitxml=shallow-integ-results.xml
349-
350- - name : Upload test results
351- if : always()
352- uses : actions/upload-artifact@v4
305+ uses : aws-actions/aws-codebuild-run-build@v1
353306 with :
354- name : shallow-integ-test-results
355- path : sagemaker-train/shallow-integ-results.xml
356- if-no-files-found : warn
307+ project-name : ${{ github.event.repository.name }}-ci-sagemaker-train-fast-integ-tests
308+ source-version-override : ' refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'
357309
358310 integ-tests-us-east-1 :
359311 runs-on : ubuntu-latest
0 commit comments