pr-checks-master.yml

name: Sagemaker PR Checks (Master)
on:
  pull_request_target:
    branches:
      - "master"
      - "master-mtrl-trainer"
      - "master-mtrl-release"
      - "master-nova-reconcillation"
    paths:
      - 'sagemaker-train/**'
      - 'sagemaker-serve/**'
      - 'sagemaker-mlops/**'
      - 'sagemaker-core/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref }}
  cancel-in-progress: true

permissions:
  id-token: write

jobs:
  collab-check:
    runs-on: ubuntu-latest
    outputs:
      approval-env: ${{ steps.collab-check.outputs.result }}
    steps:
      - name: Collaborator Check
        uses: actions/github-script@v7
        id: collab-check
        with:
          github-token: ${{ secrets.COLLAB_CHECK_TOKEN }}
          result-encoding: string
          script: | 
            try {
              const res = await github.rest.repos.checkCollaborator({
                owner: context.repo.owner,
                repo: context.repo.repo,
                username: "${{ github.event.pull_request.user.login }}",
              });
              console.log("Verifed ${{ github.event.pull_request.user.login }} is a repo collaborator. Auto Approving PR Checks.")
              return res.status == "204" ? "auto-approve" : "manual-approval"
            } catch (error) {
              console.log("${{ github.event.pull_request.user.login }} is not a collaborator. Requiring Manual Approval to run PR Checks.")
              return "manual-approval"
            }
  wait-for-approval:
    runs-on: ubuntu-latest
    needs: [ collab-check ]
    environment: ${{ needs.collab-check.outputs.approval-env }}
    steps:
      - run: echo "Workflow Approved! Starting PR Checks."
  detect-changes:
    runs-on: ubuntu-latest
    needs: [wait-for-approval]
    outputs:
      submodules: ${{ steps.check-changes.outputs.submodules }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          token: ${{ secrets.GH_PAT }}
          ref: ${{ github.event.pull_request.base.ref }}
      - name: Detect Changes
        id: check-changes
        run: |
          set -e
          
          echo "Target Branch: ${{ github.event.pull_request.base.ref }}"
          echo "Current Target SHA: $(git rev-parse HEAD)"
          echo "PR Number: ${{ github.event.pull_request.number }}"
          echo "PR Latest SHA: ${{ github.event.pull_request.head.sha }}"
          
          git fetch origin pull/${{ github.event.pull_request.number }}/head
          CHANGES=$(git diff --name-only HEAD FETCH_HEAD)
          
          echo "Changed files:"
          echo "$CHANGES"
          
          # Function to extract dependencies from pyproject.toml
          get_dependencies() {
            local module=$1
            grep "sagemaker-" "$module/pyproject.toml" | grep -o 'sagemaker-[a-z]*' | sort -u
          }
          
          # Function to find all modules that depend on a given module (recursively)
          find_dependents() {
            local target=$1
            local all_modules=("sagemaker-core" "sagemaker-train" "sagemaker-serve" "sagemaker-mlops")
            local dependents=()
            
            for module in "${all_modules[@]}"; do
              if [ "$module" != "$target" ]; then
                if get_dependencies "$module" | grep -q "^$target$"; then
                  dependents+=("$module")
                fi
              fi
            done
            
            echo "${dependents[@]}"
          }
          
          # Initialize set of submodules to test (using associative array)
          declare -A SUBMODULES_SET
          
          # Function to recursively add module and all its dependents
          add_module_and_dependents() {
            local module=$1
            
            if [ -z "${SUBMODULES_SET[$module]}" ]; then
              SUBMODULES_SET["$module"]=1
              echo "Adding $module to test set"
              
              # Find all modules that depend on this one and add them recursively
              local dependents=$(find_dependents "$module")
              for dependent in $dependents; do
                add_module_and_dependents "$dependent"
              done
            fi
          }
          
          # Determine whether a module has any non-test changes. A change counts
          # as a source change if it touches anything under the module other than
          # its tests/ directory (e.g. src/, pyproject.toml, tox.ini, VERSION).
          # This is intentionally conservative: only changes confined entirely to
          # tests/ are treated as test-only.
          is_source_changed() {
            local module=$1
            echo "$CHANGES" | grep "^$module/" | grep -qv "^$module/tests/"
          }
          
          all_modules=("sagemaker-core" "sagemaker-train" "sagemaker-serve" "sagemaker-mlops")
          
          # Pass 1: modules with source changes pull in themselves plus every
          # module that (transitively) depends on them, since a source change can
          # affect downstream behaviour. This preserves the original logic.
          for module in "${all_modules[@]}"; do
            if is_source_changed "$module"; then
              echo "$module has source changes - adding it and all dependents"
              add_module_and_dependents "$module"
            fi
          done
          
          # Pass 2: modules with test-only changes add only themselves and skip
          # dependency propagation, since changing a module's tests cannot affect
          # other modules. Run after Pass 1 so source-change propagation is never
          # short-circuited by a test-only module already being in the set.
          for module in "${all_modules[@]}"; do
            if echo "$CHANGES" | grep -q "^$module/" && ! is_source_changed "$module"; then
              if [ -z "${SUBMODULES_SET[$module]}" ]; then
                echo "$module has test-only changes - adding only $module"
                SUBMODULES_SET["$module"]=1
              fi
            fi
          done
          
          # Convert associative array to JSON array
          SUBMODULES='[]'
          for submodule in "${!SUBMODULES_SET[@]}"; do
            if [ "$SUBMODULES" = '[]' ]; then
              SUBMODULES="[\"$submodule\"]"
            else
              SUBMODULES=$(echo $SUBMODULES | sed "s/\]$/,\"$submodule\"\]/")
            fi
          done
          
          echo "Final SUBMODULES: $SUBMODULES"
          echo "submodules=$SUBMODULES" >> $GITHUB_OUTPUT

  codestyle-doc-tests:
    runs-on: ubuntu-latest
    needs: [detect-changes]
    if: needs.detect-changes.outputs.submodules != '[]'
    strategy:
      fail-fast: false
      matrix:
        submodule: ${{ fromJson(needs.detect-changes.outputs.submodules) }}
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
          aws-region: us-west-2
          role-duration-seconds: 10800

      - name: Run CodeBuild for ${{ matrix.submodule }}
        uses: aws-actions/aws-codebuild-run-build@v1
        with:
          project-name: ${{ github.event.repository.name }}-ci-${{ matrix.submodule }}-codestyle-doc-tests
          source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'

  unit-tests:
    runs-on: ubuntu-latest
    needs: [detect-changes]
    if: needs.detect-changes.outputs.submodules != '[]'
    strategy:
      fail-fast: false
      matrix:
        submodule: ${{ fromJson(needs.detect-changes.outputs.submodules) }}
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
          aws-region: us-west-2
          role-duration-seconds: 10800

      - name: Run Unit Tests for ${{ matrix.submodule }}
        uses: aws-actions/aws-codebuild-run-build@v1
        with:
          # Use the single CDK-managed V3 unit-test project (driven by the SUBMODULE
          # env var), the same project the CI-health workflow uses. The previous
          # per-submodule projects (sagemaker-python-sdk-ci-<submodule>-unit-tests)
          # were created manually, are not CDK/pipeline-managed, and had drifted
          # stale (e.g. still running `--cov=.` instead of the deployed
          # `--cov=sagemaker`), so PR coverage never reflected buildspec fixes.
          project-name: ${{ github.event.repository.name }}-ci-health-unit-test-v3
          source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'
          env-vars-for-codebuild: |
            SUBMODULE
        env:
          SUBMODULE: ${{ matrix.submodule }}

  integ-tests:
    runs-on: ubuntu-latest
    needs: [detect-changes]
    if: needs.detect-changes.outputs.submodules != '[]'
    strategy:
      fail-fast: false
      matrix:
        submodule: ${{ fromJson(needs.detect-changes.outputs.submodules) }}
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
          aws-region: us-west-2
          role-duration-seconds: 10800

      - name: Run Integ Tests for ${{ matrix.submodule }}
        uses: aws-actions/aws-codebuild-run-build@v1
        with:
          project-name: ${{ github.event.repository.name }}-ci-${{ matrix.submodule }}-integ-tests
          source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'

  # Additive: runs the shallow (submit-then-stop) suite for sagemaker-train
  # alongside the existing integ-tests job above, which is unchanged.
  #
  # Runs in CodeBuild, not on the runner. It began as a runner job, but
  # actions/checkout refuses to place a fork's head commit in a
  # pull_request_target job -- correctly, because the runner also holds the base
  # repo's GITHUB_TOKEN and assumes CI_AWS_ROLE_ARN, so a fork could edit
  # conftest.py and read those credentials out. That is the "pwn request" shape,
  # and on a public repo it is a live credential-exfiltration path, so the fix is
  # to move the execution rather than override the refusal with
  # allow-unsafe-pr-checkout. Nearly every PR here comes from a fork, so a
  # same-repo guard would have left the suite with almost no gate coverage.
  # source-version-override is how the three jobs above already run PR code: the
  # build never sees the runner's token, secrets or default-branch cache.
  #
  # Why its own project rather than folding this into the sagemaker-train
  # integ-tests project: it reports as its own check, so a shallow failure is
  # distinguishable at a glance from a deep-suite failure, and it runs
  # concurrently with the deep suite instead of queueing behind it.
  #
  # Tradeoff of moving off the runner: the pytest selection now lives in the
  # CDK's buildspecs.ts (createCIShallowIntegBuildSpec) instead of this file, so
  # changing which tests run is no longer reviewable in a PR to this repo. That
  # is the price of executing fork code safely, and it is the same place the
  # other three test jobs' selections already live.
  #
  # What runs there: only tests/integ/train/shallow, deselecting gpu_intensive
  # (the CPT and MTRL classes, which need a pre-provisioned HyperPod cluster and
  # an agent runtime plus an MLflow app) and us_east_1 (Nova cases, which run in
  # the integ-tests-us-east-1 project against the Nova account). The client-side
  # tests are deliberately not repeated -- the deep suite already runs the whole
  # tests/integ tree, so widening scope would duplicate them and double the job
  # creation the shallow suite performs.
  #
  # Why submit-then-stop is worth gating on: CreateTrainingJob returns a
  # TrainingJobArn only after the request has cleared public-model validation,
  # SigV4, sagemaker:CreateTrainingJob authorization, iam:PassRole, the training
  # backend's request validators (including the role-assuming ones that resolve
  # S3 and ECR as the customer) and the final duplicate-name write. So a returned
  # ARN proves the payload and the caller's permissions are both good -- without
  # paying for a training run. The job is stopped immediately.
  #
  # It asserts nothing about training *behaviour* (artifacts, metrics,
  # convergence); that remains the deep suites' job.
  fast-integ-tests:
    runs-on: ubuntu-latest
    needs: [detect-changes]
    # No same-repo guard: nothing here checks out PR code, so fork PRs are gated
    # too. The suite only runs when sagemaker-train is in the change set.
    if: contains(fromJson(needs.detect-changes.outputs.submodules), 'sagemaker-train')
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
          aws-region: us-west-2
          role-duration-seconds: 10800

      - name: Run shallow sagemaker-train integ tests
        uses: aws-actions/aws-codebuild-run-build@v1
        with:
          project-name: ${{ github.event.repository.name }}-ci-sagemaker-train-fast-integ-tests
          source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'

  integ-tests-us-east-1:
    runs-on: ubuntu-latest
    needs: [detect-changes]
    if: needs.detect-changes.outputs.submodules != '[]'
    steps:
      - name: Configure AWS Credentials (us-east-1)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }}
          aws-region: us-east-1
          role-duration-seconds: 10800

      - name: Run us-east-1 Integ Tests (all submodules)
        uses: aws-actions/aws-codebuild-run-build@v1
        with:
          project-name: ${{ github.event.repository.name }}-ci-integ-tests-us-east-1
          source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'