diff --git a/.github/config/image/autogluon/1.6-sagemaker-cpu.yml b/.github/config/image/autogluon/1.6-sagemaker-cpu.yml new file mode 100644 index 000000000000..3d04ee467ca4 --- /dev/null +++ b/.github/config/image/autogluon/1.6-sagemaker-cpu.yml @@ -0,0 +1,33 @@ +image: + name: "autogluon-1.6-sagemaker-cpu" + description: "AutoGluon 1.6 CPU training and inference for SageMaker" + +metadata: + framework: "autogluon" + framework_version: "1.6.3" + os_version: "amzn2023" + customer_type: "sagemaker" + platform: "sagemaker" + arch_type: "x86" + device_type: "cpu" + job_type: "general" + prod_image: "autogluon:1.6-cpu-amzn2023" + +build: + dockerfile: "docker/autogluon/Dockerfile" + target: "sagemaker" + base_image: "public.ecr.aws/deep-learning-containers/pytorch:2.13-cpu-amzn2023-sagemaker" + requirements_file: "docker/autogluon/1.6/requirements.cpu.in" + python_version: "3.12" + dlc_major_version: "1" + dlc_minor_version: "0" + +release: + # Gamma-testing the unified image before production. + # Follow-up: publish the AutoGluon docs only after the production release succeeds. + release: true + force_release: false + public_registry: false + private_registry: true + enable_soci: false + environment: "gamma" diff --git a/.github/config/image/autogluon/1.6-sagemaker-cuda.yml b/.github/config/image/autogluon/1.6-sagemaker-cuda.yml new file mode 100644 index 000000000000..ddee498ba01d --- /dev/null +++ b/.github/config/image/autogluon/1.6-sagemaker-cuda.yml @@ -0,0 +1,34 @@ +image: + name: "autogluon-1.6-sagemaker-cuda" + description: "AutoGluon 1.6 GPU training and inference for SageMaker" + +metadata: + framework: "autogluon" + framework_version: "1.6.3" + os_version: "amzn2023" + customer_type: "sagemaker" + platform: "sagemaker" + arch_type: "x86" + device_type: "gpu" + job_type: "general" + prod_image: "autogluon:1.6-cu133-amzn2023" + +build: + dockerfile: "docker/autogluon/Dockerfile" + target: "sagemaker" + base_image: "public.ecr.aws/deep-learning-containers/pytorch:2.13-cu133-amzn2023-sagemaker" + requirements_file: "docker/autogluon/1.6/requirements.cuda.in" + python_version: "3.12" + cuda_version: "13.3.0" + dlc_major_version: "1" + dlc_minor_version: "0" + +release: + # Gamma-testing the unified image before production. + # Follow-up: publish the AutoGluon docs only after the production release succeeds. + release: true + force_release: false + public_registry: false + private_registry: true + enable_soci: false + environment: "gamma" diff --git a/.github/config/test-suites.yml b/.github/config/test-suites.yml index b27fe0c9c229..ff761a9db800 100644 --- a/.github/config/test-suites.yml +++ b/.github/config/test-suites.yml @@ -56,6 +56,11 @@ suites: code_paths: - test/pytorch/integration/sagemaker/** + autogluon/sagemaker: + skip_eligible: true + code_paths: + - test/autogluon/sagemaker/** + tensorflow-inference/sagemaker: skip_eligible: true code_paths: diff --git a/.github/workflows/autogluon.dispatch-release.yml b/.github/workflows/autogluon.dispatch-release.yml new file mode 100644 index 000000000000..4869039112d4 --- /dev/null +++ b/.github/workflows/autogluon.dispatch-release.yml @@ -0,0 +1,29 @@ +name: "Dispatch - AutoGluon Release" + +# AutoGluon ships a few times a year, so releases are triggered by hand rather +# than on a cron. Run once per config file from `main`; the pipeline's +# release-gate reads release.environment (gamma / production) from the config. + +on: + workflow_dispatch: + inputs: + config-file: + description: "Path to image config YAML (e.g., .github/config/image/autogluon/1.6-sagemaker-cuda.yml)" + required: true + type: string + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + pipeline: + uses: ./.github/workflows/autogluon.pipeline.yml + with: + config-file: ${{ inputs.config-file }} + tag-suffix: ${{ github.run_id }} + release: true + secrets: inherit diff --git a/.github/workflows/autogluon.pipeline.yml b/.github/workflows/autogluon.pipeline.yml new file mode 100644 index 000000000000..a755bf7f4e3d --- /dev/null +++ b/.github/workflows/autogluon.pipeline.yml @@ -0,0 +1,201 @@ +name: "Pipeline - AutoGluon" + +on: + workflow_call: + inputs: + config-file: + description: "Path to image config YAML" + required: true + type: string + tag-suffix: + description: "CI tag suffix (run_id or pr-NUMBER)" + required: true + type: string + build: + description: "Whether to build the image (false = skip build, test against prod)" + required: false + type: boolean + default: true + run-sanity-test: + description: "Run sanity tests" + required: false + type: boolean + default: true + run-unit-test: + description: "Run serving adapter unit tests" + required: false + type: boolean + default: true + run-security-test: + description: "Run security tests" + required: false + type: boolean + default: true + run-telemetry-test: + description: "Run telemetry tests" + required: false + type: boolean + default: true + run-sagemaker-test: + description: "Run SageMaker training and endpoint tests" + required: false + type: boolean + default: true + release: + description: "Run release job after tests pass" + required: false + type: boolean + default: false + outputs: + image-uri: + description: "Built image URI (empty if build was skipped)" + value: ${{ jobs.build.outputs.image-uri }} + +jobs: + build: + if: ${{ inputs.build }} + concurrency: + group: ${{ github.workflow }}-build-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: true + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:x86-build-runner + buildspec-override:true + timeout-minutes: 120 + outputs: + image-uri: ${{ steps.build.outputs.image-uri }} + steps: + - uses: actions/checkout@v6 + - id: build + uses: ./.github/actions/build-image + with: + config-file: ${{ inputs.config-file }} + aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + tag-suffix: ${{ inputs.tag-suffix }} + + check: + if: ${{ always() && !failure() && !cancelled() }} + needs: [build] + concurrency: + group: ${{ github.workflow }}-check-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: true + uses: ./.github/workflows/_reusable.check-test-pass.yml + with: + config-file: ${{ inputs.config-file }} + image-uri: ${{ needs.build.outputs.image-uri || '' }} + suites: '["sanity", "telemetry", "autogluon/sagemaker"]' + + unit-tests: + if: ${{ inputs.run-unit-test }} + uses: ./.github/workflows/autogluon.tests-unit.yml + + sanity-test: + if: ${{ always() && !cancelled() && inputs.run-sanity-test && needs.build.result != 'failure' && fromJSON(needs.check.outputs.skips || '{}')['sanity'] != true }} + needs: [build, check] + concurrency: + group: ${{ github.workflow }}-sanity-test-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: true + uses: ./.github/workflows/_reusable.sanity-tests.yml + with: + config-file: ${{ inputs.config-file }} + image-uri: ${{ needs.build.outputs.image-uri || '' }} + image-content-hash: ${{ needs.check.outputs.image-content-hash }} + suite-code-hash: ${{ fromJSON(needs.check.outputs.suite-code-hashes || '{}')['sanity'] || '' }} + + security-test: + if: ${{ always() && !failure() && !cancelled() && inputs.run-security-test }} + needs: [build] + concurrency: + group: ${{ github.workflow }}-security-test-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: true + uses: ./.github/workflows/_reusable.security-tests.yml + with: + config-file: ${{ inputs.config-file }} + image-uri: ${{ needs.build.outputs.image-uri || '' }} + + telemetry-test: + if: ${{ always() && !cancelled() && inputs.run-telemetry-test && needs.build.result != 'failure' && fromJSON(needs.check.outputs.skips || '{}')['telemetry'] != true }} + needs: [build, check] + concurrency: + group: ${{ github.workflow }}-telemetry-test-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: false + uses: ./.github/workflows/_reusable.telemetry-tests.yml + with: + config-file: ${{ inputs.config-file }} + image-uri: ${{ needs.build.outputs.image-uri || '' }} + image-content-hash: ${{ needs.check.outputs.image-content-hash }} + suite-code-hash: ${{ fromJSON(needs.check.outputs.suite-code-hashes || '{}')['telemetry'] || '' }} + + sagemaker-tests: + if: >- + ${{ always() && !cancelled() && inputs.run-sagemaker-test && + needs.build.result != 'failure' && + fromJSON(needs.check.outputs.skips || '{}')['autogluon/sagemaker'] != true }} + needs: [build, check] + concurrency: + group: ${{ github.workflow }}-sagemaker-tests-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: false + uses: ./.github/workflows/autogluon.tests-sagemaker.yml + with: + config-file: ${{ inputs.config-file }} + image-uri: ${{ needs.build.outputs.image-uri || '' }} + image-content-hash: ${{ needs.check.outputs.image-content-hash }} + suite-code-hash: ${{ fromJSON(needs.check.outputs.suite-code-hashes || '{}')['autogluon/sagemaker'] || '' }} + + release-gate: + if: >- + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release && + !failure() && !cancelled() && + needs.build.result == 'success' + needs: [build, unit-tests, sanity-test, security-test, telemetry-test, sagemaker-tests] + runs-on: ubuntu-latest + outputs: + should-release: ${{ steps.check.outputs.should-release }} + release-spec: ${{ steps.spec.outputs.release-spec }} + environment: ${{ steps.check.outputs.environment }} + steps: + - uses: actions/checkout@v6 + - id: check + run: | + SHOULD_RELEASE=$(yq '.release.release' "${{ inputs.config-file }}") + ENVIRONMENT=$(yq '.release.environment' "${{ inputs.config-file }}") + echo "should-release=${SHOULD_RELEASE}" >> $GITHUB_OUTPUT + echo "environment=${ENVIRONMENT}" >> $GITHUB_OUTPUT + - id: spec + if: steps.check.outputs.should-release == 'true' + uses: ./.github/actions/generate-release-spec + with: + config-file: ${{ inputs.config-file }} + + release: + if: ${{ !failure() && !cancelled() && needs.release-gate.outputs.should-release == 'true' }} + needs: [build, release-gate] + uses: ./.github/workflows/_reusable.release-image.yml + with: + source-image-uri: ${{ needs.build.outputs.image-uri }} + release-spec: ${{ needs.release-gate.outputs.release-spec }} + environment: ${{ needs.release-gate.outputs.environment }} + aws-region: ${{ vars.AWS_REGION }} + secrets: inherit + + notify-failure: + if: >- + failure() && + github.ref == 'refs/heads/main' && + (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + !contains(github.workflow_ref, '.pr') && + inputs.release + needs: [build, unit-tests, sanity-test, security-test, telemetry-test, sagemaker-tests, release-gate, release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/notify-slack-failure + with: + webhook-url: ${{ secrets.SLACK_RELEASE_FAILURE_WEBHOOK_URL }} + workflow-name: ${{ github.workflow }} + config-file: ${{ inputs.config-file }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/autogluon.pr-1.6.yml b/.github/workflows/autogluon.pr-1.6.yml new file mode 100644 index 000000000000..f86a09da14c8 --- /dev/null +++ b/.github/workflows/autogluon.pr-1.6.yml @@ -0,0 +1,100 @@ +name: "PR - AutoGluon 1.6" + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize] + paths: + - ".github/config/image/autogluon/1.6-sagemaker-*.yml" + - ".github/workflows/autogluon.pipeline.yml" + - ".github/workflows/autogluon.pr-1.6.yml" + - ".github/workflows/autogluon.tests-sagemaker.yml" + - ".github/workflows/autogluon.tests-unit.yml" + - "docker/autogluon/**" + - "scripts/docker/autogluon/**" + - "scripts/docker/common/setup_oss_compliance.sh" + - "scripts/docker/telemetry/**" + - "test/sanity/**" + - "test/security/data/ecr_scan_allowlist/autogluon/**" + - "test/telemetry/**" + - "test/autogluon/**" + - "!docs/**" + +permissions: + contents: read + pull-requests: read + +jobs: + gatekeeper: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 1 + - uses: ./.github/actions/pr-permission-gate + + check-changes: + needs: [gatekeeper] + runs-on: ubuntu-latest + outputs: + build-change: ${{ steps.changes.outputs.build-change }} + sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }} + unit-test-change: ${{ steps.changes.outputs.unit-test-change }} + sagemaker-test-change: ${{ steps.changes.outputs.sagemaker-test-change }} + telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v4 + id: changes + with: + filters: | + build-change: + - ".github/config/image/autogluon/1.6-sagemaker-*.yml" + - "docker/autogluon/**" + - "scripts/docker/autogluon/**" + - "scripts/docker/common/setup_oss_compliance.sh" + - "test/security/data/ecr_scan_allowlist/autogluon/**" + sanity-test-change: + - "test/sanity/**" + unit-test-change: + - "scripts/docker/autogluon/**" + - "test/autogluon/inference/test_*.py" + - ".github/workflows/autogluon.tests-unit.yml" + sagemaker-test-change: + - "test/autogluon/sagemaker/**" + - ".github/workflows/autogluon.tests-sagemaker.yml" + telemetry-test-change: + - "scripts/docker/telemetry/**" + - "test/telemetry/**" + + discover: + needs: [gatekeeper] + runs-on: ubuntu-latest + outputs: + configs: ${{ steps.discover.outputs.configs }} + steps: + - uses: actions/checkout@v6 + - id: discover + uses: ./.github/actions/discover-configs + with: + pattern: ".github/config/image/autogluon/1.6-sagemaker-*.yml" + + pipeline: + needs: [check-changes, discover] + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.discover.outputs.configs) }} + uses: ./.github/workflows/autogluon.pipeline.yml + with: + config-file: ${{ matrix.config_file }} + tag-suffix: pr-${{ github.event.pull_request.number }} + build: ${{ needs.check-changes.outputs.build-change == 'true' }} + run-sanity-test: ${{ needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sanity-test-change == 'true' }} + run-unit-test: ${{ (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.unit-test-change == 'true') && endsWith(matrix.config_file, '-cpu.yml') }} + run-security-test: ${{ needs.check-changes.outputs.build-change == 'true' }} + run-telemetry-test: ${{ needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.telemetry-test-change == 'true' }} + run-sagemaker-test: ${{ needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sagemaker-test-change == 'true' }} + release: false + secrets: inherit diff --git a/.github/workflows/autogluon.tests-sagemaker.yml b/.github/workflows/autogluon.tests-sagemaker.yml new file mode 100644 index 000000000000..3ce819046165 --- /dev/null +++ b/.github/workflows/autogluon.tests-sagemaker.yml @@ -0,0 +1,116 @@ +name: Reusable AutoGluon SageMaker Tests + +on: + workflow_call: + inputs: + config-file: + description: "Path to image config YAML" + required: true + type: string + image-uri: + description: "Image URI to test. If empty, derives prod URI from config." + required: false + type: string + default: "" + image-content-hash: + description: "Image hash computed from hash_image_content.py for caching a test pass." + required: false + type: string + default: "" + suite-code-hash: + description: "Test code hash computed from hash_suite_code.py for caching a test pass." + required: false + type: string + default: "" + +permissions: + contents: read + +jobs: + preflight: + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + outputs: + image-uri: ${{ steps.image.outputs.image-uri }} + aws-account-id: ${{ steps.image.outputs.aws-account-id }} + image-exists: ${{ steps.check.outputs.exists }} + device-type: ${{ steps.config.outputs.device-type }} + steps: + - uses: actions/checkout@v6 + + - name: Resolve image URI + id: image + uses: ./.github/actions/resolve-image-uri + with: + image-uri: ${{ inputs.image-uri }} + ci-aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + prod-aws-account-id: ${{ vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + config-file: ${{ inputs.config-file }} + + - name: Check image exists + id: check + uses: ./.github/actions/check-image-exists + with: + image-uri: ${{ steps.image.outputs.image-uri }} + + - name: Parse image device type + id: config + run: | + echo "device-type=$(yq '.metadata.device_type' '${{ inputs.config-file }}')" >> $GITHUB_OUTPUT + + e2e-test: + # Trains a model with the image, then deploys that exact artifact for inference. + if: ${{ needs.preflight.outputs.image-exists == 'true' }} + needs: [preflight] + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + timeout-minutes: 120 + steps: + - uses: actions/checkout@v6 + + - name: ECR login + uses: ./.github/actions/ecr-authenticate + with: + aws-account-id: ${{ needs.preflight.outputs.aws-account-id }} + aws-region: ${{ vars.AWS_REGION }} + image-uri: ${{ needs.preflight.outputs.image-uri }} + + - name: Run SageMaker train-to-serve test + env: + SM_ROLE_ARN: arn:aws:iam::${{ vars.CI_AWS_ACCOUNT_ID }}:role/SageMakerRole + SM_DEVICE_TYPE: ${{ needs.preflight.outputs.device-type }} + run: | + uv venv --python 3.12 + source .venv/bin/activate + uv pip install -r test/requirements.txt + uv pip install -r test/autogluon/sagemaker/requirements.txt + cd test/ + PYTHONPATH=$(pwd):$PYTHONPATH \ + uv run --active --no-project pytest -v --tb=short -rA --log-cli-level=INFO \ + --image-uri ${{ needs.preflight.outputs.image-uri }} \ + autogluon/sagemaker/test_train_and_serve.py + + record-test-pass: + if: >- + ${{ always() && !cancelled() && !failure() && + inputs.image-content-hash != '' && inputs.suite-code-hash != '' && + needs.e2e-test.result == 'success' }} + needs: [e2e-test] + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/record-test-pass + with: + suite: autogluon/sagemaker + image-content-hash: ${{ inputs.image-content-hash }} + suite-code-hash: ${{ inputs.suite-code-hash }} + ci-image-uri: ${{ inputs.image-uri }} + ci-images-table-account-id: ${{ vars.CI_IMAGES_TABLE_ACCOUNT_ID }} diff --git a/.github/workflows/autogluon.tests-unit.yml b/.github/workflows/autogluon.tests-unit.yml new file mode 100644 index 000000000000..0d054d6b2bfb --- /dev/null +++ b/.github/workflows/autogluon.tests-unit.yml @@ -0,0 +1,33 @@ +name: Reusable AutoGluon Unit Tests + +on: + workflow_call: {} + +permissions: + contents: read + +jobs: + unit-test: + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - name: Install test dependencies + run: | + uv venv --python 3.12 + source .venv/bin/activate + uv pip install "flask==3.1.3" pytest + + - name: Validate unified entrypoint + run: bash -n scripts/docker/autogluon/sagemaker_entrypoint.sh + + - name: Run serving adapter unit tests + run: | + source .venv/bin/activate + PYTHONPATH="${{ github.workspace }}/test" \ + uv run --active --no-project pytest -v --tb=short -rA --noconftest \ + test/autogluon/inference/test_server.py diff --git a/docker/autogluon/1.6/requirements.cpu.in b/docker/autogluon/1.6/requirements.cpu.in new file mode 100644 index 000000000000..dd807e408b0a --- /dev/null +++ b/docker/autogluon/1.6/requirements.cpu.in @@ -0,0 +1,112 @@ +# Exact packages added to or changed from the PyTorch 2.13 CPU SageMaker DLC. +absl-py==2.5.0 +accelerate==1.15.0 +adagio==0.2.6 +aiohttp-cors==0.8.1 +autogluon==1.6.3 +autogluon-common==1.6.3 +autogluon-core==1.6.3 +autogluon-features==1.6.3 +autogluon-multimodal==1.6.3 +autogluon-tabular==1.6.3 +autogluon-timeseries==1.6.3 +catboost==1.2.10 +chronos-forecasting==2.3.2 +colorama==0.4.6 +colorful==0.5.8 +colorlog==6.12.0 +coreforecast==0.0.16 +datasets==5.0.1 +defusedxml==0.7.1 +distlib==0.4.3 +einx==0.4.3 +evaluate==0.4.6 +formulaic==1.2.2 +frozendict==2.4.7 +fugue==0.9.7 +future==1.0.0 +gdown==6.4.0 +gluonts==0.17.0 +google-api-core==2.37.0 +googleapis-common-protos==1.75.3 +graphviz==0.21 +grpcio==1.84.0 +hf-xet==1.6.0 +huggingface-hub==1.32.0 +hyperopt==0.2.7 +interface-meta==2.0.1 +jsonschema==4.23.0 +lazy-loader==0.5 +lightgbm==4.7.0 +lightning==2.6.6 +lightning-utilities==0.15.3 +loguru==0.7.3 +markdown==3.10.3 +mlforecast==0.14.0 +model-index==0.1.11 +multiprocess==0.70.19 +nlpaug==1.1.11 +nltk==3.10.3 +nvidia-ml-py3==7.352.0 +opencensus==0.11.4 +opencensus-context==0.1.3 +opendatalab==0.0.10 +openmim==0.3.9 +opentelemetry-api==1.44.0 +opentelemetry-exporter-prometheus==0.65b0 +opentelemetry-sdk==1.44.0 +opentelemetry-semantic-conventions==0.65b0 +openxlab==0.0.11 +optuna==5.0.0 +ordered-set==4.1.0 +orjson==3.12.0 +patsy==1.0.3 +pbr==7.0.3 +pdf2image==1.17.0 +peft==0.19.1 +prometheus-client==0.26.0 +proto-plus==1.28.4 +py-spy==0.4.2 +py4j==0.10.9.9 +pycryptodome==3.23.0 +pysocks==1.7.1 +pytesseract==0.3.13 +python-discovery==1.6.0 +pytorch-lightning==2.6.6 +pytorch-metric-learning==2.8.1 +ray==2.58.0 +regex==2026.9.10 +safetensors==0.8.0 +scikit-image==0.25.2 +sentencepiece==0.2.2 +seqeval==1.2.2 +statsforecast==2.0.1 +statsmodels==0.15.0 +stevedore==5.4.1 +tensorboard==2.21.0 +tensorboard-data-server==0.7.2 +tensorboardx==2.6.5 +text-unidecode==1.3 +tifffile==2026.9.15 +timm==1.0.3 +tokenizers==0.22.2 +toolz==0.12.1 +torchmetrics==1.7.4 +transformers==5.14.1 +triad==1.0.2 +utilsforecast==0.2.11 +virtualenv==21.7.11 +window-ops==0.0.15 +xgboost-cpu==3.3.0 +xxhash==4.0.1 + +# Temporary security overrides inherited from the published PyTorch 2.13 +# SageMaker DLC. The source base is already fixed by DLC PR #6749; remove these +# after the published base tag includes that change. MLflow 3.16 relaxes its +# cryptography cap and its three packages must remain version-aligned. +cryptography==50.0.1 +mlflow==3.16.0 +mlflow-skinny==3.16.0 +mlflow-tracing==3.16.0 +sqlparse==0.6.0 +tornado==6.5.10 diff --git a/docker/autogluon/1.6/requirements.cuda.in b/docker/autogluon/1.6/requirements.cuda.in new file mode 100644 index 000000000000..cb0648af8d83 --- /dev/null +++ b/docker/autogluon/1.6/requirements.cuda.in @@ -0,0 +1,112 @@ +# Exact packages added to or changed from the PyTorch 2.13 CUDA SageMaker DLC. +absl-py==2.5.0 +accelerate==1.15.0 +adagio==0.2.6 +aiohttp-cors==0.8.1 +autogluon==1.6.3 +autogluon-common==1.6.3 +autogluon-core==1.6.3 +autogluon-features==1.6.3 +autogluon-multimodal==1.6.3 +autogluon-tabular==1.6.3 +autogluon-timeseries==1.6.3 +catboost==1.2.10 +chronos-forecasting==2.3.2 +colorama==0.4.6 +colorful==0.5.8 +colorlog==6.12.0 +coreforecast==0.0.16 +datasets==5.0.1 +defusedxml==0.7.1 +distlib==0.4.3 +einx==0.4.3 +evaluate==0.4.6 +formulaic==1.2.2 +frozendict==2.4.7 +fugue==0.9.7 +future==1.0.0 +gdown==6.4.0 +gluonts==0.17.0 +google-api-core==2.30.3 +googleapis-common-protos==1.75.3 +graphviz==0.21 +grpcio==1.84.0 +hf-xet==1.6.0 +huggingface-hub==1.32.0 +hyperopt==0.2.7 +interface-meta==2.0.1 +jsonschema==4.23.0 +lazy-loader==0.5 +lightgbm==4.7.0 +lightning==2.6.6 +lightning-utilities==0.15.3 +loguru==0.7.3 +markdown==3.10.3 +mlforecast==0.14.0 +model-index==0.1.11 +multiprocess==0.70.19 +nlpaug==1.1.11 +nltk==3.10.3 +nvidia-ml-py3==7.352.0 +opencensus==0.11.4 +opencensus-context==0.1.3 +opendatalab==0.0.10 +openmim==0.3.9 +opentelemetry-api==1.44.0 +opentelemetry-exporter-prometheus==0.65b0 +opentelemetry-sdk==1.44.0 +opentelemetry-semantic-conventions==0.65b0 +openxlab==0.0.11 +optuna==5.0.0 +ordered-set==4.1.0 +orjson==3.12.0 +patsy==1.0.3 +pbr==7.0.3 +pdf2image==1.17.0 +peft==0.19.1 +prometheus-client==0.26.0 +proto-plus==1.28.4 +py-spy==0.4.2 +py4j==0.10.9.9 +pycryptodome==3.23.0 +pysocks==1.7.1 +pytesseract==0.3.13 +python-discovery==1.6.0 +pytorch-lightning==2.6.6 +pytorch-metric-learning==2.8.1 +ray==2.58.0 +regex==2026.9.10 +safetensors==0.8.0 +scikit-image==0.25.2 +scipy==1.15.3 +sentencepiece==0.2.2 +seqeval==1.2.2 +statsforecast==2.0.3 +statsmodels==0.15.0 +stevedore==5.4.1 +tensorboard==2.21.0 +tensorboard-data-server==0.7.2 +tensorboardx==2.6.5 +text-unidecode==1.3 +tifffile==2026.9.15 +timm==1.0.3 +tokenizers==0.22.2 +toolz==0.12.1 +torchmetrics==1.7.4 +transformers==5.14.1 +triad==1.0.2 +utilsforecast==0.2.11 +virtualenv==21.7.11 +window-ops==0.0.15 +xgboost-cpu==3.3.0 +xxhash==4.0.1 + +# Temporary security overrides inherited from the published PyTorch 2.13 +# SageMaker DLC. The source base is already fixed by DLC PR #6749; remove these +# after the published base tag includes that change. MLflow 3.16 relaxes its +# cryptography cap and its three packages must remain version-aligned. +cryptography==50.0.1 +mlflow==3.16.0 +mlflow-skinny==3.16.0 +mlflow-tracing==3.16.0 +tornado==6.5.10 diff --git a/docker/autogluon/Dockerfile b/docker/autogluon/Dockerfile new file mode 100644 index 000000000000..e81dc01c13fc --- /dev/null +++ b/docker/autogluon/Dockerfile @@ -0,0 +1,93 @@ +# AutoGluon SageMaker DLC, layered on an AWS PyTorch training DLC +# (Amazon Linux 2023). The image supports both SageMaker commands: +# `train` uses the inherited training toolkit and `serve` uses the lightweight +# AutoGluon HTTP adapter below. +# +# BASE_IMAGE is supplied by: +# - .github/config/image/autogluon/1.6-sagemaker-cpu.yml +# - .github/config/image/autogluon/1.6-sagemaker-cuda.yml +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} AS sagemaker + +ARG REQUIREMENTS_FILE +ARG FRAMEWORK=autogluon +ARG FRAMEWORK_VERSION=1.6.3 +ARG CONTAINER_TYPE=general +ARG DLC_MAJOR_VERSION=1 +ARG DLC_MINOR_VERSION=0 +ARG VENV_PYTHON=/opt/venv/bin/python + +LABEL maintainer="Amazon AI" +LABEL dlc_major_version="${DLC_MAJOR_VERSION}" +LABEL dlc_minor_version="${DLC_MINOR_VERSION}" +LABEL framework="autogluon" +LABEL framework_version="${FRAMEWORK_VERSION}" +# Allow SageMaker to select the listening port through SAGEMAKER_BIND_TO_PORT. +LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true +# AutoGluon serves one model artifact per container; SageMaker MME is unsupported. +LABEL com.amazonaws.sagemaker.capabilities.multi-models=false + +# jq for the inherited SageMaker hostname setup, rsync for Ray file sync, +# poppler-utils for pdf2image (AutoMM documents). +# AL2023 has no tesseract package, so pytesseract-based AutoMM OCR is unavailable. +RUN dnf install -y jq rsync poppler-utils && dnf clean all + +# REQUIREMENTS_FILE is supplied by the matching CPU/CUDA image config. It +# contains exact pins only for packages added to or changed from that PyTorch +# DLC base. +COPY ${REQUIREMENTS_FILE} /tmp/autogluon-requirements.in +RUN uv pip install --python ${VENV_PYTHON} \ + --no-deps \ + -r /tmp/autogluon-requirements.in \ + && rm /tmp/autogluon-requirements.in \ + && uv cache clean + +# Some GPU base images include Transformer Engine. It is unused by AutoGluon, +# and peft imports it at load time, which fails without an NVIDIA driver +# (libcuda.so.1). This is a no-op when the packages are absent, including on CPU. +RUN TE_PKGS=$(uv pip list --python ${VENV_PYTHON} 2>/dev/null | awk '{print $1}' | grep -i '^transformer-engine' || true) \ + && if [ -n "${TE_PKGS}" ]; then uv pip uninstall --python ${VENV_PYTHON} ${TE_PKGS}; fi + +RUN uv pip check --python ${VENV_PYTHON} + +# A deliberately small SageMaker adapter. User handler code remains in the +# model artifact (/opt/ml/model/code) and keeps the established SageMaker +# inference-function contract used by AutoGluon Cloud. +COPY scripts/docker/autogluon/server.py /opt/autogluon-server/server.py +COPY scripts/docker/autogluon/gunicorn.conf.py /opt/autogluon-server/gunicorn.conf.py +COPY --chmod=755 scripts/docker/autogluon/install_requirements.py /opt/autogluon-server/install_requirements.py +COPY --chmod=755 scripts/docker/autogluon/sagemaker_entrypoint.sh /usr/local/bin/autogluon-sagemaker-entrypoint.sh + +# Telemetry: report this image's framework instead of the base's. +COPY ./scripts/docker/telemetry/deep_learning_container.py /usr/local/bin/deep_learning_container.py +COPY ./scripts/docker/telemetry/bash_telemetry.sh.template /tmp/bash_telemetry.sh.template +RUN chmod +x /usr/local/bin/deep_learning_container.py \ + && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ + -e "s/{{FRAMEWORK_VERSION}}/${FRAMEWORK_VERSION}/g" \ + -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ + /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ + && chmod +x /usr/local/bin/bash_telemetry.sh \ + && rm /tmp/bash_telemetry.sh.template + +ENV DLC_CONTAINER_TYPE=${CONTAINER_TYPE} + +# OSS compliance for the added packages, plus AutoGluon's third-party license bundle. +COPY ./scripts/docker/common/setup_oss_compliance.sh /tmp/setup_oss_compliance.sh +RUN bash /tmp/setup_oss_compliance.sh python3 && rm /tmp/setup_oss_compliance.sh \ + && curl -fsSL -o /licenses-autogluon.txt https://autogluon.s3.us-west-2.amazonaws.com/licenses/THIRD-PARTY-LICENSES.txt + +# OS security refresh; CACHE_REFRESH busts the cache daily. +ARG CACHE_REFRESH=0 +RUN echo "cache refresh: ${CACHE_REFRESH}" \ + && dnf upgrade -y --security --releasever latest \ + && dnf clean all \ + && rm -rf /tmp/tmp* /tmp/pip-* /tmp/uv-* + +# The security refresh may replace /etc/bashrc, so restore the telemetry hook. +RUN echo 'source /usr/local/bin/bash_telemetry.sh' >>/etc/bashrc \ + && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc + +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/autogluon-sagemaker-entrypoint.sh"] +CMD ["bash"] \ No newline at end of file diff --git a/scripts/docker/autogluon/gunicorn.conf.py b/scripts/docker/autogluon/gunicorn.conf.py new file mode 100644 index 000000000000..47958fd83f43 --- /dev/null +++ b/scripts/docker/autogluon/gunicorn.conf.py @@ -0,0 +1,22 @@ +"""Gunicorn configuration for the AutoGluon SageMaker server.""" + +import os + +bind = f"0.0.0.0:{os.getenv('SAGEMAKER_BIND_TO_PORT', '8080')}" +workers = int(os.getenv("SAGEMAKER_MODEL_SERVER_WORKERS", "1")) +timeout = int(os.getenv("SAGEMAKER_MODEL_SERVER_TIMEOUT", "60")) +graceful_timeout = timeout + +worker_class = "sync" + +accesslog = "-" +errorlog = "-" +capture_output = True +raw_log_level = os.getenv("SAGEMAKER_CONTAINER_LOG_LEVEL", "info").strip().lower() or "info" +loglevel = { + "10": "debug", + "20": "info", + "30": "warning", + "40": "error", + "50": "critical", +}.get(raw_log_level, raw_log_level) diff --git a/scripts/docker/autogluon/install_requirements.py b/scripts/docker/autogluon/install_requirements.py new file mode 100755 index 000000000000..263cb5cab294 --- /dev/null +++ b/scripts/docker/autogluon/install_requirements.py @@ -0,0 +1,78 @@ +#!/opt/venv/bin/python +"""Install model-artifact requirements once before Gunicorn starts.""" + +from __future__ import annotations + +import logging +import os +import re +import subprocess +import sys +from pathlib import Path +from urllib.parse import quote, urlparse, urlunparse + +LOGGER = logging.getLogger("autogluon-serving") + + +def _codeartifact_index(repository_arn: str) -> str: + # Adapted from scripts/docker/ray/sagemaker_serve.py. + match = re.fullmatch( + r"arn:([^:]+):codeartifact:([^:]+):([^:]+):repository/([^/]+)/(.+)", + repository_arn, + ) + if not match: + raise ValueError(f"Invalid CA_REPOSITORY_ARN: {repository_arn!r}") + + import boto3 + + _, region, account, domain, repository = match.groups() + client = boto3.client("codeartifact", region_name=region) + arguments = { + "domain": domain, + "domainOwner": account, + } + token = client.get_authorization_token(**arguments)["authorizationToken"] + endpoint = client.get_repository_endpoint( + **arguments, + repository=repository, + format="pypi", + )["repositoryEndpoint"] + + parsed = urlparse(endpoint) + authenticated = parsed._replace(netloc=f"aws:{quote(token, safe='')}@{parsed.netloc}") + return urlunparse(authenticated._replace(path=f"{parsed.path.rstrip('/')}/simple/")) + + +def install_requirements() -> None: + requirements_path = ( + Path(os.getenv("SAGEMAKER_BASE_DIR", "/opt/ml")) / "model" / "code" / "requirements.txt" + ) + if not requirements_path.is_file(): + return + + command = [ + "uv", + "pip", + "install", + "--python", + sys.executable, + "--no-cache", + "-r", + str(requirements_path), + ] + environment = os.environ.copy() + repository_arn = os.getenv("CA_REPOSITORY_ARN") + if repository_arn: + environment["UV_INDEX_URL"] = _codeartifact_index(repository_arn) + + LOGGER.info("Installing packages from %s", requirements_path) + subprocess.check_call(command, env=environment) + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + install_requirements() + + +if __name__ == "__main__": + main() diff --git a/scripts/docker/autogluon/sagemaker_entrypoint.sh b/scripts/docker/autogluon/sagemaker_entrypoint.sh new file mode 100755 index 000000000000..7bfb08aa5eb0 --- /dev/null +++ b/scripts/docker/autogluon/sagemaker_entrypoint.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Unified AutoGluon SageMaker entrypoint. SageMaker invokes training containers +# with `train` and inference containers with `serve`. +set -euo pipefail + +case "${1:-}" in + train) + # Preserve the PyTorch training DLC's hostname workaround, CUDA setup, and + # SageMaker training-toolkit launcher. + cd / + exec bash -m start_with_right_hostname.sh "$@" + ;; + serve) + shift + # Serving does not start a login shell, so fire telemetry explicitly. + bash /usr/local/bin/bash_telemetry.sh >/dev/null 2>&1 || true + # Install model-artifact dependencies once, before Gunicorn forks workers. + /opt/autogluon-server/install_requirements.py + # Delegate CUDA forward compatibility to the inherited generic entrypoint. + exec /usr/local/bin/entrypoint.sh gunicorn \ + --config /opt/autogluon-server/gunicorn.conf.py \ + --chdir /opt/autogluon-server \ + "$@" \ + server:app + ;; + *) + exec /usr/local/bin/entrypoint.sh "$@" + ;; +esac diff --git a/scripts/docker/autogluon/server.py b/scripts/docker/autogluon/server.py new file mode 100644 index 000000000000..4152fb397582 --- /dev/null +++ b/scripts/docker/autogluon/server.py @@ -0,0 +1,99 @@ +"""SageMaker HTTP adapter for AutoGluon inference handlers.""" + +from __future__ import annotations + +import importlib.util +import logging +import os +import sys +from pathlib import Path +from types import ModuleType + +from flask import Flask, Response, jsonify, request + +MODEL_DIR = Path(os.getenv("SAGEMAKER_BASE_DIR", "/opt/ml")) / "model" +CODE_DIR = MODEL_DIR / "code" +PROGRAM = os.getenv("SAGEMAKER_PROGRAM", "inference.py").strip() or "inference.py" +DEFAULT_ACCEPT = ( + os.getenv("SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT", "application/json").strip() + or "application/json" +) +LOG_LEVEL = os.getenv("SAGEMAKER_CONTAINER_LOG_LEVEL", "INFO").strip() or "INFO" +logging.basicConfig(level=int(LOG_LEVEL) if LOG_LEVEL.isdigit() else LOG_LEVEL.upper()) +LOGGER = logging.getLogger("autogluon-serving") + + +def _load_handler() -> ModuleType: + """Load the user-provided inference module from the model artifact.""" + program = Path(PROGRAM) + path = program if program.is_absolute() else CODE_DIR / program + if not path.is_file(): + raise RuntimeError(f"AutoGluon inference handler not found: {path}") + + sys.path.insert(0, str(path.parent)) + spec = importlib.util.spec_from_file_location("autogluon_user_handler", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import AutoGluon inference handler: {path}") + + handler = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = handler + try: + spec.loader.exec_module(handler) + except Exception: + sys.modules.pop(spec.name, None) + raise + return handler + + +def _media_type(value: str | None, default: str) -> str: + value = (value or default).split(",", 1)[0].split(";", 1)[0].strip().lower() + return default if not value or value == "*/*" else value + + +def _request_body(body: bytes, content_type: str) -> bytes | str: + if content_type.startswith("text/") or content_type in ( + "application/json", + "application/jsonl", + ): + return body.decode("utf-8") + return body + + +HANDLER = _load_handler() +if not callable(getattr(HANDLER, "model_fn", None)): + raise TypeError("AutoGluon inference handler must define model_fn") +if not callable(getattr(HANDLER, "transform_fn", None)): + raise TypeError("AutoGluon inference handler must define transform_fn") +MODEL = HANDLER.model_fn(str(MODEL_DIR)) +app = Flask(__name__) + + +@app.get("/ping") +def ping() -> Response: + return Response(status=200) + + +@app.post("/invocations") +def invocations() -> Response: + content_type = _media_type(request.headers.get("content-type"), "application/json") + accept = _media_type(request.headers.get("accept"), DEFAULT_ACCEPT) + + try: + body = _request_body(request.get_data(cache=False), content_type) + result = HANDLER.transform_fn(MODEL, body, content_type, accept) + if not isinstance(result, tuple) or len(result) != 2: + raise TypeError("transform_fn must return (body, content_type)") + output, output_content_type = result + except ValueError as error: + return jsonify(detail=str(error)), 400 + except Exception: + LOGGER.exception("AutoGluon inference failed") + return jsonify(detail="inference failed"), 500 + + output_content_type = _media_type(output_content_type, accept) + if isinstance(output, (bytes, str)): + return Response(response=output, content_type=output_content_type) + return ( + jsonify(detail=f"handler returned {type(output).__name__} for {output_content_type}"), + 500, + ) diff --git a/scripts/docker/telemetry/deep_learning_container.py b/scripts/docker/telemetry/deep_learning_container.py index 6601d5cc86d1..9c586ddf098d 100755 --- a/scripts/docker/telemetry/deep_learning_container.py +++ b/scripts/docker/telemetry/deep_learning_container.py @@ -230,6 +230,7 @@ def parse_args(): parser.add_argument( "--framework", choices=[ + "autogluon", "tensorflow", "mxnet", "pytorch", diff --git a/test/autogluon/inference/test_server.py b/test/autogluon/inference/test_server.py new file mode 100644 index 000000000000..063e6743412b --- /dev/null +++ b/test/autogluon/inference/test_server.py @@ -0,0 +1,371 @@ +"""CPU-only tests for the AutoGluon SageMaker server.""" + +from __future__ import annotations + +import importlib.util +import sys +import time +import types +from pathlib import Path + +import pytest + +SERVER_DIR = Path(__file__).resolve().parents[3] / "scripts" / "docker" / "autogluon" +SERVER_PATH = SERVER_DIR / "server.py" +INSTALL_REQUIREMENTS_PATH = SERVER_DIR / "install_requirements.py" +GUNICORN_CONFIG_PATH = SERVER_DIR / "gunicorn.conf.py" + +SERVING_ENVIRONMENT_VARIABLES = { + "SAGEMAKER_PROGRAM", + "SAGEMAKER_BIND_TO_PORT", + "SAGEMAKER_MODEL_SERVER_WORKERS", + "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT", + "SAGEMAKER_CONTAINER_LOG_LEVEL", + "SAGEMAKER_BASE_DIR", + "CA_REPOSITORY_ARN", +} + + +def _load(path: Path, prefix: str): + module_name = f"{prefix}_{time.time_ns()}" + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _clear_serving_environment(monkeypatch): + for name in SERVING_ENVIRONMENT_VARIABLES: + monkeypatch.delenv(name, raising=False) + + +def _write_handler(base_dir: Path, source: str, program: str = "inference.py") -> Path: + model_dir = base_dir / "model" + code_dir = model_dir / "code" + code_dir.mkdir(parents=True) + (code_dir / program).write_text(source) + return model_dir + + +def _load_server(monkeypatch, base_dir: Path, program: str | None = None): + _clear_serving_environment(monkeypatch) + monkeypatch.setenv("SAGEMAKER_BASE_DIR", str(base_dir)) + if program is not None: + monkeypatch.setenv("SAGEMAKER_PROGRAM", program) + monkeypatch.syspath_prepend(str(SERVER_DIR)) + return _load(SERVER_PATH, "autogluon_server") + + +def _request(app, method: str, path: str, **kwargs): + with app.test_client() as client: + return client.open(path, method=method, **kwargs) + + +def test_model_artifact_handler_receives_normalized_json_headers(monkeypatch, tmp_path): + model_dir = _write_handler( + tmp_path, + """ +import json + +def model_fn(model_dir): + return {"model_dir": model_dir} + +def transform_fn(model, body, content_type, accept): + return json.dumps({ + "body": body, + "content_type": content_type, + "accept": accept, + "model_dir": model["model_dir"], + }), "application/json" +""", + ) + server = _load_server(monkeypatch, tmp_path) + + assert _request(server.app, "GET", "/ping").data == b"" + response = _request( + server.app, + "POST", + "/invocations", + data='[{"feature": 1}]', + headers={ + "content-type": "application/json; charset=utf-8", + "accept": "application/json, */*", + }, + ) + + assert response.status_code == 200 + assert response.get_json() == { + "body": '[{"feature": 1}]', + "content_type": "application/json", + "accept": "application/json", + "model_dir": str(model_dir), + } + + +def test_binary_request_and_response_are_not_decoded(monkeypatch, tmp_path): + _write_handler( + tmp_path, + """ +def model_fn(model_dir): + return None + +def transform_fn(model, body, content_type, accept): + assert isinstance(body, bytes) + assert content_type == "application/x-parquet" + return body[::-1], "application/x-parquet" +""", + ) + server = _load_server(monkeypatch, tmp_path) + + response = _request( + server.app, + "POST", + "/invocations", + data=b"parquet-bytes", + headers={"content-type": "application/x-parquet", "accept": "application/x-parquet"}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"] == "application/x-parquet" + assert response.data == b"setyb-teuqrap" + + +@pytest.mark.parametrize( + "content_type,body", + [ + ("application/json", '{"feature": 1}'), + ("application/jsonl", '{"feature": 1}\n{"feature": 2}'), + ("text/csv", "feature\n1"), + ], +) +def test_text_request_body_is_decoded(monkeypatch, tmp_path, content_type, body): + _write_handler( + tmp_path, + """ +def model_fn(model_dir): + return None + +def transform_fn(model, body, content_type, accept): + assert isinstance(body, str) + return body, content_type +""", + ) + server = _load_server(monkeypatch, tmp_path) + + response = _request( + server.app, + "POST", + "/invocations", + data=body, + headers={"content-type": content_type}, + ) + + assert response.status_code == 200 + assert response.text == body + assert response.headers["content-type"] == content_type + + +def test_default_accept_environment_variable(monkeypatch, tmp_path): + _write_handler( + tmp_path, + """ +def model_fn(model_dir): + return None + +def transform_fn(model, body, content_type, accept): + return accept, "text/plain" +""", + ) + _clear_serving_environment(monkeypatch) + monkeypatch.setenv("SAGEMAKER_BASE_DIR", str(tmp_path)) + monkeypatch.setenv("SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT", "text/csv") + monkeypatch.syspath_prepend(str(SERVER_DIR)) + server = _load(SERVER_PATH, "autogluon_server") + + response = _request(server.app, "POST", "/invocations", data=b"input") + + assert response.status_code == 200 + assert response.text == "text/csv" + + +def test_sagemaker_program_selects_custom_handler(monkeypatch, tmp_path): + _write_handler( + tmp_path, + """ +def model_fn(model_dir): + return model_dir + +def transform_fn(model, body, content_type, accept): + return model, "text/plain" +""", + program="tabular_serve.py", + ) + server = _load_server(monkeypatch, tmp_path, program="tabular_serve.py") + + response = _request(server.app, "POST", "/invocations", data=b"input") + + assert response.status_code == 200 + assert response.text == str(tmp_path / "model") + + +def test_handler_module_is_registered_during_execution(monkeypatch, tmp_path): + _write_handler( + tmp_path, + """ +from __future__ import annotations + +from dataclasses import dataclass + +@dataclass +class Model: + model_dir: str + +def model_fn(model_dir): + return Model(model_dir) + +def transform_fn(model, body, content_type, accept): + return model.model_dir, "text/plain" +""", + ) + server = _load_server(monkeypatch, tmp_path) + + response = _request(server.app, "POST", "/invocations", data=b"input") + + assert response.status_code == 200 + assert response.text == str(tmp_path / "model") + + +def test_handler_value_error_is_a_client_error(monkeypatch, tmp_path): + _write_handler( + tmp_path, + """ +def model_fn(model_dir): + return None + +def transform_fn(model, body, content_type, accept): + raise ValueError("unsupported payload") +""", + ) + server = _load_server(monkeypatch, tmp_path) + + response = _request(server.app, "POST", "/invocations", data=b"bad") + + assert response.status_code == 400 + assert response.get_json()["detail"] == "unsupported payload" + + +def test_gunicorn_environment_configuration(monkeypatch): + _clear_serving_environment(monkeypatch) + monkeypatch.setenv("SAGEMAKER_BIND_TO_PORT", "9000") + monkeypatch.setenv("SAGEMAKER_MODEL_SERVER_WORKERS", "3") + monkeypatch.setenv("SAGEMAKER_MODEL_SERVER_TIMEOUT", "120") + monkeypatch.setenv("SAGEMAKER_CONTAINER_LOG_LEVEL", "10") + + gunicorn_config = _load(GUNICORN_CONFIG_PATH, "autogluon_gunicorn_config") + + assert gunicorn_config.bind == "0.0.0.0:9000" + assert gunicorn_config.workers == 3 + assert gunicorn_config.timeout == 120 + assert gunicorn_config.graceful_timeout == 120 + assert gunicorn_config.loglevel == "debug" + assert gunicorn_config.worker_class == "sync" + + +def test_requirements_installed_once_with_codeartifact_index(monkeypatch, tmp_path): + model_dir = _write_handler( + tmp_path, + """ +def model_fn(model_dir): + return None + +def transform_fn(model, body, content_type, accept): + return body +""", + ) + requirements = model_dir / "code" / "requirements.txt" + requirements.write_text("customer-package==1.2.3\n") + _clear_serving_environment(monkeypatch) + monkeypatch.setenv("SAGEMAKER_BASE_DIR", str(tmp_path)) + monkeypatch.setenv( + "CA_REPOSITORY_ARN", + "arn:aws:codeartifact:us-west-2:123456789012:repository/domain/repository", + ) + monkeypatch.syspath_prepend(str(SERVER_DIR)) + installer = _load(INSTALL_REQUIREMENTS_PATH, "autogluon_install_requirements") + monkeypatch.setattr( + installer, + "_codeartifact_index", + lambda arn: "https://aws:secret@example.com/pypi/repository/simple/", + ) + calls = [] + monkeypatch.setattr( + installer.subprocess, + "check_call", + lambda command, **kwargs: calls.append((command, kwargs)), + ) + + installer.install_requirements() + + assert len(calls) == 1 + command, kwargs = calls[0] + assert command == [ + "uv", + "pip", + "install", + "--python", + sys.executable, + "--no-cache", + "-r", + str(requirements), + ] + assert "secret" not in " ".join(command) + assert kwargs["env"]["UV_INDEX_URL"] == ( + "https://aws:secret@example.com/pypi/repository/simple/" + ) + + +def test_codeartifact_arn_is_resolved_without_logging_credentials(monkeypatch): + monkeypatch.syspath_prepend(str(SERVER_DIR)) + installer = _load(INSTALL_REQUIREMENTS_PATH, "autogluon_install_requirements") + calls = [] + + class CodeArtifactClient: + def get_authorization_token(self, **kwargs): + calls.append(("token", kwargs)) + return {"authorizationToken": "token+/="} + + def get_repository_endpoint(self, **kwargs): + calls.append(("endpoint", kwargs)) + return {"repositoryEndpoint": "https://example.com/pypi/repository/"} + + boto3 = types.SimpleNamespace( + client=lambda service, region_name: ( + calls.append(("client", service, region_name)) or CodeArtifactClient() + ) + ) + monkeypatch.setitem(sys.modules, "boto3", boto3) + + index = installer._codeartifact_index( + "arn:aws:codeartifact:us-west-2:123456789012:repository/domain/repository" + ) + + assert index == "https://aws:token%2B%2F%3D@example.com/pypi/repository/simple/" + assert calls == [ + ("client", "codeartifact", "us-west-2"), + ( + "token", + {"domain": "domain", "domainOwner": "123456789012"}, + ), + ( + "endpoint", + { + "domain": "domain", + "domainOwner": "123456789012", + "repository": "repository", + "format": "pypi", + }, + ), + ] diff --git a/test/autogluon/sagemaker/requirements.txt b/test/autogluon/sagemaker/requirements.txt new file mode 100644 index 000000000000..521e63c93a43 --- /dev/null +++ b/test/autogluon/sagemaker/requirements.txt @@ -0,0 +1 @@ +sagemaker>=3.0.0 diff --git a/test/autogluon/sagemaker/resources/code/tabular/serve.py b/test/autogluon/sagemaker/resources/code/tabular/serve.py new file mode 100644 index 000000000000..c2ad36d4fd3e --- /dev/null +++ b/test/autogluon/sagemaker/resources/code/tabular/serve.py @@ -0,0 +1,23 @@ +"""Inference handler packaged with the trained AutoGluon tabular model.""" + +from io import StringIO + +import pandas as pd +from autogluon.tabular import TabularPredictor + + +def model_fn(model_dir): + predictor = TabularPredictor.load(model_dir) + predictor.persist() + return predictor + + +def transform_fn(model, request_body, input_content_type, output_content_type="application/json"): + if input_content_type != "application/json": + raise ValueError(f"unsupported input content type: {input_content_type}") + if output_content_type != "application/json": + raise ValueError(f"unsupported output content type: {output_content_type}") + + data = pd.read_json(StringIO(request_body)) + predictions = model.predict(data, as_pandas=True) + return predictions.to_frame(name=model.label).to_json(orient="records"), "application/json" diff --git a/test/autogluon/sagemaker/resources/code/tabular/train.py b/test/autogluon/sagemaker/resources/code/tabular/train.py new file mode 100644 index 000000000000..f4fd2cc945db --- /dev/null +++ b/test/autogluon/sagemaker/resources/code/tabular/train.py @@ -0,0 +1,49 @@ +"""Train an AutoGluon tabular predictor and package its inference handler.""" + +import os +import shutil +from pathlib import Path + +from autogluon.tabular import TabularDataset, TabularPredictor + + +def first_file(directory): + return Path(directory) / min(os.listdir(directory)) + + +if __name__ == "__main__": + model_dir = Path(os.environ["SM_MODEL_DIR"]) + num_gpus = int(os.environ.get("SM_NUM_GPUS", "0")) + + if num_gpus: + import torch + + assert torch.cuda.is_available(), "SM_NUM_GPUS > 0 but CUDA is unavailable" + + fit_args = { + "hyperparameters": { + "GBM": {"num_boost_round": 20}, + "NN_TORCH": {"num_epochs": 2}, + }, + "num_bag_folds": 3, + "num_bag_sets": 1, + "num_stack_levels": 0, + } + if num_gpus: + fit_args["num_gpus"] = num_gpus + + predictor = TabularPredictor( + label="class", + eval_metric="roc_auc", + path=str(model_dir), + ).fit( + TabularDataset(first_file(os.environ["SM_CHANNEL_TRAIN"])), + **fit_args, + ) + + assert TabularPredictor.load(str(model_dir)).model_names(), "reloaded predictor has no models" + + resource_dir = Path(__file__).parent + code_dir = model_dir / "code" + code_dir.mkdir(exist_ok=True) + shutil.copy2(resource_dir / "serve.py", code_dir / "serve.py") diff --git a/test/autogluon/sagemaker/resources/code/timeseries/serve.py b/test/autogluon/sagemaker/resources/code/timeseries/serve.py new file mode 100644 index 000000000000..abd7951492b9 --- /dev/null +++ b/test/autogluon/sagemaker/resources/code/timeseries/serve.py @@ -0,0 +1,27 @@ +"""Inference handler packaged with the trained AutoGluon time series model.""" + +from io import StringIO + +import pandas as pd +from autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor + + +def model_fn(model_dir): + predictor = TimeSeriesPredictor.load(model_dir) + predictor.persist() + return predictor + + +def transform_fn(model, request_body, input_content_type, output_content_type="application/json"): + if input_content_type != "application/json": + raise ValueError(f"unsupported input content type: {input_content_type}") + if output_content_type != "application/json": + raise ValueError(f"unsupported output content type: {output_content_type}") + + data = TimeSeriesDataFrame.from_data_frame( + pd.read_json(StringIO(request_body)), + id_column="item_id", + timestamp_column="timestamp", + ) + predictions = model.predict(data).reset_index() + return predictions.to_json(orient="records", date_format="iso"), "application/json" diff --git a/test/autogluon/sagemaker/resources/code/timeseries/train.py b/test/autogluon/sagemaker/resources/code/timeseries/train.py new file mode 100644 index 000000000000..cf34bd831bb6 --- /dev/null +++ b/test/autogluon/sagemaker/resources/code/timeseries/train.py @@ -0,0 +1,60 @@ +"""Train an AutoGluon time series predictor and package its inference handler.""" + +import os +import shutil +from pathlib import Path + +import pandas as pd +from autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor + +PREDICTION_LENGTH = 5 + + +def first_file(directory): + return Path(directory) / min(os.listdir(directory)) + + +if __name__ == "__main__": + model_dir = Path(os.environ["SM_MODEL_DIR"]) + num_gpus = int(os.environ.get("SM_NUM_GPUS", "0")) + + if num_gpus: + import torch + + assert torch.cuda.is_available(), "SM_NUM_GPUS > 0 but CUDA is unavailable" + + train_data = TimeSeriesDataFrame.from_data_frame( + pd.read_csv(first_file(os.environ["SM_CHANNEL_TRAIN"])), + id_column="item_id", + timestamp_column="timestamp", + ) + predictor = TimeSeriesPredictor( + prediction_length=PREDICTION_LENGTH, + target="target", + path=str(model_dir), + ).fit( + train_data, + hyperparameters={ + "DeepAR": { + "batch_size": 4, + "hidden_size": 8, + "max_epochs": 2, + "num_batches_per_epoch": 2, + "num_layers": 1, + "trainer_kwargs": { + "accelerator": "gpu" if num_gpus else "cpu", + "devices": 1, + }, + }, + "SeasonalNaive": {"seasonal_period": 6, "n_jobs": 1}, + }, + enable_ensemble=False, + ) + + reloaded = TimeSeriesPredictor.load(str(model_dir)) + assert {"DeepAR", "SeasonalNaive"} <= set(reloaded.model_names()) + + resource_dir = Path(__file__).parent + code_dir = model_dir / "code" + code_dir.mkdir(exist_ok=True) + shutil.copy2(resource_dir / "serve.py", code_dir / "serve.py") diff --git a/test/autogluon/sagemaker/resources/data/tabular/train.csv b/test/autogluon/sagemaker/resources/data/tabular/train.csv new file mode 100644 index 000000000000..23f83dd758e0 --- /dev/null +++ b/test/autogluon/sagemaker/resources/data/tabular/train.csv @@ -0,0 +1,101 @@ +age,workclass,fnlwgt,education,education-num,marital-status,occupation,relationship,race,sex,capital-gain,capital-loss,hours-per-week,native-country,class +25, Private,178478, Bachelors,13, Never-married, Tech-support, Own-child, White, Female,0,0,40, United-States, <=50K +23, State-gov,61743, 5th-6th,3, Never-married, Transport-moving, Not-in-family, White, Male,0,0,35, United-States, <=50K +46, Private,376789, HS-grad,9, Never-married, Other-service, Not-in-family, White, Male,0,0,15, United-States, <=50K +55, ?,200235, HS-grad,9, Married-civ-spouse, ?, Husband, White, Male,0,0,50, United-States, >50K +36, Private,224541, 7th-8th,4, Married-civ-spouse, Handlers-cleaners, Husband, White, Male,0,0,40, El-Salvador, <=50K +51, Private,178054, Some-college,10, Married-civ-spouse, Sales, Husband, White, Male,0,0,40, ?, >50K +33, Private,263561, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,60, United-States, >50K +46, Private,173613, HS-grad,9, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K +18, Private,214617, Some-college,10, Never-married, Handlers-cleaners, Own-child, White, Male,0,0,30, United-States, <=50K +43, Private,84661, Assoc-voc,11, Married-civ-spouse, Sales, Husband, White, Male,0,0,45, United-States, <=50K +41, Private,225892, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,48, United-States, >50K +41, Private,191547, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,99999,0,55, United-States, >50K +22, Private,150175, Some-college,10, Never-married, Adm-clerical, Own-child, White, Female,0,0,20, United-States, <=50K +41, Private,227890, Some-college,10, Divorced, Craft-repair, Not-in-family, White, Male,0,0,46, United-States, >50K +39, Private,80479, Bachelors,13, Divorced, Transport-moving, Not-in-family, White, Male,0,0,55, United-States, <=50K +44, Local-gov,100479, Assoc-acdm,12, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,48, United-States, <=50K +37, Private,191524, Assoc-voc,11, Separated, Prof-specialty, Own-child, White, Female,0,0,38, United-States, <=50K +39, Private,173175, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,15024,0,50, United-States, >50K +40, Private,72887, Assoc-voc,11, Married-civ-spouse, Machine-op-inspct, Husband, Asian-Pac-Islander, Male,0,0,40, United-States, >50K +24, Private,295763, Bachelors,13, Never-married, Exec-managerial, Own-child, White, Female,0,0,50, United-States, <=50K +61, ?,451327, Bachelors,13, Married-civ-spouse, ?, Husband, Other, Male,0,0,24, United-States, >50K +39, Private,188540, HS-grad,9, Divorced, Exec-managerial, Not-in-family, White, Male,0,0,45, United-States, <=50K +19, Private,318061, Some-college,10, Never-married, Machine-op-inspct, Not-in-family, White, Female,0,0,80, United-States, <=50K +27, Private,70034, Some-college,10, Never-married, Other-service, Not-in-family, White, Male,0,0,40, United-States, <=50K +47, Private,269620, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, ?, <=50K +47, Private,181758, HS-grad,9, Never-married, Transport-moving, Not-in-family, Black, Male,0,0,40, United-States, <=50K +51, State-gov,103063, Bachelors,13, Married-civ-spouse, Protective-serv, Husband, White, Male,7298,0,40, United-States, >50K +34, Private,154667, HS-grad,9, Married-civ-spouse, Machine-op-inspct, Wife, White, Female,0,0,40, United-States, <=50K +29, Private,198704, Assoc-voc,11, Never-married, Machine-op-inspct, Not-in-family, White, Male,0,0,40, United-States, <=50K +23, Private,134045, Assoc-voc,11, Never-married, Adm-clerical, Unmarried, White, Female,0,0,40, United-States, <=50K +45, Private,353083, Some-college,10, Separated, Other-service, Unmarried, White, Female,0,0,40, United-States, <=50K +55, Private,160631, HS-grad,9, Married-civ-spouse, Sales, Husband, White, Male,0,0,56, United-States, >50K +31, Private,80511, Assoc-acdm,12, Divorced, Tech-support, Not-in-family, White, Female,0,0,44, United-States, <=50K +59, Private,96459, 11th,7, Never-married, Machine-op-inspct, Not-in-family, White, Male,0,0,40, United-States, <=50K +24, Private,233280, Assoc-acdm,12, Never-married, Sales, Own-child, White, Female,0,0,37, United-States, <=50K +35, Self-emp-not-inc,278557, HS-grad,9, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,30, United-States, <=50K +63, Local-gov,80655, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K +43, Private,334991, Bachelors,13, Married-civ-spouse, Sales, Husband, White, Male,0,0,50, United-States, >50K +49, Private,122206, Assoc-voc,11, Married-civ-spouse, Sales, Husband, White, Male,0,0,25, United-States, <=50K +23, Private,436798, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, United-States, <=50K +44, Private,374423, Masters,14, Married-civ-spouse, Exec-managerial, Husband, Black, Male,0,1902,40, United-States, >50K +49, Private,50567, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,32, United-States, <=50K +18, Private,90934, Some-college,10, Never-married, Sales, Own-child, White, Male,0,0,28, United-States, <=50K +21, Private,117210, HS-grad,9, Never-married, Machine-op-inspct, Own-child, White, Male,0,0,40, United-States, <=50K +34, Private,179877, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, United-States, >50K +69, Private,361561, Prof-school,15, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,3, United-States, <=50K +38, Private,247111, HS-grad,9, Married-civ-spouse, Tech-support, Husband, White, Male,0,0,40, United-States, >50K +37, Private,156266, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, Amer-Indian-Eskimo, Male,0,0,40, United-States, <=50K +42, Private,173938, HS-grad,9, Separated, Sales, Own-child, White, Female,0,0,40, United-States, <=50K +43, Self-emp-inc,260960, Bachelors,13, Divorced, Farming-fishing, Not-in-family, White, Male,0,0,35, United-States, <=50K +58, Private,49893, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,50, United-States, >50K +29, Self-emp-not-inc,58744, Assoc-acdm,12, Never-married, Other-service, Own-child, White, Male,0,0,60, United-States, <=50K +51, Local-gov,74784, Some-college,10, Widowed, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K +51, Private,54465, Assoc-voc,11, Divorced, Prof-specialty, Not-in-family, White, Female,0,0,48, United-States, <=50K +31, Private,169122, HS-grad,9, Divorced, Other-service, Not-in-family, White, Male,0,0,28, United-States, <=50K +40, Private,131899, Some-college,10, Divorced, Exec-managerial, Not-in-family, White, Male,0,0,40, United-States, <=50K +46, Private,423222, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,3103,0,60, United-States, >50K +48, State-gov,327886, Doctorate,16, Divorced, Prof-specialty, Own-child, White, Male,0,0,50, United-States, >50K +29, Private,342989, HS-grad,9, Never-married, Adm-clerical, Own-child, White, Female,0,0,40, United-States, <=50K +69, State-gov,170458, HS-grad,9, Married-civ-spouse, Other-service, Husband, White, Male,0,0,20, United-States, <=50K +19, Private,239057, HS-grad,9, Never-married, Craft-repair, Own-child, White, Male,0,0,40, United-States, <=50K +20, Private,221661, HS-grad,9, Married-civ-spouse, Adm-clerical, Wife, White, Female,0,0,40, Mexico, <=50K +26, Private,292692, 12th,8, Never-married, Craft-repair, Not-in-family, White, Male,0,0,40, Mexico, <=50K +45, Private,461725, HS-grad,9, Married-civ-spouse, Handlers-cleaners, Husband, White, Male,0,0,40, United-States, <=50K +48, Local-gov,148121, Bachelors,13, Married-spouse-absent, Adm-clerical, Unmarried, Asian-Pac-Islander, Female,0,0,40, Philippines, <=50K +56, Private,174209, 5th-6th,3, Married-civ-spouse, Machine-op-inspct, Husband, Black, Male,0,0,40, United-States, <=50K +72, Self-emp-inc,149689, Prof-school,15, Married-civ-spouse, Prof-specialty, Husband, White, Male,20051,0,48, United-States, >50K +53, Private,164198, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, Black, Male,0,0,40, United-States, >50K +50, Private,33931, HS-grad,9, Divorced, Craft-repair, Not-in-family, White, Male,0,0,40, United-States, <=50K +42, Self-emp-not-inc,27821, Assoc-voc,11, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,60, United-States, <=50K +19, Private,369463, HS-grad,9, Never-married, Adm-clerical, Own-child, White, Female,0,0,40, United-States, <=50K +43, Private,152958, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,7298,0,40, United-States, >50K +19, Self-emp-not-inc,194205, Some-college,10, Never-married, Other-service, Own-child, White, Female,0,0,40, Mexico, <=50K +18, Private,201871, 12th,8, Never-married, Other-service, Own-child, White, Male,0,0,20, United-States, <=50K +49, Self-emp-inc,34998, Some-college,10, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,60, United-States, <=50K +41, Private,110562, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K +49, Local-gov,159641, Bachelors,13, Divorced, Exec-managerial, Unmarried, White, Female,0,625,40, United-States, <=50K +23, Private,278391, HS-grad,9, Never-married, Transport-moving, Own-child, White, Male,0,0,40, United-States, <=50K +41, Private,175642, Some-college,10, Never-married, Craft-repair, Not-in-family, White, Female,0,0,40, United-States, <=50K +35, Private,317153, HS-grad,9, Never-married, Sales, Not-in-family, White, Female,0,0,40, United-States, <=50K +65, Private,90907, 5th-6th,3, Married-civ-spouse, Other-service, Husband, Asian-Pac-Islander, Male,0,0,40, United-States, <=50K +61, Private,159822, 7th-8th,4, Married-civ-spouse, Craft-repair, Husband, White, Male,7688,0,40, Poland, >50K +51, Private,205100, Some-college,10, Divorced, Sales, Not-in-family, White, Female,0,0,45, United-States, >50K +64, Private,207188, Bachelors,13, Divorced, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K +26, ?,102541, 10th,6, Never-married, ?, Unmarried, Amer-Indian-Eskimo, Female,0,0,40, United-States, <=50K +33, Private,182401, 10th,6, Never-married, Adm-clerical, Not-in-family, Black, Male,0,0,40, United-States, <=50K +23, Private,348092, HS-grad,9, Never-married, Transport-moving, Own-child, Black, Male,0,0,40, Haiti, <=50K +39, Private,144169, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K +27, Private,167031, Bachelors,13, Never-married, Prof-specialty, Unmarried, Other, Female,0,0,33, United-States, <=50K +66, Private,216856, Masters,14, Married-civ-spouse, Sales, Husband, White, Male,0,0,50, United-States, <=50K +23, Private,214236, HS-grad,9, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K +22, Private,52596, Some-college,10, Never-married, Sales, Own-child, White, Male,0,0,8, United-States, <=50K +22, Private,176131, Some-college,10, Married-civ-spouse, Exec-managerial, Husband, Black, Male,0,0,40, United-States, <=50K +61, Private,137733, Some-college,10, Divorced, Other-service, Not-in-family, White, Male,0,0,25, United-States, <=50K +18, Private,400616, Some-college,10, Never-married, Adm-clerical, Own-child, White, Male,0,0,40, United-States, <=50K +43, Private,26252, Some-college,10, Separated, Other-service, Unmarried, Amer-Indian-Eskimo, Female,0,0,36, United-States, <=50K +29, Federal-gov,182344, HS-grad,9, Married-spouse-absent, Other-service, Unmarried, Black, Male,0,0,40, United-States, <=50K +31, Private,91964, Some-college,10, Never-married, Adm-clerical, Other-relative, White, Male,0,0,40, United-States, <=50K +31, Private,359249, Assoc-voc,11, Never-married, Protective-serv, Own-child, Black, Male,0,0,40, United-States, <=50K +34, Private,24529, HS-grad,9, Widowed, Adm-clerical, Unmarried, White, Male,0,0,15, United-States, <=50K diff --git a/test/autogluon/sagemaker/resources/data/timeseries/train.csv b/test/autogluon/sagemaker/resources/data/timeseries/train.csv new file mode 100644 index 000000000000..add96af365ac --- /dev/null +++ b/test/autogluon/sagemaker/resources/data/timeseries/train.csv @@ -0,0 +1,61 @@ +item_id,timestamp,target +series_1,2024-01-01 00:00:00,10 +series_1,2024-01-01 01:00:00,12 +series_1,2024-01-01 02:00:00,15 +series_1,2024-01-01 03:00:00,14 +series_1,2024-01-01 04:00:00,11 +series_1,2024-01-01 05:00:00,9 +series_1,2024-01-01 06:00:00,11 +series_1,2024-01-01 07:00:00,13 +series_1,2024-01-01 08:00:00,16 +series_1,2024-01-01 09:00:00,15 +series_1,2024-01-01 10:00:00,12 +series_1,2024-01-01 11:00:00,10 +series_1,2024-01-01 12:00:00,12 +series_1,2024-01-01 13:00:00,14 +series_1,2024-01-01 14:00:00,17 +series_1,2024-01-01 15:00:00,16 +series_1,2024-01-01 16:00:00,13 +series_1,2024-01-01 17:00:00,11 +series_1,2024-01-01 18:00:00,13 +series_1,2024-01-01 19:00:00,15 +series_1,2024-01-01 20:00:00,18 +series_1,2024-01-01 21:00:00,17 +series_1,2024-01-01 22:00:00,14 +series_1,2024-01-01 23:00:00,12 +series_1,2024-01-02 00:00:00,14 +series_1,2024-01-02 01:00:00,16 +series_1,2024-01-02 02:00:00,19 +series_1,2024-01-02 03:00:00,18 +series_1,2024-01-02 04:00:00,15 +series_1,2024-01-02 05:00:00,13 +series_2,2024-01-01 00:00:00,30 +series_2,2024-01-01 01:00:00,28 +series_2,2024-01-01 02:00:00,27 +series_2,2024-01-01 03:00:00,29 +series_2,2024-01-01 04:00:00,32 +series_2,2024-01-01 05:00:00,34 +series_2,2024-01-01 06:00:00,31 +series_2,2024-01-01 07:00:00,29 +series_2,2024-01-01 08:00:00,28 +series_2,2024-01-01 09:00:00,30 +series_2,2024-01-01 10:00:00,33 +series_2,2024-01-01 11:00:00,35 +series_2,2024-01-01 12:00:00,32 +series_2,2024-01-01 13:00:00,30 +series_2,2024-01-01 14:00:00,29 +series_2,2024-01-01 15:00:00,31 +series_2,2024-01-01 16:00:00,34 +series_2,2024-01-01 17:00:00,36 +series_2,2024-01-01 18:00:00,33 +series_2,2024-01-01 19:00:00,31 +series_2,2024-01-01 20:00:00,30 +series_2,2024-01-01 21:00:00,32 +series_2,2024-01-01 22:00:00,35 +series_2,2024-01-01 23:00:00,37 +series_2,2024-01-02 00:00:00,34 +series_2,2024-01-02 01:00:00,32 +series_2,2024-01-02 02:00:00,31 +series_2,2024-01-02 03:00:00,33 +series_2,2024-01-02 04:00:00,36 +series_2,2024-01-02 05:00:00,38 diff --git a/test/autogluon/sagemaker/test_train_and_serve.py b/test/autogluon/sagemaker/test_train_and_serve.py new file mode 100644 index 000000000000..d7ffc82a0fc6 --- /dev/null +++ b/test/autogluon/sagemaker/test_train_and_serve.py @@ -0,0 +1,193 @@ +"""Train with the unified AutoGluon image, then serve the resulting artifact.""" + +import csv +import json +import logging +import os +from pathlib import Path + +import pytest +from sagemaker.core.resources import Endpoint, EndpointConfig, Model +from sagemaker.core.shapes import ContainerDefinition, ProductionVariant +from sagemaker.core.training.configs import Compute, InputData, SourceCode +from sagemaker.train import ModelTrainer +from test_utils import random_suffix_name +from test_utils.constants import INFERENCE_AMI_VERSION + +LOGGER = logging.getLogger(__name__) +RESOURCE_DIR = Path(__file__).parent / "resources" +CODE_DIR = RESOURCE_DIR / "code" +DATA_DIR = RESOURCE_DIR / "data" +PREDICTION_LENGTH = 5 + +_WORKLOADS = { + "tabular": { + "source_dir": CODE_DIR / "tabular", + "data": DATA_DIR / "tabular" / "train.csv", + }, + "timeseries": { + "source_dir": CODE_DIR / "timeseries", + "data": DATA_DIR / "timeseries" / "train.csv", + }, +} + +_INSTANCE_BY_DEVICE = { + "cpu": { + "training": "ml.m5.2xlarge", + "inference": "ml.c5.xlarge", + }, + "gpu": { + "training": "ml.g4dn.2xlarge", + "inference": "ml.g6.xlarge", + }, +} +_NUMERIC_COLUMNS = { + "age", + "fnlwgt", + "education-num", + "capital-gain", + "capital-loss", + "hours-per-week", +} + + +def _cleanup(resources): + for resource in reversed(resources): + try: + resource.delete() + except Exception as error: # noqa: BLE001 - cleanup must not mask test failures + LOGGER.warning("Cleanup of %s failed: %s", type(resource).__name__, error) + + +@pytest.fixture(scope="module") +def device_config(): + device = os.environ.get("SM_DEVICE_TYPE", "").lower() + assert device in _INSTANCE_BY_DEVICE, f"SM_DEVICE_TYPE must be cpu or gpu, got {device!r}" + return device, _INSTANCE_BY_DEVICE[device] + + +@pytest.fixture(scope="module", params=_WORKLOADS) +def workload(request): + return request.param, _WORKLOADS[request.param] + + +@pytest.fixture(scope="module") +def model_data_url(image_uri, sagemaker_session, device_config, workload): + """Train once and return the model artifact produced by the tested image.""" + device, instances = device_config + workload_name, config = workload + key_prefix = random_suffix_name(f"ag-e2e-{workload_name}", 32) + trainer = ModelTrainer( + training_image=image_uri, + source_code=SourceCode(source_dir=str(config["source_dir"]), entry_script="train.py"), + compute=Compute(instance_type=instances["training"], instance_count=1), + role=os.environ["SM_ROLE_ARN"], + base_job_name=random_suffix_name(f"ag-{workload_name}-{device}", 32), + ) + trainer.train( + input_data_config=[ + InputData( + channel_name="train", + data_source=sagemaker_session.upload_data( + path=str(config["data"]), + key_prefix=f"{key_prefix}/train", + ), + ) + ], + wait=True, + ) + return trainer._latest_training_job.model_artifacts.s3_model_artifacts + + +@pytest.fixture(scope="module") +def endpoint(aws_session, image_uri, model_data_url, device_config, workload): + """Deploy the training artifact with the same unified image.""" + device, instances = device_config + workload_name, _ = workload + endpoint_name = random_suffix_name(f"ag-{workload_name}", 63) + model_name = random_suffix_name(f"ag-{workload_name}-model", 63) + resources = [] + try: + model = Model.create( + model_name=model_name, + primary_container=ContainerDefinition( + image=image_uri, + model_data_url=model_data_url, + environment={"SAGEMAKER_PROGRAM": "serve.py"}, + ), + execution_role_arn=os.environ["SM_ROLE_ARN"], + session=aws_session.session, + ) + resources.append(model) + + variant = { + "variant_name": "AllTraffic", + "model_name": model_name, + "initial_instance_count": 1, + "instance_type": instances["inference"], + "container_startup_health_check_timeout_in_seconds": 600, + } + if device == "gpu": + variant["inference_ami_version"] = INFERENCE_AMI_VERSION + endpoint_config = EndpointConfig.create( + endpoint_config_name=endpoint_name, + production_variants=[ProductionVariant(**variant)], + session=aws_session.session, + ) + resources.append(endpoint_config) + + deployed = Endpoint.create( + endpoint_name=endpoint_name, + endpoint_config_name=endpoint_name, + session=aws_session.session, + ) + resources.append(deployed) + deployed.wait_for_status("InService", timeout=1800) + yield deployed + finally: + _cleanup(resources) + + +def _assert_tabular_prediction(endpoint, data_path): + with data_path.open(newline="") as data_file: + row = next(csv.DictReader(data_file)) + expected_labels = {"<=50K", ">50K"} + row.pop("class") + for column in _NUMERIC_COLUMNS: + row[column] = int(row[column]) + + response = endpoint.invoke( + body=json.dumps([row]), + content_type="application/json", + accept="application/json", + ) + prediction = json.loads(response.body.read()) + + assert len(prediction) == 1 + assert prediction[0]["class"].strip() in expected_labels + + +def _assert_timeseries_prediction(endpoint, data_path): + with data_path.open(newline="") as data_file: + rows = list(csv.DictReader(data_file)) + for row in rows: + row["target"] = float(row["target"]) + + response = endpoint.invoke( + body=json.dumps(rows), + content_type="application/json", + accept="application/json", + ) + prediction = json.loads(response.body.read()) + + assert len(prediction) == 2 * PREDICTION_LENGTH + assert {row["item_id"] for row in prediction} == {"series_1", "series_2"} + assert all(isinstance(row["mean"], (int, float)) for row in prediction) + + +def test_train_then_serve(endpoint, workload): + workload_name, config = workload + if workload_name == "tabular": + _assert_tabular_prediction(endpoint, config["data"]) + else: + _assert_timeseries_prediction(endpoint, config["data"]) diff --git a/test/security/data/ecr_scan_allowlist/autogluon/framework_allowlist.json b/test/security/data/ecr_scan_allowlist/autogluon/framework_allowlist.json new file mode 100644 index 000000000000..048795fd6a0b --- /dev/null +++ b/test/security/data/ecr_scan_allowlist/autogluon/framework_allowlist.json @@ -0,0 +1,12 @@ +[ + { + "vulnerability_id": "CVE-2026-54399", + "reason": "Ray 2.58.0 bundles httpcore5 5.0.2 inside ray_dist.jar. The Java dependency cannot be upgraded independently from the Ray wheel; remediation requires an upstream Ray release that updates the bundled Apache HttpComponents Core.", + "review_by": "2026-10-20" + }, + { + "vulnerability_id": "CVE-2026-58659", + "reason": "False positive for lightning 2.6.6: the advisory affects versions through 2.6.5, and 2.6.6 includes the fix from commit d710d68. The scanner incorrectly reports 2022.6.15 as the fixed version because the advisory's patched-version metadata is malformed.", + "review_by": "2026-10-20" + } +]