Skip to content

Commit c4f9b5b

Browse files
rsareddy0329Roja Reddy Sareddy
andauthored
fix: Add pre-validation for Bedrock and MLflow role permissions (#6162)
* feat(evaluate): Add pre-validation for execution role permissions Validate execution role permissions before submitting evaluation jobs to fail fast with actionable guidance instead of failing during execution. Reuses existing _simulate_denied_actions and _role_trusts_service from iam_role_resolver.py. Applies to all evaluators via BaseEvaluator. * refactor(evaluate): Use evaluation role type via iam_role_resolver Add 'evaluation' role type to IAM_POLICY_CONFIG with Bedrock and MLflow permissions. BaseEvaluator now calls resolve_and_validate_role with role_type='evaluation' which validates permissions and trust using the existing iam_role_resolver infrastructure. Removes the standalone role_permission_validator utility in favor of the existing centralized approach. * fix(evaluate): Address review feedback on role validation - Remove bedrock.amazonaws.com from evaluation trust policy. The serverless backend calls Bedrock using the role's own credentials; Bedrock does not assume the role, so trust is not required. - Update existing base_evaluator tests to expect role_type='evaluation'. - Remove stray blank line in llm_as_judge_evaluator.py. - Update test assertions to reflect trust policy change. * fix: scope model_eval role gate to LLM-as-Judge, address review comments * fix: scope InvokeModel to resource ARN to avoid false denials --------- Co-authored-by: Roja Reddy Sareddy <rsareddy@amazon.com>
1 parent fbcd306 commit c4f9b5b

6 files changed

Lines changed: 386 additions & 12 deletions

File tree

‎sagemaker-core/src/sagemaker/core/helper/iam_policies.py‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -812,6 +812,157 @@
812812
},
813813
},
814814
},
815+
"model_eval": {
816+
"role_name": "SageMaker-AutoRole-Evaluation",
817+
"trust_policy": {
818+
"Version": "2012-10-17",
819+
"Statement": [
820+
{
821+
"Effect": "Allow",
822+
"Principal": {
823+
"Service": "sagemaker.amazonaws.com"
824+
},
825+
"Action": "sts:AssumeRole",
826+
"Condition": {
827+
"StringEquals": {"aws:SourceAccount": "ACCOUNT_PLACEHOLDER"}
828+
},
829+
}
830+
],
831+
},
832+
"policies": {
833+
# --- Training permissions (superset of "training" role type) ---
834+
"s3_policy": {
835+
"Version": "2012-10-17",
836+
"Statement": [
837+
{
838+
"Effect": "Allow",
839+
"Action": [
840+
"s3:GetObject",
841+
"s3:PutObject",
842+
"s3:ListBucket",
843+
"s3:GetBucketLocation",
844+
],
845+
"Resource": "S3_PLACEHOLDER",
846+
}
847+
],
848+
},
849+
"ecr_policy": {
850+
"Version": "2012-10-17",
851+
"Statement": [
852+
{
853+
"Effect": "Allow",
854+
"Action": ["ecr:GetAuthorizationToken"],
855+
"Resource": "*",
856+
},
857+
{
858+
"Effect": "Allow",
859+
"Action": [
860+
"ecr:GetDownloadUrlForLayer",
861+
"ecr:BatchGetImage",
862+
"ecr:BatchCheckLayerAvailability",
863+
],
864+
"Resource": "arn:aws:ecr:*:*:repository/*",
865+
},
866+
],
867+
},
868+
"cloudwatch_policy": {
869+
"Version": "2012-10-17",
870+
"Statement": [
871+
{
872+
"Effect": "Allow",
873+
"Action": ["cloudwatch:PutMetricData"],
874+
"Resource": "*",
875+
},
876+
{
877+
"Effect": "Allow",
878+
"Action": [
879+
"logs:CreateLogGroup",
880+
"logs:CreateLogStream",
881+
"logs:PutLogEvents",
882+
"logs:DescribeLogStreams",
883+
],
884+
"Resource": "arn:aws:logs:*:*:log-group:/aws/sagemaker/TrainingJobs*",
885+
},
886+
],
887+
},
888+
"kms_policy": {
889+
"Version": "2012-10-17",
890+
"Statement": [
891+
{
892+
"Effect": "Allow",
893+
"Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"],
894+
"Resource": "KMS_PLACEHOLDER",
895+
}
896+
],
897+
},
898+
# --- Evaluation-specific permissions ---
899+
"bedrock_evaluation_policy": {
900+
"Version": "2012-10-17",
901+
"Statement": [
902+
{
903+
"Effect": "Allow",
904+
"Action": [
905+
"bedrock:CreateEvaluationJob",
906+
"bedrock:GetEvaluationJob",
907+
],
908+
"Resource": "*",
909+
},
910+
{
911+
"Effect": "Allow",
912+
"Action": [
913+
"bedrock:InvokeModel",
914+
"bedrock:InvokeModelWithResponseStream",
915+
],
916+
"Resource": [
917+
"arn:aws:bedrock:*:*:foundation-model/*",
918+
"arn:aws:bedrock:*::foundation-model/*",
919+
],
920+
},
921+
],
922+
},
923+
"mlflow_policy": {
924+
"Version": "2012-10-17",
925+
"Statement": [
926+
{
927+
"Effect": "Allow",
928+
"Action": [
929+
"sagemaker-mlflow:GetExperimentByName",
930+
"sagemaker-mlflow:CreateExperiment",
931+
"sagemaker-mlflow:CreateRun",
932+
"sagemaker-mlflow:LogBatch",
933+
"sagemaker-mlflow:LogMetric",
934+
"sagemaker-mlflow:LogParam",
935+
"sagemaker-mlflow:SetTag",
936+
"sagemaker-mlflow:UpdateRun",
937+
],
938+
"Resource": "arn:aws:sagemaker:*:*:mlflow-app/*",
939+
}
940+
],
941+
},
942+
"sagemaker_evaluation_policy": {
943+
"Version": "2012-10-17",
944+
"Statement": [
945+
{
946+
"Effect": "Allow",
947+
"Action": [
948+
"sagemaker:CreateTrainingJob",
949+
"sagemaker:DescribeTrainingJob",
950+
"sagemaker:StopTrainingJob",
951+
"sagemaker:CreatePipeline",
952+
"sagemaker:DescribePipeline",
953+
"sagemaker:StartPipelineExecution",
954+
"sagemaker:DescribePipelineExecution",
955+
"sagemaker:AddTags",
956+
],
957+
"Resource": [
958+
"arn:aws:sagemaker:*:*:training-job/*",
959+
"arn:aws:sagemaker:*:*:pipeline/*",
960+
],
961+
}
962+
],
963+
},
964+
},
965+
},
815966
}
816967

817968
# Actions the *caller* must have to orchestrate Pipeline-based evaluations

‎sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
logger = logging.getLogger(__name__)
2727

28-
ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod")
28+
ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod", "model_eval")
2929

3030
# Permissions the HyperPod CLI flow needs on the *caller* identity — the local
3131
# principal that runs `hyperpod connect-cluster` and `hyperpod start-job`. The CLI

‎sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -738,14 +738,23 @@ def _generate_default_eval_name(cls, v: Optional[str], values: dict) -> str:
738738
return f"eval-{model_name}-{short_uuid}"
739739
return v
740740

741-
def _get_aws_execution_context(self) -> Dict[str, str]:
741+
def _get_aws_execution_context(self, role_type: str = "training") -> Dict[str, str]:
742742
"""Get AWS execution context (role ARN, region, account ID).
743743
744744
Validates both the *execution* role (what the pipeline assumes to run
745745
training jobs) and the *caller* role (what your identity needs to
746746
create/start the pipeline). The execution role is validated via
747-
:func:`resolve_and_validate_role` with ``role_type="training"``. The
748-
caller role is validated via :func:`verify_evaluation_caller_permissions`.
747+
:func:`resolve_and_validate_role`. The caller role is validated via
748+
:func:`verify_evaluation_caller_permissions`.
749+
750+
Args:
751+
role_type (str): Role type to validate the execution role against.
752+
Defaults to ``"training"`` — every evaluator runs as a SageMaker
753+
Pipeline / Training Job, so the standard training permissions are
754+
the right smoke test. Only :class:`LLMAsJudgeEvaluator` passes
755+
``"model_eval"`` to additionally gate the Amazon Bedrock
756+
Evaluations permissions it requires; the other evaluators do not
757+
use Bedrock Evaluations and must not be gated on it.
749758
750759
Returns:
751760
dict: Dictionary containing:
@@ -757,13 +766,13 @@ def _get_aws_execution_context(self) -> Dict[str, str]:
757766
# 1. self.role, if explicitly provided.
758767
# 2. Otherwise the caller's own identity role.
759768
# The resolved role is validated (read-only) for the required permissions.
760-
# This is the job execution role for the
761-
# serverless / SMTJ evaluation backends. The HyperPod backend submits via
762-
# the CLI under the caller's own credentials (see _submit_hyperpod_eval_job)
763-
# and does not resolve a role here, so "training" is always correct here.
769+
# This is the job execution role for the serverless / SMTJ evaluation
770+
# backends. The HyperPod backend submits via the CLI under the caller's own
771+
# credentials (see _submit_hyperpod_eval_job) and does not resolve a role
772+
# here.
764773
role_arn = resolve_and_validate_role(
765774
provided_role=self.role,
766-
role_type="training",
775+
role_type=role_type,
767776
sagemaker_session=self.sagemaker_session,
768777
)
769778

‎sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -753,7 +753,7 @@ def _get_llmaj_template_additions(self, eval_name: str) -> dict:
753753
'top_p': str(1.0),
754754
'evaluate_base_model': self.evaluate_base_model,
755755
}
756-
756+
757757
@_telemetry_emitter(
758758
feature=Feature.MODEL_CUSTOMIZATION,
759759
func_name="LLMAsJudgeEvaluator.evaluate",
@@ -819,8 +819,11 @@ def evaluate(self, dry_run: bool = False):
819819
"Please use a Model Package ARN or JumpStart model ID instead."
820820
)
821821

822-
# Get AWS execution context (role ARN, region, account ID)
823-
aws_context = self._get_aws_execution_context()
822+
# Get AWS execution context (role ARN, region, account ID).
823+
# LLM-as-Judge is the only evaluator backed by Amazon Bedrock Evaluations,
824+
# so it validates the execution role against the "model_eval" role type,
825+
# which additionally gates the required Bedrock permissions.
826+
aws_context = self._get_aws_execution_context(role_type="model_eval")
824827
region = aws_context['region']
825828
role_arn = aws_context['role_arn']
826829

‎sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -787,6 +787,35 @@ def test_get_aws_execution_context_without_region(self, mock_resolve, mock_role,
787787
assert context['region'] == DEFAULT_REGION # From mock_session
788788
assert context['account_id'] == '123456789012'
789789

790+
@patch("sagemaker.train.evaluate.base_evaluator.resolve_and_validate_role")
791+
@patch("sagemaker.train.common_utils.model_resolution._resolve_base_model")
792+
def test_get_aws_execution_context_role_type_override(self, mock_resolve, mock_role, mock_session, mock_model_info):
793+
"""An explicit role_type (e.g. LLM-as-Judge's "model_eval") is forwarded.
794+
795+
Only the Bedrock-backed LLM-as-Judge path passes role_type="model_eval";
796+
all other evaluators keep the "training" default so they are not gated on
797+
Bedrock permissions they never use.
798+
"""
799+
mock_resolve.return_value = mock_model_info
800+
mock_role.return_value = DEFAULT_ROLE_ARN
801+
802+
evaluator = BaseEvaluator(
803+
model=DEFAULT_MODEL,
804+
s3_output_path=DEFAULT_S3_OUTPUT,
805+
mlflow_resource_arn=DEFAULT_MLFLOW_ARN,
806+
model_package_group=DEFAULT_MODEL_PACKAGE_GROUP_ARN,
807+
sagemaker_session=mock_session,
808+
region=DEFAULT_REGION,
809+
)
810+
811+
evaluator._get_aws_execution_context(role_type="model_eval")
812+
813+
mock_role.assert_called_once_with(
814+
provided_role=None,
815+
role_type="model_eval",
816+
sagemaker_session=mock_session,
817+
)
818+
790819

791820
class TestTemplateRendering:
792821
"""Tests for template selection and rendering."""

0 commit comments

Comments
 (0)