fix: refresh LLMAsJudgeEvaluator allowed evaluator models (#5987)

* fix: refresh LLMAsJudgeEvaluator allowed evaluator models

The hardcoded _ALLOWED_EVALUATOR_MODELS list in sagemaker/train/constants.py
had gone stale: several entries were EOL on Bedrock and current judge models
were missing, so LLMAsJudgeEvaluator rejected every currently usable model at
client-side validation.

Re-sync the allowlist (model IDs and per-region availability) with the Bedrock
Console judge-model selector, cross-checked against the Bedrock user guide
supported-evaluator-models list. Adds Claude Sonnet 4 / 4.5, Claude Haiku 4.5,
Claude Opus 4.5, and Nova 2 Lite / Micro / Premier, and widens the region lists
for existing models to match the source of truth.

Update the docstring examples and unit tests that pinned the previous default
judge model (Claude 3.5 Sonnet v1), which is now only available in
ap-northeast-1.

* fix: drop claude-3-5-haiku-20241022 from evaluator allowlist

Remove anthropic.claude-3-5-haiku-20241022-v1:0 from the LLM-as-Judge
evaluator allowlist and update the unit and integration tests that
referenced it as the judge model.

Author: jam-jee

sagemaker-train/src/sagemaker/train/constants.py

@@ -58,15 +58,25 @@ def get_sagemaker_hub_name() -> str:
     "qwen.qwen3-235b-a22b-2507-v1:0": ["us-west-2", "ap-northeast-1"]
 }
 
-# Allowed evaluator models for LLM as Judge evaluator with region restrictions
+# Allowed evaluator models for LLM as Judge evaluator with region restrictions.
+#
+# Source of truth: the Bedrock Console judge-model regional
+# allowlist.cross-checked against
+# https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation-judge.html#evaluation-judge-supported
 _ALLOWED_EVALUATOR_MODELS = {
-    "anthropic.claude-3-5-sonnet-20240620-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1"],
-    "anthropic.claude-3-5-sonnet-20241022-v2:0": ["us-west-2"],
-    "anthropic.claude-3-haiku-20240307-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
-    "anthropic.claude-3-5-haiku-20241022-v1:0": ["us-west-2"],
-    "meta.llama3-1-70b-instruct-v1:0": ["us-west-2"],
     "mistral.mistral-large-2402-v1:0": ["us-west-2", "us-east-1", "eu-west-1"],
-    "amazon.nova-pro-v1:0": ["us-east-1"]
+    "meta.llama3-1-70b-instruct-v1:0": ["us-west-2", "us-east-1"],
+    "anthropic.claude-3-haiku-20240307-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "anthropic.claude-sonnet-4-20250514-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "anthropic.claude-haiku-4-5-20251001-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "anthropic.claude-sonnet-4-5-20250929-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "anthropic.claude-opus-4-5-20251101-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "amazon.nova-pro-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "amazon.nova-2-lite-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "amazon.nova-micro-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
+    "amazon.nova-premier-v1:0": ["us-west-2", "us-east-1"],
+    "anthropic.claude-3-5-sonnet-20240620-v1:0": ["ap-northeast-1"],
+    "anthropic.claude-3-5-sonnet-20241022-v2:0": ["ap-northeast-1"],
 }
 
 SM_RECIPE = "recipe"

sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py

@@ -110,7 +110,7 @@ class LLMAsJudgeEvaluator(BaseEvaluator):
             # Both formats work - with or without 'Builtin.' prefix
             evaluator = LLMAsJudgeEvaluator(
                 base_model="llama-3-3-70b-instruct",
-                evaluator_model="anthropic.claude-3-5-sonnet-20240620-v1:0",
+                evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0",
                 dataset="s3://my-bucket/my-dataset.jsonl",
                 builtin_metrics=["Correctness", "Helpfulness"],  # Prefix optional
                 mlflow_resource_arn="arn:aws:sagemaker:us-west-2:123456789012:mlflow-tracking-server/my-server",
@@ -144,7 +144,7 @@ class LLMAsJudgeEvaluator(BaseEvaluator):
             # Example evaluating only custom model (skip base model)
             evaluator = LLMAsJudgeEvaluator(
                 base_model="llama-3-3-70b-instruct",
-                evaluator_model="anthropic.claude-3-5-sonnet-20240620-v1:0",
+                evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0",
                 dataset="s3://my-bucket/my-dataset.jsonl",
                 builtin_metrics=["Correctness"],  # Prefix optional
                 evaluate_base_model=False,
@@ -776,7 +776,7 @@ def evaluate(self):
             
                 evaluator = LLMAsJudgeEvaluator(
                     base_model="llama-3-3-70b-instruct",
-                    evaluator_model="anthropic.claude-3-5-sonnet-20240620-v1:0",
+                    evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0",
                     dataset="s3://my-bucket/my-dataset.jsonl",
                     builtin_metrics=["Correctness", "Helpfulness"],
                     s3_output_path="s3://my-bucket/output"

sagemaker-train/tests/integ/train/test_llm_as_judge_base_model_fix.py

@@ -70,7 +70,7 @@
 ACCOUNT_ID = "729646638167"
 
 TEST_CONFIG = {
-    "evaluator_model": "anthropic.claude-3-5-haiku-20241022-v1:0",
+    "evaluator_model": "anthropic.claude-sonnet-4-20250514-v1:0",
     "dataset_s3_uri": f"s3://sagemaker-{REGION}-{ACCOUNT_ID}/model-customization/eval/gen_qa.jsonl",
     "builtin_metrics": ["Completeness", "Faithfulness"],
     "custom_metrics_json": json.dumps([CUSTOM_METRIC_DICT]),

sagemaker-train/tests/integ/train/test_llm_as_judge_evaluator.py

@@ -72,7 +72,7 @@
 
 TEST_CONFIG = {
     "model_package_arn": "arn:aws:sagemaker:us-west-2:729646638167:model-package/sdk-test-finetuned-models/1",
-    "evaluator_model": "anthropic.claude-3-5-haiku-20241022-v1:0",
+    "evaluator_model": "anthropic.claude-sonnet-4-20250514-v1:0",
     "dataset_s3_uri": "s3://sagemaker-us-west-2-729646638167/model-customization/eval/gen_qa.jsonl",
     "builtin_metrics": ["Completeness", "Faithfulness"],
     "custom_metrics_json": json.dumps([CUSTOM_METRIC_DICT]),

sagemaker-train/tests/unit/train/evaluate/test_llm_as_judge_evaluator.py

@@ -31,7 +31,7 @@
 DEFAULT_MODEL_PACKAGE_GROUP_ARN = "arn:aws:sagemaker:us-west-2:123456789012:model-package-group/test-group"
 DEFAULT_BASE_MODEL_ARN = "arn:aws:sagemaker:us-west-2:aws:hub-content/SageMakerPublicHub/Model/llama3-2-1b-instruct/1.0.0"
 DEFAULT_ARTIFACT_ARN = "arn:aws:sagemaker:us-west-2:123456789012:artifact/test-artifact"
-DEFAULT_EVALUATOR_MODEL = "anthropic.claude-3-5-sonnet-20240620-v1:0"
+DEFAULT_EVALUATOR_MODEL = "anthropic.claude-sonnet-4-20250514-v1:0"
 
 
 @patch('sagemaker.train.common_utils.model_resolution._resolve_base_model')
@@ -847,13 +847,21 @@ def test_llm_as_judge_evaluator_with_mlflow_names(mock_artifact, mock_resolve):
 @patch('sagemaker.core.resources.Artifact')
 def test_llm_as_judge_evaluator_valid_evaluator_models(mock_artifact, mock_resolve):
     """Test LLMAsJudgeEvaluator with valid evaluator models."""
+    # Models available in us-west-2 per the regional allowlist. The Claude 3.5
+    # Sonnet v1/v2 models are intentionally omitted here — they are only allowed
+    # in ap-northeast-1 (see test_llm_as_judge_evaluator_region_restriction).
     valid_models = [
-        "anthropic.claude-3-5-sonnet-20240620-v1:0",
-        "anthropic.claude-3-5-sonnet-20241022-v2:0",
         "anthropic.claude-3-haiku-20240307-v1:0",
-        "anthropic.claude-3-5-haiku-20241022-v1:0",
+        "anthropic.claude-sonnet-4-20250514-v1:0",
+        "anthropic.claude-haiku-4-5-20251001-v1:0",
+        "anthropic.claude-sonnet-4-5-20250929-v1:0",
+        "anthropic.claude-opus-4-5-20251101-v1:0",
         "meta.llama3-1-70b-instruct-v1:0",
         "mistral.mistral-large-2402-v1:0",
+        "amazon.nova-pro-v1:0",
+        "amazon.nova-2-lite-v1:0",
+        "amazon.nova-micro-v1:0",
+        "amazon.nova-premier-v1:0",
     ]
 
     mock_info = Mock()