Skip to content

Commit 3394c7d

Browse files
authored
fix: drop claude-sonnet-4-20250514 from evaluator allowlist (#6009)
Remove anthropic.claude-sonnet-4-20250514-v1:0 from the LLM-as-Judge evaluator allowlist and re-point the docstring examples and unit-test default judge model to anthropic.claude-sonnet-4-5-20250929-v1:0.
1 parent 943ce0a commit 3394c7d

3 files changed

Lines changed: 4 additions & 6 deletions

File tree

‎sagemaker-train/src/sagemaker/train/constants.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ def get_sagemaker_hub_name() -> str:
6767
"mistral.mistral-large-2402-v1:0": ["us-west-2", "us-east-1", "eu-west-1"],
6868
"meta.llama3-1-70b-instruct-v1:0": ["us-west-2", "us-east-1"],
6969
"anthropic.claude-3-haiku-20240307-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
70-
"anthropic.claude-sonnet-4-20250514-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
7170
"anthropic.claude-haiku-4-5-20251001-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
7271
"anthropic.claude-sonnet-4-5-20250929-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],
7372
"anthropic.claude-opus-4-5-20251101-v1:0": ["us-west-2", "us-east-1", "ap-northeast-1", "eu-west-1"],

‎sagemaker-train/src/sagemaker/train/evaluate/llm_as_judge_evaluator.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ class LLMAsJudgeEvaluator(BaseEvaluator):
110110
# Both formats work - with or without 'Builtin.' prefix
111111
evaluator = LLMAsJudgeEvaluator(
112112
base_model="llama-3-3-70b-instruct",
113-
evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0",
113+
evaluator_model="anthropic.claude-sonnet-4-5-20250929-v1:0",
114114
dataset="s3://my-bucket/my-dataset.jsonl",
115115
builtin_metrics=["Correctness", "Helpfulness"], # Prefix optional
116116
mlflow_resource_arn="arn:aws:sagemaker:us-west-2:123456789012:mlflow-tracking-server/my-server",
@@ -144,7 +144,7 @@ class LLMAsJudgeEvaluator(BaseEvaluator):
144144
# Example evaluating only custom model (skip base model)
145145
evaluator = LLMAsJudgeEvaluator(
146146
base_model="llama-3-3-70b-instruct",
147-
evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0",
147+
evaluator_model="anthropic.claude-sonnet-4-5-20250929-v1:0",
148148
dataset="s3://my-bucket/my-dataset.jsonl",
149149
builtin_metrics=["Correctness"], # Prefix optional
150150
evaluate_base_model=False,
@@ -776,7 +776,7 @@ def evaluate(self):
776776
777777
evaluator = LLMAsJudgeEvaluator(
778778
base_model="llama-3-3-70b-instruct",
779-
evaluator_model="anthropic.claude-sonnet-4-20250514-v1:0",
779+
evaluator_model="anthropic.claude-sonnet-4-5-20250929-v1:0",
780780
dataset="s3://my-bucket/my-dataset.jsonl",
781781
builtin_metrics=["Correctness", "Helpfulness"],
782782
s3_output_path="s3://my-bucket/output"

‎sagemaker-train/tests/unit/train/evaluate/test_llm_as_judge_evaluator.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
DEFAULT_MODEL_PACKAGE_GROUP_ARN = "arn:aws:sagemaker:us-west-2:123456789012:model-package-group/test-group"
3232
DEFAULT_BASE_MODEL_ARN = "arn:aws:sagemaker:us-west-2:aws:hub-content/SageMakerPublicHub/Model/llama3-2-1b-instruct/1.0.0"
3333
DEFAULT_ARTIFACT_ARN = "arn:aws:sagemaker:us-west-2:123456789012:artifact/test-artifact"
34-
DEFAULT_EVALUATOR_MODEL = "anthropic.claude-sonnet-4-20250514-v1:0"
34+
DEFAULT_EVALUATOR_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0"
3535

3636

3737
@patch('sagemaker.train.common_utils.model_resolution._resolve_base_model')
@@ -851,7 +851,6 @@ def test_llm_as_judge_evaluator_valid_evaluator_models(mock_artifact, mock_resol
851851
# (covered by test_llm_as_judge_evaluator_region_restriction).
852852
valid_models = [
853853
"anthropic.claude-3-haiku-20240307-v1:0",
854-
"anthropic.claude-sonnet-4-20250514-v1:0",
855854
"anthropic.claude-haiku-4-5-20251001-v1:0",
856855
"anthropic.claude-sonnet-4-5-20250929-v1:0",
857856
"anthropic.claude-opus-4-5-20251101-v1:0",

0 commit comments

Comments
 (0)