6161 "region" : "us-west-2" ,
6262}
6363
64- # Nova model evaluation configuration (uses our own test account in us-east-1)
64+ # Nova model evaluation configuration (uses dedicated test account in us-east-1)
6565NOVA_CONFIG = {
66- "model_package_arn" : "arn:aws:sagemaker:us-east-1:729646638167:model-package/sdk-test-finetuned-models/65" ,
67- "dataset_s3_uri" : "s3://sagemaker-us-east-1-729646638167/model-customization/eval/zc_test.jsonl" ,
68- "s3_output_path" : "s3://sagemaker-us-east-1-729646638167/model-customization/eval/" ,
69- "model_package_group_arn" : "arn:aws:sagemaker:us-east-1:729646638167:model-package-group/sdk-test-finetuned-models" ,
66+ "dataset_s3_uri" : "s3://sagemaker-us-east-1-784379639078/model-customization/eval/zc_test.jsonl" ,
67+ "s3_output_path" : "s3://sagemaker-us-east-1-784379639078/model-customization/eval/" ,
68+ "model_package_group_arn" : "arn:aws:sagemaker:us-east-1:784379639078:model-package-group/sdk-test-finetuned-models" ,
7069 "region" : "us-east-1" ,
7170}
7271
@@ -339,25 +338,44 @@ def test_benchmark_evaluation_base_model_only(self):
339338 assert execution .status .overall_status == "Succeeded"
340339 logger .info ("Base model only evaluation completed successfully" )
341340
342- @pytest .mark .skip (reason = "Pending us-east-1 test infrastructure migration to dedicated test account" )
341+ @pytest .mark .gpu_intensive
342+ @pytest .mark .us_east_1
343343 def test_benchmark_evaluation_nova_model (self ):
344344 """
345345 Test benchmark evaluation with Nova model.
346346
347347 This test uses a Nova fine-tuned model package in us-east-1 region.
348348 Configuration from commented section in benchmark_demo.ipynb.
349349
350- Note: This test is currently skipped pending us-east-1 test infra migration.
350+ Note: This test requires a model package to exist in the model package group.
351+ It should be run after a successful SFT or RLVR training job has produced one.
351352 """
353+ import boto3
354+
352355 # Get benchmarks
353356 Benchmark = get_benchmarks ()
354357
358+ # Dynamically find the latest model package in the group
359+ sm_client = boto3 .client ("sagemaker" , region_name = NOVA_CONFIG ["region" ])
360+ packages = sm_client .list_model_packages (
361+ ModelPackageGroupName = "sdk-test-finetuned-models" ,
362+ SortBy = "CreationTime" ,
363+ SortOrder = "Descending" ,
364+ MaxResults = 1 ,
365+ )
366+
367+ if not packages ["ModelPackageSummaryList" ]:
368+ pytest .skip ("No model packages available in sdk-test-finetuned-models group. Run SFT/RLVR training first." )
369+
370+ model_package_arn = packages ["ModelPackageSummaryList" ][0 ]["ModelPackageArn" ]
371+ logger .info (f"Using model package: { model_package_arn } " )
372+
355373 logger .info ("Creating BenchmarkEvaluator with Nova model" )
356374
357375 # Create evaluator with Nova model package
358376 evaluator = BenchMarkEvaluator (
359377 benchmark = Benchmark .MMLU ,
360- model = NOVA_CONFIG [ " model_package_arn" ] ,
378+ model = model_package_arn ,
361379 s3_output_path = NOVA_CONFIG ["s3_output_path" ],
362380 model_package_group = NOVA_CONFIG ["model_package_group_arn" ],
363381 base_eval_name = "integ-test-nova-eval" ,
@@ -367,7 +385,7 @@ def test_benchmark_evaluation_nova_model(self):
367385 # Verify evaluator was created
368386 assert evaluator is not None
369387 assert evaluator .benchmark == Benchmark .MMLU
370- assert evaluator .model == NOVA_CONFIG [ " model_package_arn" ]
388+ assert evaluator .model == model_package_arn
371389 assert evaluator .region == NOVA_CONFIG ["region" ]
372390
373391 logger .info (f"Created evaluator: { evaluator .base_eval_name } " )
@@ -397,8 +415,8 @@ def test_benchmark_evaluation_nova_model(self):
397415 logger .info (f"Status after refresh: { execution .status .overall_status } " )
398416
399417 # Wait for completion
400- logger .info ("Waiting for evaluation to complete (timeout: 1 hour )" )
401- execution .wait (target_status = "Succeeded" , poll = 30 , timeout = 3600 )
418+ logger .info ("Waiting for evaluation to complete (timeout: 3 hours )" )
419+ execution .wait (target_status = "Succeeded" , poll = 30 , timeout = 10800 )
402420
403421 # Verify completion
404422 assert execution .status .overall_status == "Succeeded"
0 commit comments