Skip to content

Commit 24e6ef0

Browse files
authored
test: unskip nova integ tests (#5901)
* test: unskip nova tests * test: extend max_wait_time * extend timeout for test_benchmark_evaluation_nova_model
1 parent 3d83bbc commit 24e6ef0

5 files changed

Lines changed: 48 additions & 25 deletions

File tree

‎sagemaker-train/tests/integ/ai_registry/test_dataset.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@ def test_create_dataset_from_s3_oss_dpo(self, unique_name, test_bucket, cleanup_
7979
assert dataset.name == unique_name
8080
assert dataset.customization_technique == CustomizationTechnique.DPO
8181

82+
@pytest.mark.us_east_1
8283
def test_create_dataset_from_s3_nova_sft(self, unique_name, test_bucket, cleanup_list):
83-
"""Test creating RLVR dataset from S3 URI."""
84+
"""Test creating Nova SFT dataset from S3 URI."""
8485
s3_uri = f"s3://{test_bucket}/test_datasets/Nova/nova_sft_train.jsonl"
8586
dataset = DataSet.create(
8687
name=unique_name,
@@ -92,8 +93,9 @@ def test_create_dataset_from_s3_nova_sft(self, unique_name, test_bucket, cleanup
9293
assert dataset.name == unique_name
9394
assert dataset.customization_technique == CustomizationTechnique.SFT
9495

96+
@pytest.mark.us_east_1
9597
def test_create_dataset_from_s3_nova_dpo(self, unique_name, test_bucket, cleanup_list):
96-
"""Test creating RLVR dataset from S3 URI."""
98+
"""Test creating Nova DPO dataset from S3 URI."""
9799
s3_uri = f"s3://{test_bucket}/test_datasets/Nova/nova_dpo_train.jsonl"
98100
dataset = DataSet.create(
99101
name=unique_name,
@@ -105,8 +107,9 @@ def test_create_dataset_from_s3_nova_dpo(self, unique_name, test_bucket, cleanup
105107
assert dataset.name == unique_name
106108
assert dataset.customization_technique == CustomizationTechnique.DPO
107109

110+
@pytest.mark.us_east_1
108111
def test_create_dataset_from_s3_nova_rft(self, unique_name, test_bucket, cleanup_list):
109-
"""Test creating RLVR dataset from S3 URI."""
112+
"""Test creating Nova RFT dataset from S3 URI."""
110113
s3_uri = f"s3://{test_bucket}/test_datasets/Nova/nova_rft_train.jsonl"
111114
dataset = DataSet.create(
112115
name=unique_name,
@@ -118,8 +121,9 @@ def test_create_dataset_from_s3_nova_rft(self, unique_name, test_bucket, cleanup
118121
assert dataset.name == unique_name
119122
assert dataset.customization_technique == CustomizationTechnique.RLVR
120123

124+
@pytest.mark.us_east_1
121125
def test_create_dataset_from_s3_nova_eval(self, unique_name, test_bucket, cleanup_list):
122-
"""Test creating RLVR dataset from S3 URI."""
126+
"""Test creating Nova eval dataset from S3 URI."""
123127
s3_uri = f"s3://{test_bucket}/test_datasets/Nova/nova_eval.jsonl"
124128
dataset = DataSet.create(
125129
name=unique_name,

‎sagemaker-train/tests/integ/train/test_benchmark_evaluator.py‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,11 @@
6161
"region": "us-west-2",
6262
}
6363

64-
# Nova model evaluation configuration (uses our own test account in us-east-1)
64+
# Nova model evaluation configuration (uses dedicated test account in us-east-1)
6565
NOVA_CONFIG = {
66-
"model_package_arn": "arn:aws:sagemaker:us-east-1:729646638167:model-package/sdk-test-finetuned-models/65",
67-
"dataset_s3_uri": "s3://sagemaker-us-east-1-729646638167/model-customization/eval/zc_test.jsonl",
68-
"s3_output_path": "s3://sagemaker-us-east-1-729646638167/model-customization/eval/",
69-
"model_package_group_arn": "arn:aws:sagemaker:us-east-1:729646638167:model-package-group/sdk-test-finetuned-models",
66+
"dataset_s3_uri": "s3://sagemaker-us-east-1-784379639078/model-customization/eval/zc_test.jsonl",
67+
"s3_output_path": "s3://sagemaker-us-east-1-784379639078/model-customization/eval/",
68+
"model_package_group_arn": "arn:aws:sagemaker:us-east-1:784379639078:model-package-group/sdk-test-finetuned-models",
7069
"region": "us-east-1",
7170
}
7271

@@ -339,25 +338,44 @@ def test_benchmark_evaluation_base_model_only(self):
339338
assert execution.status.overall_status == "Succeeded"
340339
logger.info("Base model only evaluation completed successfully")
341340

342-
@pytest.mark.skip(reason="Pending us-east-1 test infrastructure migration to dedicated test account")
341+
@pytest.mark.gpu_intensive
342+
@pytest.mark.us_east_1
343343
def test_benchmark_evaluation_nova_model(self):
344344
"""
345345
Test benchmark evaluation with Nova model.
346346
347347
This test uses a Nova fine-tuned model package in us-east-1 region.
348348
Configuration from commented section in benchmark_demo.ipynb.
349349
350-
Note: This test is currently skipped pending us-east-1 test infra migration.
350+
Note: This test requires a model package to exist in the model package group.
351+
It should be run after a successful SFT or RLVR training job has produced one.
351352
"""
353+
import boto3
354+
352355
# Get benchmarks
353356
Benchmark = get_benchmarks()
354357

358+
# Dynamically find the latest model package in the group
359+
sm_client = boto3.client("sagemaker", region_name=NOVA_CONFIG["region"])
360+
packages = sm_client.list_model_packages(
361+
ModelPackageGroupName="sdk-test-finetuned-models",
362+
SortBy="CreationTime",
363+
SortOrder="Descending",
364+
MaxResults=1,
365+
)
366+
367+
if not packages["ModelPackageSummaryList"]:
368+
pytest.skip("No model packages available in sdk-test-finetuned-models group. Run SFT/RLVR training first.")
369+
370+
model_package_arn = packages["ModelPackageSummaryList"][0]["ModelPackageArn"]
371+
logger.info(f"Using model package: {model_package_arn}")
372+
355373
logger.info("Creating BenchmarkEvaluator with Nova model")
356374

357375
# Create evaluator with Nova model package
358376
evaluator = BenchMarkEvaluator(
359377
benchmark=Benchmark.MMLU,
360-
model=NOVA_CONFIG["model_package_arn"],
378+
model=model_package_arn,
361379
s3_output_path=NOVA_CONFIG["s3_output_path"],
362380
model_package_group=NOVA_CONFIG["model_package_group_arn"],
363381
base_eval_name="integ-test-nova-eval",
@@ -367,7 +385,7 @@ def test_benchmark_evaluation_nova_model(self):
367385
# Verify evaluator was created
368386
assert evaluator is not None
369387
assert evaluator.benchmark == Benchmark.MMLU
370-
assert evaluator.model == NOVA_CONFIG["model_package_arn"]
388+
assert evaluator.model == model_package_arn
371389
assert evaluator.region == NOVA_CONFIG["region"]
372390

373391
logger.info(f"Created evaluator: {evaluator.base_eval_name}")
@@ -397,8 +415,8 @@ def test_benchmark_evaluation_nova_model(self):
397415
logger.info(f"Status after refresh: {execution.status.overall_status}")
398416

399417
# Wait for completion
400-
logger.info("Waiting for evaluation to complete (timeout: 1 hour)")
401-
execution.wait(target_status="Succeeded", poll=30, timeout=3600)
418+
logger.info("Waiting for evaluation to complete (timeout: 3 hours)")
419+
execution.wait(target_status="Succeeded", poll=30, timeout=10800)
402420

403421
# Verify completion
404422
assert execution.status.overall_status == "Succeeded"

‎sagemaker-train/tests/integ/train/test_rlvr_trainer_integration.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ def test_rlvr_trainer_with_custom_reward_function(sagemaker_session):
102102

103103

104104
@pytest.mark.gpu_intensive
105-
@pytest.mark.skip(reason="TODO: Nova test to be enabled in us-east-1")
105+
@pytest.mark.us_east_1
106106
def test_rlvr_trainer_nova_workflow(sagemaker_session_us_east_1):
107107
"""Test RLVR training workflow with Nova model."""
108108
# sagemaker_session_us_east_1 fixture is defined in conftest.py (us-east-1 region)
@@ -113,18 +113,18 @@ def test_rlvr_trainer_nova_workflow(sagemaker_session_us_east_1):
113113
model_package_group="sdk-test-finetuned-models",
114114
mlflow_experiment_name="test-nova-rlvr-finetuned-models-exp",
115115
mlflow_run_name="test-nova-rlvr-finetuned-models-run",
116-
training_dataset="s3://mc-flows-sdk-testing-us-east-1/input_data/rlvr-nova/grpo-64-sample.jsonl",
117-
validation_dataset="s3://mc-flows-sdk-testing-us-east-1/input_data/rlvr-nova/grpo-64-sample.jsonl",
118-
s3_output_path="s3://mc-flows-sdk-testing-us-east-1/output/",
119-
custom_reward_function="arn:aws:sagemaker:us-east-1:729646638167:hub-content/sdktest/JsonDoc/rlvr-nova-test-rf/0.0.1",
116+
training_dataset="s3://sagemaker-us-east-1-784379639078/input_data/rlvr-nova/grpo-64-sample.jsonl",
117+
validation_dataset="s3://sagemaker-us-east-1-784379639078/input_data/rlvr-nova/grpo-64-sample.jsonl",
118+
s3_output_path="s3://sagemaker-us-east-1-784379639078/output/",
119+
custom_reward_function="arn:aws:sagemaker:us-east-1:784379639078:hub-content/sdktest/JsonDoc/rlvr-nova-test-rf/0.0.1",
120120
accept_eula=True,
121121
sagemaker_session=sagemaker_session_us_east_1,
122122
base_job_name=f"rlvr-nova-integ-{unique_id}",
123123
)
124124
training_job = rlvr_trainer.train(wait=False)
125125

126126
# Manual wait loop
127-
max_wait_time = 3600
127+
max_wait_time = 10800 # 3 hour timeout (Nova training takes >1 hour)
128128
poll_interval = 30
129129
start_time = time.time()
130130

‎sagemaker-train/tests/integ/train/test_sft_trainer_integration.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ def test_sft_trainer_with_validation_dataset(sagemaker_session):
9696

9797

9898
@pytest.mark.gpu_intensive
99-
@pytest.mark.skip(reason="TODO: Nova test to be enabled in us-east-1")
99+
@pytest.mark.us_east_1
100100
def test_sft_trainer_nova_workflow(sagemaker_session_us_east_1):
101101
"""Test SFT trainer with Nova model."""
102102
# sagemaker_session_us_east_1 fixture is defined in conftest.py (us-east-1 region)
@@ -108,8 +108,8 @@ def test_sft_trainer_nova_workflow(sagemaker_session_us_east_1):
108108
model_package_group="sdk-test-finetuned-models",
109109
mlflow_experiment_name="test-nova-finetuned-models-exp",
110110
mlflow_run_name="test-nova-finetuned-models-run",
111-
training_dataset="s3://mc-flows-sdk-testing-us-east-1/input_data/sft-nova/sft_200_samples.jsonl",
112-
s3_output_path="s3://mc-flows-sdk-testing-us-east-1/output/",
111+
training_dataset="s3://sagemaker-us-east-1-784379639078/input_data/sft-nova/sft_200_samples.jsonl",
112+
s3_output_path="s3://sagemaker-us-east-1-784379639078/output/",
113113
sagemaker_session=sagemaker_session_us_east_1,
114114
base_job_name=f"sft-nova-integ-{unique_id}",
115115
)
@@ -118,7 +118,7 @@ def test_sft_trainer_nova_workflow(sagemaker_session_us_east_1):
118118
training_job = sft_trainer_nova.train(wait=False)
119119

120120
# Manual wait loop
121-
max_wait_time = 3600 # 1 hour timeout
121+
max_wait_time = 10800 # 3 hour timeout (Nova training takes >1 hour)
122122
poll_interval = 30 # Check every 30 seconds
123123
start_time = time.time()
124124

‎sagemaker-train/tox.ini‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ markers =
6363
release
6464
image_uris_unit_test
6565
gpu_intensive: mark a test as GPU resource intensive (runs on scheduled CI, not PR checks).
66+
us_east_1: mark a test that requires us-east-1 test account credentials (784379639078).
6667
timeout: mark a test as a timeout.
6768
serial: marks tests that must run serially (not in parallel)
6869

0 commit comments

Comments
 (0)