|
46 | 46 |
|
47 | 47 | # Test configuration |
48 | 48 | REGION = "us-east-1" |
49 | | -CLUSTER_NAME = "riv-rig" |
50 | | -INSTANCE_TYPE = "ml.p5.48xlarge" |
| 49 | +CLUSTER_NAME = "pysdk-hp-integ-tests" |
| 50 | +# Using g6 to pass instance count validation. This recipe actually needs p5.48xlarge |
| 51 | +INSTANCE_TYPE = "ml.g6.48xlarge" |
51 | 52 | NODE_COUNT = 2 |
52 | 53 | MODEL_NAME = "nova-textgeneration-micro" |
53 | 54 | DATA_PREFIX = "test-cpt-data-mixing-integ" |
@@ -124,6 +125,7 @@ def training_resources(sagemaker_session_us_east_1): |
124 | 125 | } |
125 | 126 |
|
126 | 127 |
|
| 128 | +# TODO: Remove dry-run when capacity is available in future |
127 | 129 | @pytest.mark.gpu_intensive |
128 | 130 | @pytest.mark.us_east_1 |
129 | 131 | def test_cpt_trainer_nova_micro_with_data_mixing_hyperpod( |
@@ -162,26 +164,28 @@ def test_cpt_trainer_nova_micro_with_data_mixing_hyperpod( |
162 | 164 | base_job_name=f"hp-datamix-m1-{unique_id}", |
163 | 165 | ) |
164 | 166 |
|
| 167 | + dry_run = True |
165 | 168 | logger.info("Submitting CPT HyperPod training job with data mixing config...") |
166 | 169 | try: |
167 | | - job_name = cpt_trainer.train(wait=False) |
| 170 | + job_name = cpt_trainer.train(wait=False, dry_run=dry_run) |
168 | 171 | except ValueError as e: |
169 | 172 | if "Failed to download" in str(e) or "Forge subscription" in str(e): |
170 | 173 | pytest.skip( |
171 | 174 | "Skipping: account does not have Forge subscription for data mixing recipes." |
172 | 175 | ) |
173 | 176 | raise |
174 | 177 |
|
175 | | - # _train_hyperpod returns the job name as a string |
176 | | - assert job_name is not None |
177 | | - logger.info(f"HyperPod CPT job submitted: {job_name}") |
178 | | - |
179 | | - # Verify the job exists on the cluster via hyperpod get-job |
180 | | - get_job_result = subprocess.run( |
181 | | - ["hyperpod", "get-job", "--job-name", job_name], |
182 | | - capture_output=True, text=True, |
183 | | - ) |
184 | | - assert get_job_result.returncode == 0, ( |
185 | | - f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}" |
186 | | - ) |
187 | | - logger.info(f"Verified job '{job_name}' exists on the cluster.") |
| 178 | + if not dry_run: |
| 179 | + # _train_hyperpod returns the job name as a string |
| 180 | + assert job_name is not None |
| 181 | + logger.info(f"HyperPod CPT job submitted: {job_name}") |
| 182 | + |
| 183 | + # Verify the job exists on the cluster via hyperpod get-job |
| 184 | + get_job_result = subprocess.run( |
| 185 | + ["hyperpod", "get-job", "--job-name", job_name], |
| 186 | + capture_output=True, text=True, |
| 187 | + ) |
| 188 | + assert get_job_result.returncode == 0, ( |
| 189 | + f"hyperpod get-job failed for '{job_name}': {get_job_result.stderr}" |
| 190 | + ) |
| 191 | + logger.info(f"Verified job '{job_name}' exists on the cluster.") |
0 commit comments