Skip to content

Commit ed104fd

Browse files
author
Roja Reddy Sareddy
committed
Fix:MTRL eval override hyperparams issue
1 parent 1534e60 commit ed104fd

2 files changed

Lines changed: 6 additions & 0 deletions

File tree

‎sagemaker-train/tests/integ/train/test_mtrl_evaluator_3p_agent.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ def test_evaluate_base_model_with_lambda_agent(self, lambda_agent_arn):
262262
logger.info(f"Started 3P agent base model evaluation: {execution.arn}")
263263
logger.info(f"Status: {execution.status.overall_status}")
264264

265+
@pytest.mark.skip(reason="Quota limited (1 concurrent eval job) - run manually")
265266
def test_evaluate_base_model_with_agent_lambda_object(self, lambda_agent_arn):
266267
"""Test evaluating using an CustomAgentLambda object as agent_config.
267268
@@ -287,6 +288,7 @@ def test_evaluate_base_model_with_agent_lambda_object(self, lambda_agent_arn):
287288
assert execution.arn is not None
288289
logger.info(f"Started CustomAgentLambda object evaluation: {execution.arn}")
289290

291+
@pytest.mark.skip(reason="Quota limited (1 concurrent eval job) - run manually")
290292
def test_evaluate_with_lambda_agent_wait_for_completion(self, lambda_agent_arn):
291293
"""Test full end-to-end: start evaluation and wait for completion.
292294
@@ -316,6 +318,7 @@ def test_evaluate_with_lambda_agent_wait_for_completion(self, lambda_agent_arn):
316318
if execution.status.overall_status == "Failed":
317319
logger.error(f"Failure reason: {execution.status.failure_reason}")
318320

321+
@pytest.mark.skip(reason="Quota limited (1 concurrent eval job) - run manually")
319322
def test_evaluate_lambda_agent_discoverable_via_get_all(self, lambda_agent_arn):
320323
"""Test that 3P agent evaluations are discoverable via get_all.
321324
@@ -355,6 +358,7 @@ def test_evaluate_lambda_agent_discoverable_via_get_all(self, lambda_agent_arn):
355358

356359

357360

361+
@pytest.mark.skip(reason="Quota limited (1 concurrent eval job) - run manually")
358362
def test_evaluate_with_attached_trainer(self, lambda_agent_arn):
359363
"""Test evaluating a fine-tuned model by attaching to an existing training job."""
360364
from sagemaker.train.multi_turn_rl_trainer import MultiTurnRLTrainer

‎sagemaker-train/tests/integ/train/test_mtrl_trainer_integration.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ def test_evaluate_finetuned_model(self, attached_trainer, config):
187187
f"reason: {execution.status.failure_reason}"
188188
)
189189

190+
@pytest.mark.skip(reason="Quota limited (1 concurrent eval job) - run manually")
190191
def test_evaluate_base_model(self, config):
191192
"""Evaluate the base model only — submit and wait for completion."""
192193
evaluator = MultiTurnRLEvaluator(
@@ -248,6 +249,7 @@ def test_evaluate_comparison(self, attached_trainer, config):
248249
f"reason: {execution.status.failure_reason}"
249250
)
250251

252+
@pytest.mark.skip(reason="Quota limited (1 concurrent eval job) - run manually")
251253
def test_evaluate_with_hyperparam_override(self, attached_trainer, config):
252254
"""Test that hyperparameter overrides are passed through to the eval job."""
253255
evaluator = MultiTurnRLEvaluator(

0 commit comments

Comments
 (0)