Merge pull request #3186 from bghira/bugfix/3184

Fix adapter-only validation adapter runs

Author: bghira

simpletuner/helpers/training/validation_adapters.py

@@ -235,20 +235,19 @@ def _make_run(label: str | None, specs: Sequence[ValidationAdapterSpec]) -> Vali
             is_base=False,
         )
 
-    if adapter_path and mode != "none":
+    if mode != "none" and adapter_path:
         specs = [_build_adapter_spec(adapter_path, adapter_strength, adapter_name)]
         preferred_label = adapter_name or _stem_from_path(adapter_path)
         runs.append(_make_run(preferred_label, specs))
 
-    for entry in _iter_config_entries(adapter_config):
-        label, specs = _normalize_run_entry(entry)
-        if not specs:
-            continue
-        runs.append(_make_run(label, specs))
+    if mode != "none":
+        for entry in _iter_config_entries(adapter_config):
+            label, specs = _normalize_run_entry(entry)
+            if not specs:
+                continue
+            runs.append(_make_run(label, specs))
 
-    include_base = True
-    if adapter_path and mode == "adapter_only" and adapter_config in (None, [], {}):
-        include_base = False
+    include_base = mode == "comparison" or not runs
 
     ordered_runs: List[ValidationAdapterRun] = []
     if include_base:

tests/test_validation_adapters.py

@@ -48,22 +48,20 @@ def test_config_supports_multiple_runs(self):
             },
         ]
         runs = build_validation_adapter_runs(None, config)
-        self.assertEqual(3, len(runs))
+        self.assertEqual(2, len(runs))
 
         first = runs[0]
-        self.assertTrue(first.is_base)
+        self.assertFalse(first.is_base)
+        self.assertEqual("base_adapter", first.slug)
+        self.assertEqual("org/base_adapter", first.adapters[0].repo_id)
 
         second = runs[1]
-        self.assertEqual("base_adapter", second.slug)
-        self.assertEqual("org/base_adapter", second.adapters[0].repo_id)
-
-        third = runs[2]
-        self.assertEqual("combo", third.label)
-        self.assertEqual(2, len(third.adapters))
-        strengths = [adapter.strength for adapter in third.adapters]
+        self.assertEqual("combo", second.label)
+        self.assertEqual(2, len(second.adapters))
+        strengths = [adapter.strength for adapter in second.adapters]
         self.assertEqual([0.5, 1.0], strengths)
-        self.assertEqual("repo/style", third.adapters[1].repo_id)
-        self.assertEqual("style.safetensors", third.adapters[1].weight_name)
+        self.assertEqual("repo/style", second.adapters[1].repo_id)
+        self.assertEqual("style.safetensors", second.adapters[1].weight_name)
 
     def test_adapter_mode_comparison_includes_base(self):
         runs = build_validation_adapter_runs(
@@ -76,11 +74,40 @@ def test_adapter_mode_comparison_includes_base(self):
         self.assertEqual("sample", second.adapters[0].adapter_name)
         self.assertEqual(0.8, second.adapters[0].strength)
 
+    def test_adapter_mode_comparison_with_config_includes_base(self):
+        runs = build_validation_adapter_runs(
+            None,
+            [{"label": "custom", "path": "repo/hero"}],
+            adapter_mode="comparison",
+        )
+        self.assertEqual(2, len(runs))
+        self.assertTrue(runs[0].is_base)
+        self.assertEqual("custom", runs[1].slug)
+
+    def test_adapter_mode_adapter_only_with_config_excludes_base(self):
+        runs = build_validation_adapter_runs(
+            None,
+            [{"label": "custom", "path": "repo/hero"}],
+            adapter_mode="adapter_only",
+        )
+        self.assertEqual(1, len(runs))
+        self.assertFalse(runs[0].is_base)
+        self.assertEqual("custom", runs[0].slug)
+
     def test_adapter_mode_none_skips_loading(self):
         runs = build_validation_adapter_runs("foo/bar", None, adapter_mode="none")
         self.assertEqual(1, len(runs))
         self.assertTrue(runs[0].is_base)
 
+    def test_adapter_mode_none_skips_config_loading(self):
+        runs = build_validation_adapter_runs(
+            None,
+            [{"label": "custom", "path": "repo/hero"}],
+            adapter_mode="none",
+        )
+        self.assertEqual(1, len(runs))
+        self.assertTrue(runs[0].is_base)
+
     def test_config_entry_with_strength_and_name(self):
         config = [
             {
@@ -91,8 +118,8 @@ def test_config_entry_with_strength_and_name(self):
             }
         ]
         runs = build_validation_adapter_runs(None, config)
-        self.assertEqual(2, len(runs))
-        run = runs[1]
+        self.assertEqual(1, len(runs))
+        run = runs[0]
         self.assertEqual("custom", run.label)
         adapter = run.adapters[0]
         self.assertEqual("hero_adapter", adapter.adapter_name)
@@ -111,7 +138,7 @@ def test_config_entry_supports_run_level_target_stage(self):
         ]
         runs = build_validation_adapter_runs(None, config)
 
-        run = runs[1]
+        run = runs[0]
         self.assertEqual("repo/detail", run.adapters[0].repo_id)
         self.assertEqual("two", run.adapters[0].target_stage)
         self.assertEqual("one", run.adapters[1].target_stage)
@@ -246,7 +273,7 @@ def test_targeted_adapter_loads_only_for_matching_stage(self):
         run = build_validation_adapter_runs(
             None,
             [{"label": "refiner", "path": "repo/refiner", "target_stage": "two", "strength": 0.4}],
-        )[1]
+        )[0]
 
         with validator._temporary_validation_adapters(run):
             self.assertEqual([], base_pipeline.load_calls)
@@ -266,7 +293,7 @@ def test_wan_low_stage_uses_transformer_2_component(self):
         run = build_validation_adapter_runs(
             None,
             [{"label": "low-noise", "path": "repo/low", "target_stage": "low", "strength": 0.6}],
-        )[1]
+        )[0]
 
         with validator._temporary_validation_adapters(run):
             with validator._temporary_validation_stage_adapters(pipeline, ("high", "low")):
@@ -291,7 +318,7 @@ def test_stage_adapter_preserves_global_adapter_on_component_target(self):
                     ],
                 }
             ],
-        )[1]
+        )[0]
 
         with validator._temporary_validation_adapters(run):
             self.assertEqual(("combo", 0.25), pipeline.set_calls[0])
@@ -314,7 +341,7 @@ def test_validation_adapter_restore_ignores_loaded_temporary_adapter(self):
         run = build_validation_adapter_runs(
             None,
             [{"label": "Krea2 Turbo adapter", "path": "repo/krea2", "adapter_name": "krea2_turbo_adapter"}],
-        )[1]
+        )[0]
 
         with validator._temporary_validation_adapters(run):
             self.assertEqual("repo/krea2", pipeline.load_calls[0][0])
@@ -332,7 +359,7 @@ def test_validation_adapter_restore_preserves_existing_active_adapter(self):
         run = build_validation_adapter_runs(
             None,
             [{"label": "Krea2 Turbo adapter", "path": "repo/krea2", "adapter_name": "krea2_turbo_adapter"}],
-        )[1]
+        )[0]
 
         with validator._temporary_validation_adapters(run):
             self.assertEqual(