|
| 1 | +from copy import deepcopy |
| 2 | +from types import SimpleNamespace |
1 | 3 | from unittest.mock import Mock |
2 | 4 | from unittest.mock import call |
3 | 5 |
|
|
6 | 8 | from tenacity import stop_after_attempt |
7 | 9 | from tenacity import wait_none |
8 | 10 |
|
| 11 | +from understack_workflows.ironic.client import IronicClient |
9 | 12 | from understack_workflows.main import configure_snmpv3_user as cli |
10 | 13 |
|
11 | 14 |
|
12 | | -def test_conflict_retries_the_same_request_without_lifecycle_calls(): |
13 | | - client = Mock() |
14 | | - client.client.node.vendor_passthru.side_effect = [ |
15 | | - exceptions.Conflict("Node is busy"), |
16 | | - {"configured": True}, |
17 | | - ] |
18 | | - # Tenacity adds retry_with at runtime but does not expose it in its types. |
19 | | - configure = cli.configure_snmpv3_user.retry_with( # pyright: ignore[reportFunctionMemberAccess] |
20 | | - wait=wait_none(), stop=stop_after_attempt(2) |
| 15 | +@pytest.fixture(autouse=True) |
| 16 | +def _fast_retries(monkeypatch): |
| 17 | + for name in ( |
| 18 | + "get_idle_node", |
| 19 | + "request_transition", |
| 20 | + "suppress_cleaning", |
| 21 | + "restore_cleaning_policy", |
| 22 | + ): |
| 23 | + policy = getattr(cli, name).retry |
| 24 | + monkeypatch.setattr(policy, "wait", wait_none()) |
| 25 | + monkeypatch.setattr(policy, "stop", stop_after_attempt(3)) |
| 26 | + |
| 27 | + |
| 28 | +@pytest.fixture |
| 29 | +def client(): |
| 30 | + client = Mock(spec=IronicClient) |
| 31 | + node = SimpleNamespace( |
| 32 | + uuid="node-uuid", |
| 33 | + provision_state="manageable", |
| 34 | + target_provision_state=None, |
| 35 | + target_power_state=None, |
| 36 | + reservation=None, |
| 37 | + last_error=None, |
| 38 | + maintenance=False, |
| 39 | + automated_clean=None, |
| 40 | + extra={"existing": "value"}, |
21 | 41 | ) |
| 42 | + client.node = node |
| 43 | + client.get_node.side_effect = lambda _: deepcopy(node) |
| 44 | + client.get_runbook.return_value = SimpleNamespace( |
| 45 | + uuid="runbook-uuid", disable_ramdisk=True, traits=["DELL"] |
| 46 | + ) |
| 47 | + client.get_node_traits.return_value = ["DELL"] |
| 48 | + |
| 49 | + def update(_, patch): |
| 50 | + for operation in patch: |
| 51 | + path = operation["path"] |
| 52 | + if path == "/automated_clean": |
| 53 | + node.automated_clean = operation["value"] |
| 54 | + elif operation["op"] == "remove": |
| 55 | + del node.extra[path.split("/")[-1]] |
| 56 | + else: |
| 57 | + node.extra[path.split("/")[-1]] = operation["value"] |
| 58 | + |
| 59 | + def transition(_, action, **kwargs): |
| 60 | + if action == "provide": |
| 61 | + assert node.automated_clean is False |
| 62 | + node.provision_state = { |
| 63 | + "manage": "manageable", |
| 64 | + "clean": "cleaning", |
| 65 | + "service": "servicing", |
| 66 | + "provide": "cleaning", |
| 67 | + }[action] |
| 68 | + node.target_provision_state = { |
| 69 | + "clean": "manageable", |
| 70 | + "service": "active", |
| 71 | + "provide": "available", |
| 72 | + }.get(action) |
| 73 | + |
| 74 | + def wait(_, expected): |
| 75 | + node.provision_state = expected |
| 76 | + node.target_provision_state = None |
| 77 | + |
| 78 | + client.update_node.side_effect = update |
| 79 | + client.set_node_provision_state.side_effect = transition |
| 80 | + client.wait_for_node_provision_state.side_effect = wait |
| 81 | + return client |
| 82 | + |
| 83 | + |
| 84 | +@pytest.mark.parametrize("original", [None, True, False]) |
| 85 | +def test_available_node_skips_automated_cleaning_and_restores_policy(client, original): |
| 86 | + client.node.provision_state = "available" |
| 87 | + client.node.automated_clean = original |
| 88 | + |
| 89 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 90 | + |
| 91 | + assert client.set_node_provision_state.call_args_list == [ |
| 92 | + call("node-uuid", "manage"), |
| 93 | + call("node-uuid", "clean", runbook="runbook-uuid"), |
| 94 | + call("node-uuid", "provide"), |
| 95 | + ] |
| 96 | + assert client.node.provision_state == "available" |
| 97 | + assert client.node.automated_clean is original |
| 98 | + assert client.node.extra == {"existing": "value"} |
| 99 | + |
22 | 100 |
|
23 | | - configure(client, "node-uuid", 5) |
| 101 | +@pytest.mark.parametrize( |
| 102 | + ("state", "action"), [("manageable", "clean"), ("active", "service")] |
| 103 | +) |
| 104 | +def test_runbook_uses_existing_state_without_changing_cleaning_policy( |
| 105 | + client, state, action |
| 106 | +): |
| 107 | + client.node.provision_state = state |
24 | 108 |
|
25 | | - request = call.client.node.vendor_passthru( |
26 | | - "node-uuid", "configure_snmpv3_user", args={"slot": 5}, http_method="POST" |
| 109 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 110 | + |
| 111 | + client.set_node_provision_state.assert_called_once_with( |
| 112 | + "node-uuid", action, runbook="runbook-uuid" |
27 | 113 | ) |
28 | | - assert client.mock_calls == [request, request] |
| 114 | + client.update_node.assert_not_called() |
| 115 | + assert client.node.provision_state == state |
29 | 116 |
|
30 | 117 |
|
31 | | -@pytest.mark.parametrize("error", [exceptions.BadRequest, exceptions.Forbidden]) |
32 | | -def test_permanent_errors_are_not_retried(error): |
33 | | - client = Mock() |
34 | | - client.client.node.vendor_passthru.side_effect = error("Cannot configure") |
| 118 | +@pytest.mark.parametrize( |
| 119 | + "state", ["inspect failed", "enroll", "rescue", "clean failed"] |
| 120 | +) |
| 121 | +def test_unsupported_states_are_unchanged(client, state): |
| 122 | + client.node.provision_state = state |
| 123 | + with pytest.raises(ValueError, match="Cannot run the SNMP runbook"): |
| 124 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 125 | + client.set_node_provision_state.assert_not_called() |
| 126 | + client.update_node.assert_not_called() |
35 | 127 |
|
36 | | - with pytest.raises(error): |
37 | | - cli.configure_snmpv3_user(client, "node-uuid", 5) |
38 | 128 |
|
39 | | - client.client.node.vendor_passthru.assert_called_once() |
| 129 | +@pytest.mark.parametrize("invalid", ["maintenance", "traits", "ramdisk"]) |
| 130 | +def test_validation_precedes_policy_and_state_changes(client, invalid): |
| 131 | + client.node.provision_state = "available" |
| 132 | + if invalid == "maintenance": |
| 133 | + client.node.maintenance = True |
| 134 | + elif invalid == "traits": |
| 135 | + client.get_node_traits.return_value = [] |
| 136 | + else: |
| 137 | + client.get_runbook.return_value.disable_ramdisk = False |
| 138 | + with pytest.raises( |
| 139 | + ValueError, match=r"maintenance|matching trait|disable the ramdisk" |
| 140 | + ): |
| 141 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 142 | + client.set_node_provision_state.assert_not_called() |
| 143 | + client.update_node.assert_not_called() |
40 | 144 |
|
41 | 145 |
|
42 | | -def test_busy_node_eventually_fails(): |
43 | | - client = Mock() |
44 | | - client.client.node.vendor_passthru.side_effect = exceptions.Conflict("Busy") |
45 | | - # Tenacity adds retry_with at runtime but does not expose it in its types. |
46 | | - configure = cli.configure_snmpv3_user.retry_with( # pyright: ignore[reportFunctionMemberAccess] |
47 | | - wait=wait_none(), stop=stop_after_attempt(2) |
| 146 | +def test_cleaning_failure_restores_policy_without_providing(client): |
| 147 | + client.node.provision_state = "available" |
| 148 | + client.node.automated_clean = True |
| 149 | + wait = client.wait_for_node_provision_state.side_effect |
| 150 | + |
| 151 | + def fail_cleaning(node_uuid, expected): |
| 152 | + if expected == "manageable" and client.node.provision_state == "cleaning": |
| 153 | + client.node.provision_state = "clean failed" |
| 154 | + raise RuntimeError("SNMP failed") |
| 155 | + wait(node_uuid, expected) |
| 156 | + |
| 157 | + client.wait_for_node_provision_state.side_effect = fail_cleaning |
| 158 | + with pytest.raises(RuntimeError, match="SNMP failed"): |
| 159 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 160 | + assert client.node.automated_clean is True |
| 161 | + assert cli.CLEANING_POLICY_KEY not in client.node.extra |
| 162 | + assert client.node.provision_state == "clean failed" |
| 163 | + assert ( |
| 164 | + call("node-uuid", "provide") |
| 165 | + not in client.set_node_provision_state.call_args_list |
48 | 166 | ) |
49 | 167 |
|
50 | | - with pytest.raises(exceptions.Conflict): |
51 | | - configure(client, "node-uuid", 5) |
52 | 168 |
|
53 | | - assert client.client.node.vendor_passthru.call_count == 2 |
| 169 | +@pytest.mark.parametrize("target", ["manageable", "available"]) |
| 170 | +def test_timeout_keeps_suppression_and_saved_policy_until_recovery(client, target): |
| 171 | + client.node.provision_state = "available" |
| 172 | + client.node.automated_clean = True |
| 173 | + wait = client.wait_for_node_provision_state.side_effect |
| 174 | + |
| 175 | + def timeout(node_uuid, expected): |
| 176 | + if client.node.target_provision_state == target: |
| 177 | + raise TimeoutError("Still cleaning") |
| 178 | + wait(node_uuid, expected) |
| 179 | + |
| 180 | + client.wait_for_node_provision_state.side_effect = timeout |
| 181 | + with pytest.raises( |
| 182 | + exceptions.Conflict, match="automated cleaning remains disabled" |
| 183 | + ): |
| 184 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 185 | + assert client.node.automated_clean is False |
| 186 | + assert client.node.extra[cli.CLEANING_POLICY_KEY] is True |
| 187 | + |
| 188 | + client.wait_for_node_provision_state.side_effect = wait |
| 189 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 190 | + assert client.node.provision_state == "available" |
| 191 | + assert client.node.automated_clean is True |
| 192 | + assert cli.CLEANING_POLICY_KEY not in client.node.extra |
| 193 | + |
| 194 | + |
| 195 | +@pytest.mark.parametrize("state", ["available", "manageable"]) |
| 196 | +def test_retry_recovers_original_null_policy_after_interruption(client, state): |
| 197 | + client.node.provision_state = state |
| 198 | + client.node.automated_clean = False |
| 199 | + client.node.extra[cli.CLEANING_POLICY_KEY] = None |
| 200 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 201 | + assert client.node.provision_state == "available" |
| 202 | + assert client.node.automated_clean is None |
| 203 | + assert client.node.extra == {"existing": "value"} |
| 204 | + |
| 205 | + |
| 206 | +def test_provide_rejection_keeps_cleaning_disabled_for_retry(client): |
| 207 | + client.node.provision_state = "available" |
| 208 | + client.node.automated_clean = True |
| 209 | + transition = client.set_node_provision_state.side_effect |
| 210 | + |
| 211 | + def reject_provide(node_uuid, action, **kwargs): |
| 212 | + if action == "provide": |
| 213 | + raise exceptions.BadRequest("Cannot provide") |
| 214 | + transition(node_uuid, action, **kwargs) |
| 215 | + |
| 216 | + client.set_node_provision_state.side_effect = reject_provide |
| 217 | + with pytest.raises( |
| 218 | + exceptions.Conflict, match="automated cleaning remains disabled" |
| 219 | + ): |
| 220 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 221 | + assert client.node.automated_clean is False |
| 222 | + assert client.node.extra[cli.CLEANING_POLICY_KEY] is True |
54 | 223 |
|
55 | 224 |
|
56 | | -def test_main_resolves_node_and_uses_monitoring_slot(mocker): |
| 225 | +def test_policy_restore_failure_keeps_recovery_data(client): |
| 226 | + client.node.provision_state = "available" |
| 227 | + update = client.update_node.side_effect |
| 228 | + |
| 229 | + def fail_restoration(node_uuid, patch): |
| 230 | + if patch[-1]["op"] == "remove": |
| 231 | + raise exceptions.ServiceUnavailable("Cannot restore policy") |
| 232 | + update(node_uuid, patch) |
| 233 | + |
| 234 | + client.update_node.side_effect = fail_restoration |
| 235 | + with pytest.raises(exceptions.ServiceUnavailable): |
| 236 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 237 | + assert client.node.automated_clean is False |
| 238 | + assert cli.CLEANING_POLICY_KEY in client.node.extra |
| 239 | + |
| 240 | + client.update_node.side_effect = update |
| 241 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 242 | + assert client.node.automated_clean is None |
| 243 | + assert cli.CLEANING_POLICY_KEY not in client.node.extra |
| 244 | + |
| 245 | + |
| 246 | +def test_busy_node_is_read_again_before_any_mutation(client): |
| 247 | + busy = deepcopy(client.node) |
| 248 | + busy.target_power_state = "power on" |
| 249 | + client.get_node.side_effect = [busy, client.node] |
| 250 | + get_idle = cli.get_idle_node.retry_with( # pyright: ignore[reportFunctionMemberAccess] |
| 251 | + wait=wait_none(), stop=stop_after_attempt(2) |
| 252 | + ) |
| 253 | + assert get_idle(client, "node-uuid") is client.node |
| 254 | + assert client.get_node.call_count == 2 |
| 255 | + client.update_node.assert_not_called() |
| 256 | + |
| 257 | + |
| 258 | +def test_main_resolves_node(mocker): |
57 | 259 | mocker.patch("sys.argv", ["configure-snmpv3-user", "--node", "Dell-ABC1234"]) |
58 | 260 | client = mocker.patch.object(cli, "IronicClient").return_value |
59 | 261 | client.get_node.return_value.uuid = "node-uuid" |
60 | 262 | configure = mocker.patch.object(cli, "configure_snmpv3_user") |
61 | | - |
62 | 263 | cli.main() |
| 264 | + configure.assert_called_once_with(client, "node-uuid") |
| 265 | + |
| 266 | + |
| 267 | +@pytest.mark.parametrize("action", ["manage", "clean", "service", "provide"]) |
| 268 | +def test_conflicting_transition_is_retried_without_repeating_runbook(client, action): |
| 269 | + client.node.provision_state = "active" if action == "service" else "available" |
| 270 | + transition = client.set_node_provision_state.side_effect |
| 271 | + rejected = False |
| 272 | + |
| 273 | + def conflict_once(node_uuid, requested, **kwargs): |
| 274 | + nonlocal rejected |
| 275 | + if requested == action and not rejected: |
| 276 | + rejected = True |
| 277 | + raise exceptions.Conflict("Reserved by another conductor") |
| 278 | + transition(node_uuid, requested, **kwargs) |
| 279 | + |
| 280 | + client.set_node_provision_state.side_effect = conflict_once |
| 281 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 282 | + actions = [c.args[1] for c in client.set_node_provision_state.call_args_list] |
| 283 | + assert actions.count(action) == 2 |
| 284 | + if action not in {"clean", "service"}: |
| 285 | + assert actions.count("clean") == 1 |
| 286 | + |
| 287 | + |
| 288 | +@pytest.mark.parametrize("restoring", [True, False]) |
| 289 | +def test_conflicting_policy_update_is_retried(client, restoring): |
| 290 | + client.node.provision_state = "available" |
| 291 | + client.node.automated_clean = True |
| 292 | + update = client.update_node.side_effect |
| 293 | + rejected = False |
| 294 | + |
| 295 | + def conflict_once(node_uuid, patch): |
| 296 | + nonlocal rejected |
| 297 | + if (patch[-1]["op"] == "remove") == restoring and not rejected: |
| 298 | + rejected = True |
| 299 | + raise exceptions.Conflict("Locked") |
| 300 | + update(node_uuid, patch) |
| 301 | + |
| 302 | + client.update_node.side_effect = conflict_once |
| 303 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 304 | + assert client.update_node.call_count == 3 |
| 305 | + assert client.node.automated_clean is True |
| 306 | + assert cli.CLEANING_POLICY_KEY not in client.node.extra |
| 307 | + |
| 308 | + |
| 309 | +def test_conductor_reservation_blocks_mutation(client): |
| 310 | + client.node.reservation = "conductor" |
| 311 | + with pytest.raises(exceptions.Conflict): |
| 312 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 313 | + client.set_node_provision_state.assert_not_called() |
| 314 | + |
| 315 | + |
| 316 | +def test_changed_state_is_revalidated_after_conflict(client): |
| 317 | + def conflict_and_change(*args, **kwargs): |
| 318 | + client.node.provision_state = "active" |
| 319 | + raise exceptions.Conflict("Changed concurrently") |
| 320 | + |
| 321 | + client.set_node_provision_state.side_effect = conflict_and_change |
| 322 | + with pytest.raises(ValueError, match="no longer ready"): |
| 323 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 324 | + assert client.set_node_provision_state.call_count == 1 |
| 325 | + |
| 326 | + |
| 327 | +def test_conflict_retries_are_bounded(client): |
| 328 | + client.set_node_provision_state.side_effect = exceptions.Conflict("Locked") |
| 329 | + with pytest.raises(exceptions.Conflict): |
| 330 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 331 | + assert client.set_node_provision_state.call_count == 3 |
| 332 | + client.wait_for_node_provision_state.assert_not_called() |
| 333 | + |
63 | 334 |
|
64 | | - configure.assert_called_once_with(client, "node-uuid", 5) |
| 335 | +@pytest.mark.parametrize("state", ["available", "manageable", "active"]) |
| 336 | +def test_runbook_failure_is_reported_even_when_state_is_restored(client, state): |
| 337 | + client.node.provision_state = state |
| 338 | + wait = client.wait_for_node_provision_state.side_effect |
65 | 339 |
|
| 340 | + def complete_with_error(node_uuid, expected): |
| 341 | + if client.node.provision_state in {"cleaning", "servicing"}: |
| 342 | + client.node.last_error = "SNMP credential update failed" |
| 343 | + wait(node_uuid, expected) |
66 | 344 |
|
67 | | -@pytest.mark.parametrize("slot", ["2", "17", "invalid"]) |
68 | | -def test_invalid_slot_is_rejected(slot): |
69 | | - with pytest.raises(SystemExit): |
70 | | - cli.argument_parser().parse_args(["--node", "node-uuid", "--slot", slot]) |
| 345 | + client.wait_for_node_provision_state.side_effect = complete_with_error |
| 346 | + with pytest.raises(RuntimeError, match="SNMP credential update failed"): |
| 347 | + cli.configure_snmpv3_user(client, "node-uuid") |
| 348 | + assert client.node.provision_state == state |
| 349 | + assert client.node.automated_clean is None |
| 350 | + assert cli.CLEANING_POLICY_KEY not in client.node.extra |
0 commit comments