Skip to content

Commit c2d16f7

Browse files
run SNMPv3 runbook through Ironic cleaning or servicing by temporarily disabling automated cleaning
1 parent 0a8c6d4 commit c2d16f7

4 files changed

Lines changed: 464 additions & 55 deletions

File tree

‎containers/ironic/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ RUN apt-get update && \
1212
&& apt-get clean && rm -rf /var/lib/apt/lists/*
1313

1414
# renovate: name=openstack/ironic repo=https://github.com/rackerlabs/ironic.git branch=understack/2026.1
15-
ARG IRONIC_GIT_REF=a5181744455932776ea9f38171e7f2e699918e44
15+
ARG IRONIC_GIT_REF=7800737e179295a3bf73590238af3d58875a5040
1616
ADD --keep-git-dir=true https://github.com/rackerlabs/ironic.git#${IRONIC_GIT_REF} /src/ironic
1717
RUN git -C /src/ironic fetch --unshallow --tags
1818

Lines changed: 316 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from copy import deepcopy
2+
from types import SimpleNamespace
13
from unittest.mock import Mock
24
from unittest.mock import call
35

@@ -6,65 +8,343 @@
68
from tenacity import stop_after_attempt
79
from tenacity import wait_none
810

11+
from understack_workflows.ironic.client import IronicClient
912
from understack_workflows.main import configure_snmpv3_user as cli
1013

1114

12-
def test_conflict_retries_the_same_request_without_lifecycle_calls():
13-
client = Mock()
14-
client.client.node.vendor_passthru.side_effect = [
15-
exceptions.Conflict("Node is busy"),
16-
{"configured": True},
17-
]
18-
# Tenacity adds retry_with at runtime but does not expose it in its types.
19-
configure = cli.configure_snmpv3_user.retry_with( # pyright: ignore[reportFunctionMemberAccess]
20-
wait=wait_none(), stop=stop_after_attempt(2)
15+
@pytest.fixture(autouse=True)
16+
def _fast_retries(monkeypatch):
17+
for name in (
18+
"get_idle_node",
19+
"request_transition",
20+
"suppress_cleaning",
21+
"restore_cleaning_policy",
22+
):
23+
policy = getattr(cli, name).retry
24+
monkeypatch.setattr(policy, "wait", wait_none())
25+
monkeypatch.setattr(policy, "stop", stop_after_attempt(3))
26+
27+
28+
@pytest.fixture
29+
def client():
30+
client = Mock(spec=IronicClient)
31+
node = SimpleNamespace(
32+
uuid="node-uuid",
33+
provision_state="manageable",
34+
target_provision_state=None,
35+
target_power_state=None,
36+
reservation=None,
37+
last_error=None,
38+
maintenance=False,
39+
automated_clean=None,
40+
extra={"existing": "value"},
2141
)
42+
client.node = node
43+
client.get_node.side_effect = lambda _: deepcopy(node)
44+
client.get_runbook.return_value = SimpleNamespace(
45+
uuid="runbook-uuid", disable_ramdisk=True, traits=["DELL"]
46+
)
47+
client.get_node_traits.return_value = ["DELL"]
48+
49+
def update(_, patch):
50+
for operation in patch:
51+
path = operation["path"]
52+
if path == "/automated_clean":
53+
node.automated_clean = operation["value"]
54+
elif operation["op"] == "remove":
55+
del node.extra[path.split("/")[-1]]
56+
else:
57+
node.extra[path.split("/")[-1]] = operation["value"]
58+
59+
def transition(_, action, **kwargs):
60+
if action == "provide":
61+
assert node.automated_clean is False
62+
node.provision_state = {
63+
"manage": "manageable",
64+
"clean": "cleaning",
65+
"service": "servicing",
66+
"provide": "cleaning",
67+
}[action]
68+
node.target_provision_state = {
69+
"clean": "manageable",
70+
"service": "active",
71+
"provide": "available",
72+
}.get(action)
73+
74+
def wait(_, expected):
75+
node.provision_state = expected
76+
node.target_provision_state = None
77+
78+
client.update_node.side_effect = update
79+
client.set_node_provision_state.side_effect = transition
80+
client.wait_for_node_provision_state.side_effect = wait
81+
return client
82+
83+
84+
@pytest.mark.parametrize("original", [None, True, False])
85+
def test_available_node_skips_automated_cleaning_and_restores_policy(client, original):
86+
client.node.provision_state = "available"
87+
client.node.automated_clean = original
88+
89+
cli.configure_snmpv3_user(client, "node-uuid")
90+
91+
assert client.set_node_provision_state.call_args_list == [
92+
call("node-uuid", "manage"),
93+
call("node-uuid", "clean", runbook="runbook-uuid"),
94+
call("node-uuid", "provide"),
95+
]
96+
assert client.node.provision_state == "available"
97+
assert client.node.automated_clean is original
98+
assert client.node.extra == {"existing": "value"}
99+
22100

23-
configure(client, "node-uuid", 5)
101+
@pytest.mark.parametrize(
102+
("state", "action"), [("manageable", "clean"), ("active", "service")]
103+
)
104+
def test_runbook_uses_existing_state_without_changing_cleaning_policy(
105+
client, state, action
106+
):
107+
client.node.provision_state = state
24108

25-
request = call.client.node.vendor_passthru(
26-
"node-uuid", "configure_snmpv3_user", args={"slot": 5}, http_method="POST"
109+
cli.configure_snmpv3_user(client, "node-uuid")
110+
111+
client.set_node_provision_state.assert_called_once_with(
112+
"node-uuid", action, runbook="runbook-uuid"
27113
)
28-
assert client.mock_calls == [request, request]
114+
client.update_node.assert_not_called()
115+
assert client.node.provision_state == state
29116

30117

31-
@pytest.mark.parametrize("error", [exceptions.BadRequest, exceptions.Forbidden])
32-
def test_permanent_errors_are_not_retried(error):
33-
client = Mock()
34-
client.client.node.vendor_passthru.side_effect = error("Cannot configure")
118+
@pytest.mark.parametrize(
119+
"state", ["inspect failed", "enroll", "rescue", "clean failed"]
120+
)
121+
def test_unsupported_states_are_unchanged(client, state):
122+
client.node.provision_state = state
123+
with pytest.raises(ValueError, match="Cannot run the SNMP runbook"):
124+
cli.configure_snmpv3_user(client, "node-uuid")
125+
client.set_node_provision_state.assert_not_called()
126+
client.update_node.assert_not_called()
35127

36-
with pytest.raises(error):
37-
cli.configure_snmpv3_user(client, "node-uuid", 5)
38128

39-
client.client.node.vendor_passthru.assert_called_once()
129+
@pytest.mark.parametrize("invalid", ["maintenance", "traits", "ramdisk"])
130+
def test_validation_precedes_policy_and_state_changes(client, invalid):
131+
client.node.provision_state = "available"
132+
if invalid == "maintenance":
133+
client.node.maintenance = True
134+
elif invalid == "traits":
135+
client.get_node_traits.return_value = []
136+
else:
137+
client.get_runbook.return_value.disable_ramdisk = False
138+
with pytest.raises(
139+
ValueError, match=r"maintenance|matching trait|disable the ramdisk"
140+
):
141+
cli.configure_snmpv3_user(client, "node-uuid")
142+
client.set_node_provision_state.assert_not_called()
143+
client.update_node.assert_not_called()
40144

41145

42-
def test_busy_node_eventually_fails():
43-
client = Mock()
44-
client.client.node.vendor_passthru.side_effect = exceptions.Conflict("Busy")
45-
# Tenacity adds retry_with at runtime but does not expose it in its types.
46-
configure = cli.configure_snmpv3_user.retry_with( # pyright: ignore[reportFunctionMemberAccess]
47-
wait=wait_none(), stop=stop_after_attempt(2)
146+
def test_cleaning_failure_restores_policy_without_providing(client):
147+
client.node.provision_state = "available"
148+
client.node.automated_clean = True
149+
wait = client.wait_for_node_provision_state.side_effect
150+
151+
def fail_cleaning(node_uuid, expected):
152+
if expected == "manageable" and client.node.provision_state == "cleaning":
153+
client.node.provision_state = "clean failed"
154+
raise RuntimeError("SNMP failed")
155+
wait(node_uuid, expected)
156+
157+
client.wait_for_node_provision_state.side_effect = fail_cleaning
158+
with pytest.raises(RuntimeError, match="SNMP failed"):
159+
cli.configure_snmpv3_user(client, "node-uuid")
160+
assert client.node.automated_clean is True
161+
assert cli.CLEANING_POLICY_KEY not in client.node.extra
162+
assert client.node.provision_state == "clean failed"
163+
assert (
164+
call("node-uuid", "provide")
165+
not in client.set_node_provision_state.call_args_list
48166
)
49167

50-
with pytest.raises(exceptions.Conflict):
51-
configure(client, "node-uuid", 5)
52168

53-
assert client.client.node.vendor_passthru.call_count == 2
169+
@pytest.mark.parametrize("target", ["manageable", "available"])
170+
def test_timeout_keeps_suppression_and_saved_policy_until_recovery(client, target):
171+
client.node.provision_state = "available"
172+
client.node.automated_clean = True
173+
wait = client.wait_for_node_provision_state.side_effect
174+
175+
def timeout(node_uuid, expected):
176+
if client.node.target_provision_state == target:
177+
raise TimeoutError("Still cleaning")
178+
wait(node_uuid, expected)
179+
180+
client.wait_for_node_provision_state.side_effect = timeout
181+
with pytest.raises(
182+
exceptions.Conflict, match="automated cleaning remains disabled"
183+
):
184+
cli.configure_snmpv3_user(client, "node-uuid")
185+
assert client.node.automated_clean is False
186+
assert client.node.extra[cli.CLEANING_POLICY_KEY] is True
187+
188+
client.wait_for_node_provision_state.side_effect = wait
189+
cli.configure_snmpv3_user(client, "node-uuid")
190+
assert client.node.provision_state == "available"
191+
assert client.node.automated_clean is True
192+
assert cli.CLEANING_POLICY_KEY not in client.node.extra
193+
194+
195+
@pytest.mark.parametrize("state", ["available", "manageable"])
196+
def test_retry_recovers_original_null_policy_after_interruption(client, state):
197+
client.node.provision_state = state
198+
client.node.automated_clean = False
199+
client.node.extra[cli.CLEANING_POLICY_KEY] = None
200+
cli.configure_snmpv3_user(client, "node-uuid")
201+
assert client.node.provision_state == "available"
202+
assert client.node.automated_clean is None
203+
assert client.node.extra == {"existing": "value"}
204+
205+
206+
def test_provide_rejection_keeps_cleaning_disabled_for_retry(client):
207+
client.node.provision_state = "available"
208+
client.node.automated_clean = True
209+
transition = client.set_node_provision_state.side_effect
210+
211+
def reject_provide(node_uuid, action, **kwargs):
212+
if action == "provide":
213+
raise exceptions.BadRequest("Cannot provide")
214+
transition(node_uuid, action, **kwargs)
215+
216+
client.set_node_provision_state.side_effect = reject_provide
217+
with pytest.raises(
218+
exceptions.Conflict, match="automated cleaning remains disabled"
219+
):
220+
cli.configure_snmpv3_user(client, "node-uuid")
221+
assert client.node.automated_clean is False
222+
assert client.node.extra[cli.CLEANING_POLICY_KEY] is True
54223

55224

56-
def test_main_resolves_node_and_uses_monitoring_slot(mocker):
225+
def test_policy_restore_failure_keeps_recovery_data(client):
226+
client.node.provision_state = "available"
227+
update = client.update_node.side_effect
228+
229+
def fail_restoration(node_uuid, patch):
230+
if patch[-1]["op"] == "remove":
231+
raise exceptions.ServiceUnavailable("Cannot restore policy")
232+
update(node_uuid, patch)
233+
234+
client.update_node.side_effect = fail_restoration
235+
with pytest.raises(exceptions.ServiceUnavailable):
236+
cli.configure_snmpv3_user(client, "node-uuid")
237+
assert client.node.automated_clean is False
238+
assert cli.CLEANING_POLICY_KEY in client.node.extra
239+
240+
client.update_node.side_effect = update
241+
cli.configure_snmpv3_user(client, "node-uuid")
242+
assert client.node.automated_clean is None
243+
assert cli.CLEANING_POLICY_KEY not in client.node.extra
244+
245+
246+
def test_busy_node_is_read_again_before_any_mutation(client):
247+
busy = deepcopy(client.node)
248+
busy.target_power_state = "power on"
249+
client.get_node.side_effect = [busy, client.node]
250+
get_idle = cli.get_idle_node.retry_with( # pyright: ignore[reportFunctionMemberAccess]
251+
wait=wait_none(), stop=stop_after_attempt(2)
252+
)
253+
assert get_idle(client, "node-uuid") is client.node
254+
assert client.get_node.call_count == 2
255+
client.update_node.assert_not_called()
256+
257+
258+
def test_main_resolves_node(mocker):
57259
mocker.patch("sys.argv", ["configure-snmpv3-user", "--node", "Dell-ABC1234"])
58260
client = mocker.patch.object(cli, "IronicClient").return_value
59261
client.get_node.return_value.uuid = "node-uuid"
60262
configure = mocker.patch.object(cli, "configure_snmpv3_user")
61-
62263
cli.main()
264+
configure.assert_called_once_with(client, "node-uuid")
265+
266+
267+
@pytest.mark.parametrize("action", ["manage", "clean", "service", "provide"])
268+
def test_conflicting_transition_is_retried_without_repeating_runbook(client, action):
269+
client.node.provision_state = "active" if action == "service" else "available"
270+
transition = client.set_node_provision_state.side_effect
271+
rejected = False
272+
273+
def conflict_once(node_uuid, requested, **kwargs):
274+
nonlocal rejected
275+
if requested == action and not rejected:
276+
rejected = True
277+
raise exceptions.Conflict("Reserved by another conductor")
278+
transition(node_uuid, requested, **kwargs)
279+
280+
client.set_node_provision_state.side_effect = conflict_once
281+
cli.configure_snmpv3_user(client, "node-uuid")
282+
actions = [c.args[1] for c in client.set_node_provision_state.call_args_list]
283+
assert actions.count(action) == 2
284+
if action not in {"clean", "service"}:
285+
assert actions.count("clean") == 1
286+
287+
288+
@pytest.mark.parametrize("restoring", [True, False])
289+
def test_conflicting_policy_update_is_retried(client, restoring):
290+
client.node.provision_state = "available"
291+
client.node.automated_clean = True
292+
update = client.update_node.side_effect
293+
rejected = False
294+
295+
def conflict_once(node_uuid, patch):
296+
nonlocal rejected
297+
if (patch[-1]["op"] == "remove") == restoring and not rejected:
298+
rejected = True
299+
raise exceptions.Conflict("Locked")
300+
update(node_uuid, patch)
301+
302+
client.update_node.side_effect = conflict_once
303+
cli.configure_snmpv3_user(client, "node-uuid")
304+
assert client.update_node.call_count == 3
305+
assert client.node.automated_clean is True
306+
assert cli.CLEANING_POLICY_KEY not in client.node.extra
307+
308+
309+
def test_conductor_reservation_blocks_mutation(client):
310+
client.node.reservation = "conductor"
311+
with pytest.raises(exceptions.Conflict):
312+
cli.configure_snmpv3_user(client, "node-uuid")
313+
client.set_node_provision_state.assert_not_called()
314+
315+
316+
def test_changed_state_is_revalidated_after_conflict(client):
317+
def conflict_and_change(*args, **kwargs):
318+
client.node.provision_state = "active"
319+
raise exceptions.Conflict("Changed concurrently")
320+
321+
client.set_node_provision_state.side_effect = conflict_and_change
322+
with pytest.raises(ValueError, match="no longer ready"):
323+
cli.configure_snmpv3_user(client, "node-uuid")
324+
assert client.set_node_provision_state.call_count == 1
325+
326+
327+
def test_conflict_retries_are_bounded(client):
328+
client.set_node_provision_state.side_effect = exceptions.Conflict("Locked")
329+
with pytest.raises(exceptions.Conflict):
330+
cli.configure_snmpv3_user(client, "node-uuid")
331+
assert client.set_node_provision_state.call_count == 3
332+
client.wait_for_node_provision_state.assert_not_called()
333+
63334

64-
configure.assert_called_once_with(client, "node-uuid", 5)
335+
@pytest.mark.parametrize("state", ["available", "manageable", "active"])
336+
def test_runbook_failure_is_reported_even_when_state_is_restored(client, state):
337+
client.node.provision_state = state
338+
wait = client.wait_for_node_provision_state.side_effect
65339

340+
def complete_with_error(node_uuid, expected):
341+
if client.node.provision_state in {"cleaning", "servicing"}:
342+
client.node.last_error = "SNMP credential update failed"
343+
wait(node_uuid, expected)
66344

67-
@pytest.mark.parametrize("slot", ["2", "17", "invalid"])
68-
def test_invalid_slot_is_rejected(slot):
69-
with pytest.raises(SystemExit):
70-
cli.argument_parser().parse_args(["--node", "node-uuid", "--slot", slot])
345+
client.wait_for_node_provision_state.side_effect = complete_with_error
346+
with pytest.raises(RuntimeError, match="SNMP credential update failed"):
347+
cli.configure_snmpv3_user(client, "node-uuid")
348+
assert client.node.provision_state == state
349+
assert client.node.automated_clean is None
350+
assert cli.CLEANING_POLICY_KEY not in client.node.extra

0 commit comments

Comments
 (0)