Skip to content

Commit 2e1a99d

Browse files
committed
fix(neutron-understack): roll back failed Palo Alto gateway attach
Neutron commits the gateway port before ROUTER_GATEWAY AFTER_CREATE and does not undo it when a subscriber raises. A failure partway through left the parent port, VIF and sometimes the trunk behind, and setting the gateway again did nothing. - Snapshot the router's wiring before attaching and undo only what this request added, using the same snapshot/undo as the interface path. - Raise PaloAltoGatewayWiringFailed (409) for retriable DB errors. Neutron would otherwise retry update_router, skip the event because the gateway already exists, and return 200 for an unwired gateway. - On BEFORE_CREATE, reject a router with no adopted node, or whose node has no port with physical_network and local_link_connection, before Neutron creates anything. Also small cleanups in palo_alto.py and a trimmed test suite.
1 parent b25589f commit 2e1a99d

6 files changed

Lines changed: 469 additions & 409 deletions

File tree

‎python/neutron-understack/neutron_understack/ironic.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from openstack import exceptions as sdk_exc
77
from openstack.baremetal.baremetal_service import BaremetalService
88
from openstack.baremetal.v1.node import Node as BaremetalNode
9+
from openstack.baremetal.v1.port import Port as BaremetalPort
910
from oslo_config import cfg
1011

1112
from neutron_understack import config
@@ -178,6 +179,11 @@ def node_vif_ids(self, node: str | BaremetalNode) -> list[str]:
178179
"""Return the Neutron port (VIF) ids currently attached to the node."""
179180
return self.irclient.list_node_vifs(node)
180181

182+
def node_ports(self, node: str | BaremetalNode) -> list[BaremetalPort]:
183+
"""Return the node's baremetal ports, including their network fields."""
184+
node_id = node.id if isinstance(node, BaremetalNode) else node
185+
return list(self.irclient.ports(details=True, node_id=node_id))
186+
181187
def adopt_node_for_router(
182188
self,
183189
node: str | BaremetalNode,

‎python/neutron-understack/neutron_understack/l3_router/palo_alto.py‎

Lines changed: 94 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import functools
12
import json
23
import logging
34
import weakref
@@ -7,13 +8,16 @@
78
from neutron_lib import constants as const
89
from neutron_lib import exceptions as n_exc
910
from neutron_lib.callbacks import events
11+
from neutron_lib.callbacks import exceptions as callback_exc
1012
from neutron_lib.callbacks import priority_group
1113
from neutron_lib.callbacks import registry
1214
from neutron_lib.callbacks import resources
15+
from neutron_lib.db import api as db_api
1316
from neutron_lib.plugins import constants as plugin_constants
1417
from neutron_lib.plugins import directory
1518

1619
from neutron_understack.ironic import IronicClient
20+
from neutron_understack.l3_router.palo_alto_wiring import AttachmentSnapshot
1721
from neutron_understack.l3_router.palo_alto_wiring import PaloAltoWiring
1822

1923
LOG = logging.getLogger(__name__)
@@ -52,6 +56,28 @@ class PaloAltoInterfacePortMissing(n_exc.BadRequest):
5256
)
5357

5458

59+
# Conflict -> HTTP 409: a transient error interrupted gateway wiring. The wiring
60+
# was rolled back, but Neutron keeps the gateway, so it must be set again.
61+
class PaloAltoGatewayWiringFailed(n_exc.Conflict):
62+
message = (
63+
"Router %(router_id)s gateway was created but could not be wired "
64+
"(%(reason)s); any Palo Alto wiring was rolled back. Unset and set the "
65+
"gateway again to retry."
66+
)
67+
68+
69+
def _registry_would_retry(error: Exception) -> bool:
70+
"""Return True if the callback registry would turn this error into a retry.
71+
72+
Checked on the CallbackFailure wrapper, as the registry does: that check
73+
ignores the "retries exhausted" flag that is_retriable(error) honours.
74+
"""
75+
wrapped = callback_exc.CallbackFailure(
76+
errors=[callback_exc.NotificationError(None, error)]
77+
)
78+
return db_api.is_retriable(wrapped)
79+
80+
5581
def _parse_metainfo(raw) -> dict:
5682
"""Service-profile metainfo is stored as a JSON string."""
5783
if not raw:
@@ -85,6 +111,14 @@ def __init__(self, l3_plugin):
85111
self._palo_alto_provider = f"{__name__}.{self.__class__.__name__}"
86112
self._interface_snapshots = weakref.WeakKeyDictionary()
87113
self._wiring = PaloAltoWiring(ironic=lambda: self._ironic)
114+
# Check the appliance can be wired on BEFORE_CREATE, before Neutron
115+
# creates the gateway port: rejecting there leaves nothing behind and
116+
# the request can be retried. BEFORE events always re-raise errors.
117+
registry.subscribe(
118+
self._validate_gateway_create,
119+
resources.ROUTER_GATEWAY,
120+
events.BEFORE_CREATE,
121+
)
88122
# Gateway attach must run on AFTER_CREATE (the gateway port does not
89123
# exist earlier) and must be cancellable so a wiring failure returns a
90124
# real API error instead of a swallowed 200. @registry.receives cannot
@@ -131,23 +165,15 @@ def __init__(self, l3_plugin):
131165
self._palo_alto_provider,
132166
)
133167

134-
@property
168+
@functools.cached_property
135169
def _flavor_plugin(self):
136-
try:
137-
return self._flavor_plugin_ref
138-
except AttributeError:
139-
self._flavor_plugin_ref = directory.get_plugin(plugin_constants.FLAVORS)
140-
return self._flavor_plugin_ref
170+
return directory.get_plugin(plugin_constants.FLAVORS)
141171

142-
@property
172+
@functools.cached_property
143173
def _ironic(self) -> IronicClient:
144174
# Instantiated lazily so importing/loading this provider does not require
145175
# Ironic credentials in environments that never create such a router.
146-
try:
147-
return self._ironic_ref
148-
except AttributeError:
149-
self._ironic_ref = IronicClient()
150-
return self._ironic_ref
176+
return IronicClient()
151177

152178
def _is_palo_alto_provider(self, context, router) -> bool:
153179
flavor_id = router.get("flavor_id")
@@ -173,12 +199,10 @@ def _is_palo_alto_provider(self, context, router) -> bool:
173199
)
174200
return matched
175201

176-
def _palo_alto_router(self, context, router_id: str) -> dict | None:
177-
"""Return the router if it uses the Palo Alto flavor, else None."""
202+
def _is_palo_alto_router(self, context, router_id: str) -> bool:
203+
"""Look the router up by id; return True if it uses the Palo Alto flavor."""
178204
router = self.l3plugin.get_router(context, router_id)
179-
if not self._is_palo_alto_provider(context, router):
180-
return None
181-
return router
205+
return self._is_palo_alto_provider(context, router)
182206

183207
def _resource_class_for_router(self, context, router) -> str:
184208
"""Read the target resource_class from the flavor's profile metainfo.
@@ -274,31 +298,55 @@ def _process_router_delete(self, resource, event, trigger, payload=None):
274298
router["id"],
275299
)
276300

301+
def _validate_gateway_create(self, resource, event, trigger, payload=None):
302+
"""ROUTER_GATEWAY / BEFORE_CREATE: reject gateways that cannot be wired."""
303+
router_id = payload.resource_id
304+
if not self._is_palo_alto_router(payload.context, router_id):
305+
return
306+
307+
self._wiring.check_can_attach(router_id)
308+
277309
def _process_gateway_create(self, resource, event, trigger, payload=None):
278310
"""ROUTER_GATEWAY / AFTER_CREATE (cancellable): wire the gateway.
279311
280312
Orders the building blocks so the parent port is VIF-bound before the
281313
subport is added -- the trunk driver only programs the switchport once
282314
the parent is bound.
315+
316+
Neutron has already committed the gateway port here and does not undo
317+
it when this raises, so a failure rolls back the wiring this request
318+
added before re-raising.
283319
"""
284320
router_id = payload.resource_id
285-
if self._palo_alto_router(payload.context, router_id) is None:
321+
try:
322+
self._wire_gateway(payload.context, router_id)
323+
except Exception as error:
324+
if _registry_would_retry(error):
325+
# Neutron would retry update_router; the retry finds the
326+
# gateway already created, skips this event and reports
327+
# success for a gateway that is not wired.
328+
raise PaloAltoGatewayWiringFailed(
329+
router_id=router_id, reason=type(error).__name__
330+
) from error
331+
raise
332+
333+
def _wire_gateway(self, context, router_id: str) -> None:
334+
"""Wire a Palo Alto router's gateway; undo this request's wiring on error."""
335+
if not self._is_palo_alto_router(context, router_id):
286336
return
287337

288338
gateway_port = self._wiring.gateway_port_for_router(router_id)
289339
if gateway_port is None:
290340
raise PaloAltoGatewayPortNotFound(router_id=router_id)
291341

292-
stack = self._wiring.ensure_stack(router_id)
293-
self._wiring.add_gateway_subport(router_id, stack.trunk, gateway_port)
294-
295-
LOG.info(
296-
"Attached Palo Alto router %s gateway port %s via parent %s trunk %s",
297-
router_id,
298-
gateway_port["id"],
299-
stack.parent["id"],
300-
stack.trunk["id"],
342+
snapshot = self._wiring.snapshot_attachment(
343+
router_id, gateway_port["id"], "gateway"
301344
)
345+
try:
346+
self._wiring.attach_port(router_id, gateway_port, "gateway")
347+
except Exception:
348+
self._undo_attachment(snapshot)
349+
raise
302350

303351
def _process_gateway_delete(self, resource, event, trigger, payload=None):
304352
"""ROUTER_GATEWAY / BEFORE_DELETE (cancellable): tear down the wiring.
@@ -307,7 +355,7 @@ def _process_gateway_delete(self, resource, event, trigger, payload=None):
307355
remove its subport before Neutron deletes it.
308356
"""
309357
router_id = payload.resource_id
310-
if self._palo_alto_router(payload.context, router_id) is None:
358+
if not self._is_palo_alto_router(payload.context, router_id):
311359
return
312360

313361
gateway_port = self._wiring.gateway_port_for_router(router_id)
@@ -328,7 +376,7 @@ def _process_gateway_delete(self, resource, event, trigger, payload=None):
328376
def _process_router_interface_create(self, resource, event, trigger, payload=None):
329377
"""Wire the interface before Neutron commits its RouterPort association."""
330378
router_id = payload.resource_id
331-
if self._palo_alto_router(payload.context, router_id) is None:
379+
if not self._is_palo_alto_router(payload.context, router_id):
332380
return
333381

334382
interface_port = payload.metadata.get("port")
@@ -337,44 +385,41 @@ def _process_router_interface_create(self, resource, event, trigger, payload=Non
337385
if interface_port.get("device_owner") not in const.ROUTER_INTERFACE_OWNERS:
338386
return
339387

340-
self._interface_snapshots[payload] = self._wiring.snapshot_interface_attachment(
341-
router_id, interface_port["id"]
388+
self._interface_snapshots[payload] = self._wiring.snapshot_attachment(
389+
router_id, interface_port["id"], "interface"
342390
)
343391
try:
344-
stack = self._wiring.ensure_stack(router_id)
345-
self._wiring.add_interface_subport(router_id, stack.trunk, interface_port)
392+
self._wiring.attach_port(router_id, interface_port, "interface")
346393
except Exception:
347394
# Attach-by-port is reverted with a port update by Neutron, so it
348395
# cannot rely on PORT/BEFORE_DELETE to undo partial realization.
349396
self._rollback_interface_attachment(payload)
350397
raise
351398

352-
LOG.info(
353-
"Attached Palo Alto router %s interface port %s via parent %s trunk %s",
354-
router_id,
355-
interface_port["id"],
356-
stack.parent["id"],
357-
stack.trunk["id"],
358-
)
359-
360399
def _rollback_interface_attachment(self, payload) -> None:
361400
"""Undo this request's changes, including calls that failed postcommit."""
362401
snapshot = self._interface_snapshots.get(payload)
363402
if snapshot is None:
364403
return
404+
# Preserve the original attach error. Keep the snapshot so the
405+
# subsequent ABORT_CREATE can retry compensation if it failed here.
406+
if self._undo_attachment(snapshot):
407+
self._interface_snapshots.pop(payload, None)
408+
409+
def _undo_attachment(self, snapshot: AttachmentSnapshot) -> bool:
410+
"""Undo an attach's wiring. A failed undo is logged and returns False."""
365411
try:
366-
self._wiring.undo_interface_attachment(snapshot)
412+
self._wiring.undo_attachment(snapshot)
367413
except Exception:
368-
# Preserve the original attach error. Keep the snapshot so the
369-
# subsequent ABORT_CREATE can retry compensation if it failed here.
370414
LOG.exception(
371-
"Failed to roll back Palo Alto router %s interface port %s; "
415+
"Failed to roll back Palo Alto router %s %s port %s; "
372416
"attachment cleanup is incomplete",
373417
snapshot.router_id,
418+
snapshot.label,
374419
snapshot.port_id,
375420
)
376-
return
377-
self._interface_snapshots.pop(payload, None)
421+
return False
422+
return True
378423

379424
def _process_router_interface_abort(self, resource, event, trigger, payload=None):
380425
# The registry continues invoking callbacks after validation errors. An
@@ -403,7 +448,7 @@ def _process_router_interface_port_delete(
403448
return
404449

405450
try:
406-
router = self._palo_alto_router(context, router_id)
451+
is_palo_alto = self._is_palo_alto_router(context, router_id)
407452
except Exception:
408453
LOG.debug(
409454
"Skipping Palo Alto interface cleanup for port %s; router %s "
@@ -414,7 +459,7 @@ def _process_router_interface_port_delete(
414459
)
415460
return
416461

417-
if router is None:
462+
if not is_palo_alto:
418463
return
419464

420465
# Neutron also deletes newly-created ports when attachment aborts. That

0 commit comments

Comments
 (0)