@@ -21,6 +21,7 @@ import (
2121 meta "k8s.io/apimachinery/pkg/api/meta"
2222 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2323 k8stypes "k8s.io/apimachinery/pkg/types"
24+ "k8s.io/apimachinery/pkg/util/intstr"
2425 "sigs.k8s.io/controller-runtime/pkg/client"
2526
2627 bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1"
@@ -1293,3 +1294,178 @@ func TestOperatorRestartResilience(t *testing.T) {
12931294 Should (poolAllUpdated (1 , env .NodeImageUpdateDigest ()))
12941295 t .Logf ("Operator functioned normally after restart: rollout completed" )
12951296}
1297+
1298+ // TestMultiNodeRollout provisions a pool of multiple worker nodes
1299+ // (COS-5185: scale testing for bootc-operator) and verifies a rollout
1300+ // updates every node in the pool, respecting maxUnavailable, and that
1301+ // the pool status correctly reflects all nodes once the rollout
1302+ // completes. Unlike TestMidRolloutImageChange (bink-only, 2 nodes,
1303+ // exercises mid-rollout image switches), this test targets a larger
1304+ // pool and runs on EKS to validate behavior on real cloud nodes rather
1305+ // than the simulated bink provider, since upstream CI's GitHub runners
1306+ // cannot afford enough memory/CPU for multi-node scale testing.
1307+ func TestMultiNodeRollout (t * testing.T ) {
1308+ e2eutil .Providers (t , "bink" , "eks" )
1309+ t .Parallel ()
1310+ g := NewWithT (t )
1311+ g .SetDefaultEventuallyTimeout (pollTimeout )
1312+ g .SetDefaultEventuallyPollingInterval (pollInterval )
1313+
1314+ env := e2eutil .New (t )
1315+
1316+ const nodeCount = 3
1317+ nodes := make ([]string , nodeCount )
1318+ for i := range nodes {
1319+ nodes [i ] = env .AddNode (t )
1320+ }
1321+
1322+ ctx := context .Background ()
1323+
1324+ // Phase 1: Create pool with original image and wait for all nodes Idle.
1325+ // maxUnavailable bounds how many nodes may be disrupted at once.
1326+ const maxUnavailable = 1
1327+ pool := env .NewPool ("scale" , env .NodeImageDigestedPullSpec (),
1328+ testutil .WithMaxUnavailable (intstr .FromInt (maxUnavailable )))
1329+ g .Expect (env .Client .Create (ctx , pool )).To (Succeed ())
1330+
1331+ // Verify the daemon pod actually reaches Running on every node before
1332+ // waiting for Idle - a pod stuck in e.g. ImagePullBackOff would
1333+ // otherwise just show up as a confusing Idle-wait timeout below.
1334+ for _ , nodeName := range nodes {
1335+ g .Eventually (func () ([]corev1.Pod , error ) {
1336+ var pods corev1.PodList
1337+ err := env .Client .List (ctx , & pods ,
1338+ client .InNamespace (testutil .OperatorNamespaceName ),
1339+ client.MatchingFields {"spec.nodeName" : nodeName },
1340+ client.MatchingLabels {
1341+ "app.kubernetes.io/name" : "bootc-operator" ,
1342+ "app.kubernetes.io/component" : "daemon" ,
1343+ },
1344+ )
1345+ return pods .Items , err
1346+ }).WithTimeout (3 * time .Minute ).Should (ConsistOf (And (
1347+ HaveField ("Spec.NodeName" , nodeName ),
1348+ HaveField ("Status.Phase" , corev1 .PodRunning ),
1349+ )), "expected exactly one running daemon pod on %s" , nodeName )
1350+ }
1351+
1352+ for _ , nodeName := range nodes {
1353+ testutil .WaitForNodeIdle (t , g , ctx , env .Client , nodeName , 3 * time .Minute )
1354+ }
1355+
1356+ t .Logf ("All %d nodes are Idle with original image" , nodeCount )
1357+
1358+ g .Eventually (fetchPoolStatus (ctx , env .Client , pool )).
1359+ Should (poolAllUpdated (nodeCount , env .NodeImageDigest ()))
1360+
1361+ // Phase 2: Patch pool to the update image.
1362+ updateRef := env .NodeImageUpdateDigestedPullSpec ()
1363+
1364+ modified := pool .DeepCopy ()
1365+ modified .Spec .Image .Ref = updateRef
1366+ g .Expect (env .Client .Patch (ctx , modified , client .MergeFrom (pool ))).To (Succeed ())
1367+ * pool = * modified
1368+
1369+ t .Logf ("Patched pool to update image %s" , updateRef )
1370+
1371+ // Phase 3: Wait for every node to be Idle on the update digest - proves
1372+ // the rollout reached the full pool, not just a subset - while
1373+ // continuously enforcing maxUnavailable. Both checks share one polling
1374+ // loop so the invariant is sampled throughout the rollout rather than
1375+ // once before it starts, and so the wait ends as soon as the rollout
1376+ // does. Asserting the invariant once up front would pass trivially,
1377+ // since nothing is disrupted yet at that point.
1378+ //
1379+ // A node that stays Degraded aborts the wait early and reports the
1380+ // daemon's own error message. The daemon clears Degraded on the next
1381+ // successful reconcile, so a retried transient error must not fail the
1382+ // test; only a node degraded for longer than degradedGrace does.
1383+ const degradedGrace = 90 * time .Second
1384+ degradedSince := make (map [string ]time.Time )
1385+
1386+ g .Eventually (func (g Gomega ) {
1387+ // Snapshot the whole pool in a single List. The assertions below
1388+ // abort the iteration at the first node that is behind, so the
1389+ // invariant has to be evaluated across every node before any of
1390+ // them run. It must also be one List rather than a Get per node:
1391+ // a List is a consistent snapshot at one resourceVersion, whereas
1392+ // sequential Gets can straddle a slot handoff and assemble a view
1393+ // in which two nodes hold a slot, a state that never existed.
1394+ var bnList bootcv1alpha1.BootcNodeList
1395+ g .Expect (env .Client .List (ctx , & bnList )).To (Succeed ())
1396+
1397+ inPool := make (map [string ]bool , len (nodes ))
1398+ for _ , nodeName := range nodes {
1399+ inPool [nodeName ] = true
1400+ }
1401+ bns := make ([]bootcv1alpha1.BootcNode , 0 , len (nodes ))
1402+ for i := range bnList .Items {
1403+ if inPool [bnList .Items [i ].Name ] {
1404+ bns = append (bns , bnList .Items [i ])
1405+ }
1406+ }
1407+ g .Expect (bns ).To (HaveLen (len (nodes )), "expected a BootcNode per pool node" )
1408+
1409+ // maxUnavailable bounds reboot slots, not staging. Staging only
1410+ // downloads the image and disrupts nothing, so the controller lets
1411+ // the whole pool stage at once and gates cordon/drain/reboot. A
1412+ // node holds a slot exactly while it carries the in-reboot-slot
1413+ // annotation, which is also how the controller counts them.
1414+ //
1415+ // Exceeding the bound is a violated invariant rather than a state
1416+ // to wait out - retrying would let a later poll mask it.
1417+ inRebootSlot := 0
1418+ for i := range bns {
1419+ if metav1 .HasAnnotation (bns [i ].ObjectMeta , bootcv1alpha1 .AnnotationInRebootSlot ) {
1420+ inRebootSlot ++
1421+ }
1422+ }
1423+ if inRebootSlot > maxUnavailable {
1424+ StopTrying (fmt .Sprintf (
1425+ "maxUnavailable=%d violated: %d nodes hold a reboot slot at once" ,
1426+ maxUnavailable , inRebootSlot ,
1427+ )).Now ()
1428+ }
1429+
1430+ for i := range bns {
1431+ bn := & bns [i ]
1432+ if cond := meta .FindStatusCondition (
1433+ bn .Status .Conditions , bootcv1alpha1 .NodeDegraded ,
1434+ ); cond != nil && cond .Status == metav1 .ConditionTrue {
1435+ first , seen := degradedSince [bn .Name ]
1436+ switch {
1437+ case ! seen :
1438+ degradedSince [bn .Name ] = time .Now ()
1439+ case time .Since (first ) > degradedGrace :
1440+ StopTrying (fmt .Sprintf (
1441+ "node %s degraded for over %s during rollout: %s: %s" ,
1442+ bn .Name , degradedGrace , cond .Reason , cond .Message ,
1443+ )).Now ()
1444+ }
1445+ } else {
1446+ delete (degradedSince , bn .Name )
1447+ }
1448+ }
1449+
1450+ for i := range bns {
1451+ bn := & bns [i ]
1452+ g .Expect (bn .Status .Conditions ).To (ContainElement (And (
1453+ HaveField ("Type" , bootcv1alpha1 .NodeIdle ),
1454+ HaveField ("Status" , metav1 .ConditionTrue ),
1455+ HaveField ("Reason" , bootcv1alpha1 .NodeReasonIdle ),
1456+ )), "node %s is not Idle yet" , bn .Name )
1457+
1458+ g .Expect (bn .Status .Booted ).NotTo (BeNil ())
1459+ g .Expect (bn .Status .Booted ).To (
1460+ imageMatchesDigest (env .NodeImageUpdateDigest ()),
1461+ "node %s has not booted the update digest yet" , bn .Name ,
1462+ )
1463+ }
1464+ }).WithTimeout (20 * time .Minute ).Should (Succeed ())
1465+
1466+ t .Logf ("All %d nodes are Idle with update image" , nodeCount )
1467+
1468+ // Phase 4: Verify final pool status reflects all nodes updated.
1469+ g .Eventually (fetchPoolStatus (ctx , env .Client , pool )).
1470+ Should (poolAllUpdated (nodeCount , env .NodeImageUpdateDigest ()))
1471+ }
0 commit comments