Skip to content

Commit 1d6aa4e

Browse files
committed
e2e: add eks test case for scaling node
Signed-off-by: Prachiti Talgulkar <ptalgulk01@users.noreply.github.com>
1 parent 776157e commit 1d6aa4e

1 file changed

Lines changed: 176 additions & 0 deletions

File tree

‎test/e2e/bootcnode_test.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
meta "k8s.io/apimachinery/pkg/api/meta"
2222
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2323
k8stypes "k8s.io/apimachinery/pkg/types"
24+
"k8s.io/apimachinery/pkg/util/intstr"
2425
"sigs.k8s.io/controller-runtime/pkg/client"
2526

2627
bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1"
@@ -1293,3 +1294,178 @@ func TestOperatorRestartResilience(t *testing.T) {
12931294
Should(poolAllUpdated(1, env.NodeImageUpdateDigest()))
12941295
t.Logf("Operator functioned normally after restart: rollout completed")
12951296
}
1297+
1298+
// TestMultiNodeRollout provisions a pool of multiple worker nodes
1299+
// (COS-5185: scale testing for bootc-operator) and verifies a rollout
1300+
// updates every node in the pool, respecting maxUnavailable, and that
1301+
// the pool status correctly reflects all nodes once the rollout
1302+
// completes. Unlike TestMidRolloutImageChange (bink-only, 2 nodes,
1303+
// exercises mid-rollout image switches), this test targets a larger
1304+
// pool and runs on EKS to validate behavior on real cloud nodes rather
1305+
// than the simulated bink provider, since upstream CI's GitHub runners
1306+
// cannot afford enough memory/CPU for multi-node scale testing.
1307+
func TestMultiNodeRollout(t *testing.T) {
1308+
e2eutil.Providers(t, "bink", "eks")
1309+
t.Parallel()
1310+
g := NewWithT(t)
1311+
g.SetDefaultEventuallyTimeout(pollTimeout)
1312+
g.SetDefaultEventuallyPollingInterval(pollInterval)
1313+
1314+
env := e2eutil.New(t)
1315+
1316+
const nodeCount = 3
1317+
nodes := make([]string, nodeCount)
1318+
for i := range nodes {
1319+
nodes[i] = env.AddNode(t)
1320+
}
1321+
1322+
ctx := context.Background()
1323+
1324+
// Phase 1: Create pool with original image and wait for all nodes Idle.
1325+
// maxUnavailable bounds how many nodes may be disrupted at once.
1326+
const maxUnavailable = 1
1327+
pool := env.NewPool("scale", env.NodeImageDigestedPullSpec(),
1328+
testutil.WithMaxUnavailable(intstr.FromInt(maxUnavailable)))
1329+
g.Expect(env.Client.Create(ctx, pool)).To(Succeed())
1330+
1331+
// Verify the daemon pod actually reaches Running on every node before
1332+
// waiting for Idle - a pod stuck in e.g. ImagePullBackOff would
1333+
// otherwise just show up as a confusing Idle-wait timeout below.
1334+
for _, nodeName := range nodes {
1335+
g.Eventually(func() ([]corev1.Pod, error) {
1336+
var pods corev1.PodList
1337+
err := env.Client.List(ctx, &pods,
1338+
client.InNamespace(testutil.OperatorNamespaceName),
1339+
client.MatchingFields{"spec.nodeName": nodeName},
1340+
client.MatchingLabels{
1341+
"app.kubernetes.io/name": "bootc-operator",
1342+
"app.kubernetes.io/component": "daemon",
1343+
},
1344+
)
1345+
return pods.Items, err
1346+
}).WithTimeout(3*time.Minute).Should(ConsistOf(And(
1347+
HaveField("Spec.NodeName", nodeName),
1348+
HaveField("Status.Phase", corev1.PodRunning),
1349+
)), "expected exactly one running daemon pod on %s", nodeName)
1350+
}
1351+
1352+
for _, nodeName := range nodes {
1353+
testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute)
1354+
}
1355+
1356+
t.Logf("All %d nodes are Idle with original image", nodeCount)
1357+
1358+
g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).
1359+
Should(poolAllUpdated(nodeCount, env.NodeImageDigest()))
1360+
1361+
// Phase 2: Patch pool to the update image.
1362+
updateRef := env.NodeImageUpdateDigestedPullSpec()
1363+
1364+
modified := pool.DeepCopy()
1365+
modified.Spec.Image.Ref = updateRef
1366+
g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
1367+
*pool = *modified
1368+
1369+
t.Logf("Patched pool to update image %s", updateRef)
1370+
1371+
// Phase 3: Wait for every node to be Idle on the update digest - proves
1372+
// the rollout reached the full pool, not just a subset - while
1373+
// continuously enforcing maxUnavailable. Both checks share one polling
1374+
// loop so the invariant is sampled throughout the rollout rather than
1375+
// once before it starts, and so the wait ends as soon as the rollout
1376+
// does. Asserting the invariant once up front would pass trivially,
1377+
// since nothing is disrupted yet at that point.
1378+
//
1379+
// A node that stays Degraded aborts the wait early and reports the
1380+
// daemon's own error message. The daemon clears Degraded on the next
1381+
// successful reconcile, so a retried transient error must not fail the
1382+
// test; only a node degraded for longer than degradedGrace does.
1383+
const degradedGrace = 90 * time.Second
1384+
degradedSince := make(map[string]time.Time)
1385+
1386+
g.Eventually(func(g Gomega) {
1387+
// Snapshot the whole pool in a single List. The assertions below
1388+
// abort the iteration at the first node that is behind, so the
1389+
// invariant has to be evaluated across every node before any of
1390+
// them run. It must also be one List rather than a Get per node:
1391+
// a List is a consistent snapshot at one resourceVersion, whereas
1392+
// sequential Gets can straddle a slot handoff and assemble a view
1393+
// in which two nodes hold a slot, a state that never existed.
1394+
var bnList bootcv1alpha1.BootcNodeList
1395+
g.Expect(env.Client.List(ctx, &bnList)).To(Succeed())
1396+
1397+
inPool := make(map[string]bool, len(nodes))
1398+
for _, nodeName := range nodes {
1399+
inPool[nodeName] = true
1400+
}
1401+
bns := make([]bootcv1alpha1.BootcNode, 0, len(nodes))
1402+
for i := range bnList.Items {
1403+
if inPool[bnList.Items[i].Name] {
1404+
bns = append(bns, bnList.Items[i])
1405+
}
1406+
}
1407+
g.Expect(bns).To(HaveLen(len(nodes)), "expected a BootcNode per pool node")
1408+
1409+
// maxUnavailable bounds reboot slots, not staging. Staging only
1410+
// downloads the image and disrupts nothing, so the controller lets
1411+
// the whole pool stage at once and gates cordon/drain/reboot. A
1412+
// node holds a slot exactly while it carries the in-reboot-slot
1413+
// annotation, which is also how the controller counts them.
1414+
//
1415+
// Exceeding the bound is a violated invariant rather than a state
1416+
// to wait out - retrying would let a later poll mask it.
1417+
inRebootSlot := 0
1418+
for i := range bns {
1419+
if metav1.HasAnnotation(bns[i].ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) {
1420+
inRebootSlot++
1421+
}
1422+
}
1423+
if inRebootSlot > maxUnavailable {
1424+
StopTrying(fmt.Sprintf(
1425+
"maxUnavailable=%d violated: %d nodes hold a reboot slot at once",
1426+
maxUnavailable, inRebootSlot,
1427+
)).Now()
1428+
}
1429+
1430+
for i := range bns {
1431+
bn := &bns[i]
1432+
if cond := meta.FindStatusCondition(
1433+
bn.Status.Conditions, bootcv1alpha1.NodeDegraded,
1434+
); cond != nil && cond.Status == metav1.ConditionTrue {
1435+
first, seen := degradedSince[bn.Name]
1436+
switch {
1437+
case !seen:
1438+
degradedSince[bn.Name] = time.Now()
1439+
case time.Since(first) > degradedGrace:
1440+
StopTrying(fmt.Sprintf(
1441+
"node %s degraded for over %s during rollout: %s: %s",
1442+
bn.Name, degradedGrace, cond.Reason, cond.Message,
1443+
)).Now()
1444+
}
1445+
} else {
1446+
delete(degradedSince, bn.Name)
1447+
}
1448+
}
1449+
1450+
for i := range bns {
1451+
bn := &bns[i]
1452+
g.Expect(bn.Status.Conditions).To(ContainElement(And(
1453+
HaveField("Type", bootcv1alpha1.NodeIdle),
1454+
HaveField("Status", metav1.ConditionTrue),
1455+
HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
1456+
)), "node %s is not Idle yet", bn.Name)
1457+
1458+
g.Expect(bn.Status.Booted).NotTo(BeNil())
1459+
g.Expect(bn.Status.Booted).To(
1460+
imageMatchesDigest(env.NodeImageUpdateDigest()),
1461+
"node %s has not booted the update digest yet", bn.Name,
1462+
)
1463+
}
1464+
}).WithTimeout(20 * time.Minute).Should(Succeed())
1465+
1466+
t.Logf("All %d nodes are Idle with update image", nodeCount)
1467+
1468+
// Phase 4: Verify final pool status reflects all nodes updated.
1469+
g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).
1470+
Should(poolAllUpdated(nodeCount, env.NodeImageUpdateDigest()))
1471+
}

0 commit comments

Comments
 (0)