makes bootc-operator aware of karpenter

If a cluster contains both bootc-operator and Karpenter, it is possible
that Karpenter could reap cluster nodes while in the middle of a
bootc-operator driven update. To prevent that, we apply the Karpenter Do
Not Repair annotation to each node when it is cordoned for an update and
we remove it after the update is completed.

Signed-off-by: Zack Zlotnik <zzlotnik@redhat.com>

Author: cheesesashimi

internal/controller/bootcnodepool_controller.go

@@ -886,6 +886,11 @@ func (r *BootcNodePoolReconciler) restoreCordonState(
 	}
 	modified := node.DeepCopy()
 	modified.Spec.Unschedulable = false
+
+	if metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey) {
+		delete(modified.Annotations, karpenterDoNotRepairAnnotationKey)
+	}
+
 	if err := r.Patch(ctx, modified, client.StrategicMergeFrom(node)); err != nil {
 		return fmt.Errorf("uncordoning node: %w", err)
 	}

internal/controller/rollout.go

@@ -34,6 +34,15 @@ import (
 // https://github.com/bootc-dev/bootc-operator/issues/99.
 const unhealthySlotHaltThreshold = 2
 
+// Ensures that clusters using Karpenter will not have their nodes reaped when
+// a bootc-operator driven update / reboot is in progress.
+//
+// See:
+// - https://redhat.atlassian.net/browse/BIFROST-1447
+// - https://redhat.atlassian.net/browse/MCO-2599
+// - https://github.com/kubernetes-sigs/karpenter/pull/3311
+const karpenterDoNotRepairAnnotationKey = "karpenter.sh/do-not-repair"
+
 // rolloutState holds the classified BootcNodes for a single reconcile
 // pass.
 type rolloutState struct {
@@ -225,6 +234,11 @@ func (r *BootcNodePoolReconciler) assignRebootSlot(
 		log.Info("Cordoning node", "node", node.Name)
 		modifiedNode := node.DeepCopy()
 		modifiedNode.Spec.Unschedulable = true
+		metav1.SetMetaDataAnnotation(
+			&modifiedNode.ObjectMeta,
+			karpenterDoNotRepairAnnotationKey,
+			"true",
+		)
 		if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil {
 			return fmt.Errorf("cordoning node: %w", err)
 		}

internal/controller/rollout_envtest_test.go

@@ -100,12 +100,21 @@ func TestSimpleRollout(t *testing.T) {
 			return node.Spec.Unschedulable, err
 		}).Should(BeTrue(), "node %s should be cordoned", name)
 
+		g.Eventually(func() (bool, error) {
+			var node corev1.Node
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: name}, &node)
+			return metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey), err
+		}).Should(BeTrue(), "node %s should get karpenter do-not-repair annotation", name)
+
 		// Verify remaining nodes are not yet touched.
 		for _, other := range nodeNames[i+1:] {
 			var node corev1.Node
 			g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: other}, &node)).To(Succeed())
 			g.Expect(node.Spec.Unschedulable).To(BeFalse(),
 				"node %s should not be cordoned", other)
+			g.Expect(metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey)).
+				To(BeFalse(),
+					"node %s should not have the karpenter do-not-repair annotation", other)
 
 			var bn bootcv1alpha1.BootcNode
 			g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: other}, &bn)).To(Succeed())
@@ -134,6 +143,12 @@ func TestSimpleRollout(t *testing.T) {
 			err := k8sClient.Get(ctx, client.ObjectKey{Name: name}, &node)
 			return node.Spec.Unschedulable, err
 		}).Should(BeFalse(), "node %s should be uncordoned after reboot", name)
+
+		g.Eventually(func() (bool, error) {
+			var node corev1.Node
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: name}, &node)
+			return metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey), err
+		}).Should(BeFalse(), "node %s should not have karpenter do-not-repair annotation after reboot", name)
 	}
 }
 
@@ -661,6 +676,12 @@ func TestNodeLeavesPoolCancelsDrain(t *testing.T) {
 		return n.Spec.Unschedulable, err
 	}).Should(BeTrue())
 
+	g.Eventually(func() (bool, error) {
+		var n corev1.Node
+		err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n)
+		return metav1.HasAnnotation(n.ObjectMeta, karpenterDoNotRepairAnnotationKey), err
+	}).Should(BeTrue())
+
 	// Verify desiredImageState is still Staged (drain hasn't completed).
 	var bn bootcv1alpha1.BootcNode
 	g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
@@ -686,6 +707,12 @@ func TestNodeLeavesPoolCancelsDrain(t *testing.T) {
 		return n.Spec.Unschedulable, err
 	}).Should(BeFalse())
 
+	g.Eventually(func() (bool, error) {
+		var n corev1.Node
+		err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n)
+		return metav1.HasAnnotation(n.ObjectMeta, karpenterDoNotRepairAnnotationKey), err
+	}).Should(BeFalse())
+
 	g.Eventually(func() (map[string]string, error) {
 		var n corev1.Node
 		err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n)