diff --git a/config/helm/aws-node-termination-handler/README.md b/config/helm/aws-node-termination-handler/README.md index 7175cfa65..2b85701d6 100644 --- a/config/helm/aws-node-termination-handler/README.md +++ b/config/helm/aws-node-termination-handler/README.md @@ -91,7 +91,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode | `webhookURLSecretName` | Pass the webhook URL as a Secret using the key `webhookurl`. | `""` | | `webhookHeaders` | Replace the default webhook headers (e.g. `{"Content-type":"application/json"}`). | `""` | | `webhookProxy` | Uses the specified HTTP(S) proxy for sending webhook data. | `""` | -| `webhookTemplate` | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`). | `""` | +| `webhookTemplate` | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`). See [Webhook Template](#webhook-template) for the available fields. | `""` | | `webhookTemplateConfigMapName` | Pass the webhook template file as a configmap. | "``" | | `webhookTemplateConfigMapKey` | Name of the Configmap key storing the template file. | `""` | | `enableSqsTerminationDraining` | If `true`, this turns on queue-processor mode which drains nodes when an SQS termination event is received. | `false` | @@ -177,6 +177,16 @@ The configuration in this table applies to AWS Node Termination Handler testing | `instanceMetadataURL` | (Used for testing) If specified, use the provided metadata URL. | `""` | | `procUptimeFile` | (Used for Testing) Specify the uptime file. | `/proc/uptime` | +## Webhook Template + +The webhook template is a Go [text/template](https://pkg.go.dev/text/template) with [Sprig](https://masterminds.github.io/sprig/) functions. Besides the interruption event fields (e.g. `.EventID`, `.Kind`, `.NodeName`, `.Pods`), it can read `.PodDetails`: one entry per pod on the node, with `Name`, `Namespace`, `Phase`, `OwnerKind` (the kind of the pod's controller, empty for a bare pod), `Labels` and `Annotations`. + +Annotations can hold sensitive data. For example, `kubectl.kubernetes.io/last-applied-configuration` contains the whole applied manifest, including inline environment variables. Select the keys you need with `index` instead of rendering a whole map with `{{ .Annotations }}` or `{{ toJson .Annotations }}`: + +```gotemplate +{"text":"Pods on {{ .NodeName }}:{{ range .PodDetails }}{{ if and (eq .Phase "Running") (ne .OwnerKind "DaemonSet") }} {{ .Namespace }}/{{ .Name }} owner={{ index .Annotations "example.com/owner" }}{{ end }}{{ end }}"} +``` + ## Metrics Endpoint Considerations AWS Node Termination Handler in IMDS mode runs as a DaemonSet with `useHostNetwork: true` by default. If the Prometheus server is enabled with `enablePrometheusServer: true` nothing else will be able to bind to the configured port (by default `prometheusServerPort: 9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint. diff --git a/pkg/interruptionevent/draincordon/handler.go b/pkg/interruptionevent/draincordon/handler.go index 6337a6690..35e0f7ebb 100644 --- a/pkg/interruptionevent/draincordon/handler.go +++ b/pkg/interruptionevent/draincordon/handler.go @@ -95,7 +95,8 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error { } } - podNameList, err := h.commonHandler.Node.FetchPodNameList(nodeName) + podDetails, err := h.commonHandler.Node.FetchPodDetails(nodeName) + podNameList := node.PodNames(podDetails) if err != nil { log.Warn(). Err(err). @@ -104,6 +105,7 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error { Msg("Failed to fetch pod names. Proceeding with fallback pod names") } else { drainEvent.Pods = podNameList + drainEvent.PodDetails = podDetails } err = h.commonHandler.Node.LogPods(podNameList, nodeName) diff --git a/pkg/interruptionevent/draincordon/handler_test.go b/pkg/interruptionevent/draincordon/handler_test.go index 2a600d3ea..39b1ac79e 100644 --- a/pkg/interruptionevent/draincordon/handler_test.go +++ b/pkg/interruptionevent/draincordon/handler_test.go @@ -72,3 +72,62 @@ func TestHandleEvent_PreDrainFailureSkipsCordonAndPostDrainTask(t *testing.T) { t.Fatal("node was cordoned after pre-drain failure") } } + +func TestHandleEvent_PopulatesPodDetails(t *testing.T) { + const nodeName = "test-node" + + client := fake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: nodeName}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-1", + Namespace: "ci", + Annotations: map[string]string{"example.com/build-url": "https://ci.example.com/job/1"}, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + ) + testNode, err := node.NewWithValues(config.Config{}, &drain.Helper{ + Ctx: context.TODO(), + Client: client, + }, uptime.Uptime) + if err != nil { + t.Fatalf("create test node: %v", err) + } + + nthConfig := config.Config{CordonOnly: true} + handler := New( + interruptioneventstore.New(nthConfig), + *testNode, + nthConfig, + ec2metadata.NodeMetadata{}, + observability.Metrics{}, + observability.K8sEventRecorder{}, + ) + drainEvent := &monitor.InterruptionEvent{ + EventID: "pod-details-event", + Kind: monitor.SpotITNKind, + NodeName: nodeName, + } + + if err := handler.HandleEvent(drainEvent); err != nil { + t.Fatalf("handle event: %v", err) + } + + if len(drainEvent.Pods) != 1 || drainEvent.Pods[0] != "agent-1" { + t.Fatalf("expected Pods [agent-1], got %v", drainEvent.Pods) + } + if len(drainEvent.PodDetails) != 1 { + t.Fatalf("expected 1 pod detail, got %d", len(drainEvent.PodDetails)) + } + pod := drainEvent.PodDetails[0] + if pod.Namespace != "ci" || pod.Phase != "Running" || pod.OwnerKind != "" { + t.Fatalf("unexpected pod detail: %+v", pod) + } + if got := pod.Annotations["example.com/build-url"]; got != "https://ci.example.com/job/1" { + t.Fatalf("expected build-url annotation, got %q", got) + } +} diff --git a/pkg/monitor/types.go b/pkg/monitor/types.go index 522ba5eb1..be08799fe 100644 --- a/pkg/monitor/types.go +++ b/pkg/monitor/types.go @@ -51,6 +51,7 @@ type InterruptionEvent struct { NodeName string NodeLabels map[string]string Pods []string + PodDetails []node.PodDetail `json:"-"` InstanceID string ProviderID string InstanceType string diff --git a/pkg/monitor/types_test.go b/pkg/monitor/types_test.go index 5e4373676..eb5a5c05c 100644 --- a/pkg/monitor/types_test.go +++ b/pkg/monitor/types_test.go @@ -14,10 +14,13 @@ package monitor_test import ( + "encoding/json" + "strings" "testing" "time" "github.com/aws/aws-node-termination-handler/pkg/monitor" + "github.com/aws/aws-node-termination-handler/pkg/node" h "github.com/aws/aws-node-termination-handler/pkg/test" ) @@ -64,3 +67,17 @@ func TestIsRebalanceRecommendation_Empty_Failure(t *testing.T) { event := &monitor.InterruptionEvent{} h.Equals(t, false, event.IsRebalanceRecommendation()) } + +func TestInterruptionEventJSONOmitsPodDetails(t *testing.T) { + event := monitor.InterruptionEvent{ + Pods: []string{"agent-1"}, + PodDetails: []node.PodDetail{ + {Name: "agent-1", Annotations: map[string]string{"example.com/secret": "value"}}, + }, + } + + eventJSON, err := json.Marshal(event) + h.Ok(t, err) + h.Assert(t, !strings.Contains(string(eventJSON), "PodDetails"), "PodDetails should not be serialized") + h.Assert(t, !strings.Contains(string(eventJSON), "example.com/secret"), "pod annotations should not be serialized") +} diff --git a/pkg/node/node.go b/pkg/node/node.go index 4918767bb..7042a2a41 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -89,6 +89,18 @@ type Node struct { uptime uptime.UptimeFuncType } +// PodDetail holds details of a pod on a node, as exposed to the webhook template +type PodDetail struct { + Name string + Namespace string + Phase string + // OwnerKind is the kind of the pod's controller (e.g. ReplicaSet, DaemonSet, Job), or "" for a bare pod + OwnerKind string + Labels map[string]string + // Annotations can hold sensitive data; templates should select specific keys rather than render the whole map + Annotations map[string]string +} + type ZerologWriter struct { logger zerolog.Logger } @@ -515,17 +527,37 @@ func (n Node) LogPods(podList []string, nodeName string) error { return nil } -// FetchPodNameList fetches list of all the pods names running on given nodeName -func (n Node) FetchPodNameList(nodeName string) ([]string, error) { +// FetchPodDetails fetches details of all the pods on given nodeName +func (n Node) FetchPodDetails(nodeName string) ([]PodDetail, error) { podList, err := n.fetchAllPods(nodeName) if err != nil { return nil, err } - var podNamesList []string + var podDetails []PodDetail for _, pod := range podList.Items { - podNamesList = append(podNamesList, pod.Name) + ownerKind := "" + if owner := metav1.GetControllerOf(&pod); owner != nil { + ownerKind = owner.Kind + } + podDetails = append(podDetails, PodDetail{ + Name: pod.Name, + Namespace: pod.Namespace, + Phase: string(pod.Status.Phase), + OwnerKind: ownerKind, + Labels: pod.Labels, + Annotations: pod.Annotations, + }) + } + return podDetails, nil +} + +// PodNames returns the names of the given pods +func PodNames(podDetails []PodDetail) []string { + var podNames []string + for _, pod := range podDetails { + podNames = append(podNames, pod.Name) } - return podNamesList, nil + return podNames } // TaintScheduledMaintenance adds the scheduled maintenance taint onto a node diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index bb07f4b16..5ac03d2ec 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -16,6 +16,7 @@ package node_test import ( "context" "fmt" + "sort" "strconv" "strings" "testing" @@ -291,27 +292,60 @@ func TestMarkForUncordonAfterRebootAddActionLabelFailure(t *testing.T) { h.Assert(t, err != nil, "Failed to return error on MarkForUncordonAfterReboot failing to add action Label") } -func TestFetchPodsNameList(t *testing.T) { +func TestFetchPodDetails(t *testing.T) { + isController := true client := fake.NewSimpleClientset( &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "myPod", - Labels: map[string]string{ - "spec.nodeName": nodeName, + Name: "web-1", + Namespace: "team-a", + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"example.com/owner": "team-a"}, + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicaSet", Name: "web", Controller: &isController}, }, }, + Spec: v1.PodSpec{NodeName: nodeName}, + Status: v1.PodStatus{Phase: v1.PodRunning}, }, - &v1.Node{ + &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, + Name: "agent-1", + Namespace: "ci", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ConfigMap", Name: "not-a-controller"}, + }, }, + Spec: v1.PodSpec{NodeName: nodeName}, + Status: v1.PodStatus{Phase: v1.PodSucceeded}, }, ) tNode := getNode(t, getDrainHelper(client)) - podList, err := tNode.FetchPodNameList(nodeName) - h.Ok(t, err) - h.Equals(t, []string{"myPod"}, podList) + podDetails, err := tNode.FetchPodDetails(nodeName) + h.Ok(t, err) + sort.Slice(podDetails, func(i, j int) bool { return podDetails[i].Name < podDetails[j].Name }) + h.Equals(t, []node.PodDetail{ + { + Name: "agent-1", + Namespace: "ci", + Phase: "Succeeded", + OwnerKind: "", + }, + { + Name: "web-1", + Namespace: "team-a", + Phase: "Running", + OwnerKind: "ReplicaSet", + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"example.com/owner": "team-a"}, + }, + }, podDetails) +} + +func TestPodNames(t *testing.T) { + h.Equals(t, []string(nil), node.PodNames(nil)) + h.Equals(t, []string{"a", "b"}, node.PodNames([]node.PodDetail{{Name: "a"}, {Name: "b"}})) } func TestLogPods(t *testing.T) { diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go index 87cb50199..62e303324 100644 --- a/pkg/webhook/webhook_test.go +++ b/pkg/webhook/webhook_test.go @@ -28,6 +28,7 @@ import ( "github.com/aws/aws-node-termination-handler/pkg/config" "github.com/aws/aws-node-termination-handler/pkg/ec2metadata" "github.com/aws/aws-node-termination-handler/pkg/monitor" + "github.com/aws/aws-node-termination-handler/pkg/node" h "github.com/aws/aws-node-termination-handler/pkg/test" "github.com/aws/aws-node-termination-handler/pkg/webhook" "github.com/rs/zerolog/log" @@ -114,6 +115,49 @@ func TestPostSuccess(t *testing.T) { webhook.Post(nodeMetadata, event, nthconfig) } +func TestPostPodDetails(t *testing.T) { + const podDetailsTemplate = `{{ range .PodDetails }}{{ if and (eq .Phase "Running") (ne .OwnerKind "DaemonSet") }}{{ .Namespace }}/{{ .Name }} {{ index .Labels "app" }} {{ index .Annotations "example.com/owner" }};{{ end }}{{ end }}` + + event := &monitor.InterruptionEvent{ + Pods: []string{"web-1", "ds-1", "done-1"}, + PodDetails: []node.PodDetail{ + { + Name: "web-1", + Namespace: "team-a", + Phase: "Running", + OwnerKind: "ReplicaSet", + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"example.com/owner": "alice"}, + }, + {Name: "ds-1", Namespace: "kube-system", Phase: "Running", OwnerKind: "DaemonSet"}, + {Name: "done-1", Namespace: "ci", Phase: "Succeeded"}, + }, + } + + requestBody := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + h.Ok(t, err) + requestBody <- string(body) + })) + defer server.Close() + + nthconfig := config.Config{ + WebhookURL: server.URL, + WebhookHeaders: testWebhookHeaders, + WebhookTemplate: podDetailsTemplate, + } + + webhook.Post(ec2metadata.NodeMetadata{}, event, nthconfig) + + select { + case body := <-requestBody: + h.Equals(t, "team-a/web-1 web alice;", body) + default: + t.Fatal("webhook was not posted") + } +} + func TestPostTemplateParseError(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { t.Error("Request made with invalid webhook") diff --git a/test/e2e/webhook-test b/test/e2e/webhook-test index 298ade73f..04a5648aa 100755 --- a/test/e2e/webhook-test +++ b/test/e2e/webhook-test @@ -74,7 +74,7 @@ anth_helm_args=( --set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" --set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" --set webhookURL="${WEBHOOK_URL}" - --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} - AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\}\"\}" + --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} - AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\} - Pods: \{\{ range \.PodDetails \}\}\{\{ \.Namespace \}\}/\{\{ \.Name \}\} \{\{ \.Phase \}\}/\{\{ \.OwnerKind \}\} \{\{ end \}\}\"\}" --set enableSpotInterruptionDraining="true" --set enableScheduledEventDraining="true" --set daemonsetTolerations=""