From 24f680257b5aec15cbcf0e8c68c2ea25658f21b3 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Thu, 29 Oct 2026 14:02:06 +0800 Subject: [PATCH 1/2] Expose pod details to the webhook template Add PodDetails to the webhook template data: namespace, phase, controller kind, labels and annotations for each pod on the node. It is filled from the same pod List call as Pods, so no extra API request is made. Pods and the default template are unchanged. PodDetails is excluded from JSON so pod annotations do not end up in the event logs. --- pkg/interruptionevent/draincordon/handler.go | 4 +- .../draincordon/handler_test.go | 59 +++++++++++++++++++ pkg/monitor/types.go | 1 + pkg/monitor/types_test.go | 17 ++++++ pkg/node/node.go | 44 ++++++++++++++ pkg/node/node_test.go | 57 ++++++++++++++++++ pkg/webhook/webhook_test.go | 44 ++++++++++++++ test/e2e/webhook-test | 2 +- 8 files changed, 226 insertions(+), 2 deletions(-) diff --git a/pkg/interruptionevent/draincordon/handler.go b/pkg/interruptionevent/draincordon/handler.go index 6337a6690..35e0f7ebb 100644 --- a/pkg/interruptionevent/draincordon/handler.go +++ b/pkg/interruptionevent/draincordon/handler.go @@ -95,7 +95,8 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error { } } - podNameList, err := h.commonHandler.Node.FetchPodNameList(nodeName) + podDetails, err := h.commonHandler.Node.FetchPodDetails(nodeName) + podNameList := node.PodNames(podDetails) if err != nil { log.Warn(). Err(err). @@ -104,6 +105,7 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error { Msg("Failed to fetch pod names. Proceeding with fallback pod names") } else { drainEvent.Pods = podNameList + drainEvent.PodDetails = podDetails } err = h.commonHandler.Node.LogPods(podNameList, nodeName) diff --git a/pkg/interruptionevent/draincordon/handler_test.go b/pkg/interruptionevent/draincordon/handler_test.go index 2a600d3ea..39b1ac79e 100644 --- a/pkg/interruptionevent/draincordon/handler_test.go +++ b/pkg/interruptionevent/draincordon/handler_test.go @@ -72,3 +72,62 @@ func TestHandleEvent_PreDrainFailureSkipsCordonAndPostDrainTask(t *testing.T) { t.Fatal("node was cordoned after pre-drain failure") } } + +func TestHandleEvent_PopulatesPodDetails(t *testing.T) { + const nodeName = "test-node" + + client := fake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: nodeName}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-1", + Namespace: "ci", + Annotations: map[string]string{"example.com/build-url": "https://ci.example.com/job/1"}, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + ) + testNode, err := node.NewWithValues(config.Config{}, &drain.Helper{ + Ctx: context.TODO(), + Client: client, + }, uptime.Uptime) + if err != nil { + t.Fatalf("create test node: %v", err) + } + + nthConfig := config.Config{CordonOnly: true} + handler := New( + interruptioneventstore.New(nthConfig), + *testNode, + nthConfig, + ec2metadata.NodeMetadata{}, + observability.Metrics{}, + observability.K8sEventRecorder{}, + ) + drainEvent := &monitor.InterruptionEvent{ + EventID: "pod-details-event", + Kind: monitor.SpotITNKind, + NodeName: nodeName, + } + + if err := handler.HandleEvent(drainEvent); err != nil { + t.Fatalf("handle event: %v", err) + } + + if len(drainEvent.Pods) != 1 || drainEvent.Pods[0] != "agent-1" { + t.Fatalf("expected Pods [agent-1], got %v", drainEvent.Pods) + } + if len(drainEvent.PodDetails) != 1 { + t.Fatalf("expected 1 pod detail, got %d", len(drainEvent.PodDetails)) + } + pod := drainEvent.PodDetails[0] + if pod.Namespace != "ci" || pod.Phase != "Running" || pod.OwnerKind != "" { + t.Fatalf("unexpected pod detail: %+v", pod) + } + if got := pod.Annotations["example.com/build-url"]; got != "https://ci.example.com/job/1" { + t.Fatalf("expected build-url annotation, got %q", got) + } +} diff --git a/pkg/monitor/types.go b/pkg/monitor/types.go index 522ba5eb1..be08799fe 100644 --- a/pkg/monitor/types.go +++ b/pkg/monitor/types.go @@ -51,6 +51,7 @@ type InterruptionEvent struct { NodeName string NodeLabels map[string]string Pods []string + PodDetails []node.PodDetail `json:"-"` InstanceID string ProviderID string InstanceType string diff --git a/pkg/monitor/types_test.go b/pkg/monitor/types_test.go index 5e4373676..eb5a5c05c 100644 --- a/pkg/monitor/types_test.go +++ b/pkg/monitor/types_test.go @@ -14,10 +14,13 @@ package monitor_test import ( + "encoding/json" + "strings" "testing" "time" "github.com/aws/aws-node-termination-handler/pkg/monitor" + "github.com/aws/aws-node-termination-handler/pkg/node" h "github.com/aws/aws-node-termination-handler/pkg/test" ) @@ -64,3 +67,17 @@ func TestIsRebalanceRecommendation_Empty_Failure(t *testing.T) { event := &monitor.InterruptionEvent{} h.Equals(t, false, event.IsRebalanceRecommendation()) } + +func TestInterruptionEventJSONOmitsPodDetails(t *testing.T) { + event := monitor.InterruptionEvent{ + Pods: []string{"agent-1"}, + PodDetails: []node.PodDetail{ + {Name: "agent-1", Annotations: map[string]string{"example.com/secret": "value"}}, + }, + } + + eventJSON, err := json.Marshal(event) + h.Ok(t, err) + h.Assert(t, !strings.Contains(string(eventJSON), "PodDetails"), "PodDetails should not be serialized") + h.Assert(t, !strings.Contains(string(eventJSON), "example.com/secret"), "pod annotations should not be serialized") +} diff --git a/pkg/node/node.go b/pkg/node/node.go index 4918767bb..aeaf61123 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -89,6 +89,17 @@ type Node struct { uptime uptime.UptimeFuncType } +// PodDetail holds details of a pod on a node, as exposed to the webhook template +type PodDetail struct { + Name string + Namespace string + Phase string + // OwnerKind is the kind of the pod's controller (e.g. ReplicaSet, DaemonSet, Job), or "" for a bare pod + OwnerKind string + Labels map[string]string + Annotations map[string]string +} + type ZerologWriter struct { logger zerolog.Logger } @@ -528,6 +539,39 @@ func (n Node) FetchPodNameList(nodeName string) ([]string, error) { return podNamesList, nil } +// FetchPodDetails fetches details of all the pods on given nodeName +func (n Node) FetchPodDetails(nodeName string) ([]PodDetail, error) { + podList, err := n.fetchAllPods(nodeName) + if err != nil { + return nil, err + } + var podDetails []PodDetail + for _, pod := range podList.Items { + ownerKind := "" + if owner := metav1.GetControllerOf(&pod); owner != nil { + ownerKind = owner.Kind + } + podDetails = append(podDetails, PodDetail{ + Name: pod.Name, + Namespace: pod.Namespace, + Phase: string(pod.Status.Phase), + OwnerKind: ownerKind, + Labels: pod.Labels, + Annotations: pod.Annotations, + }) + } + return podDetails, nil +} + +// PodNames returns the names of the given pods +func PodNames(podDetails []PodDetail) []string { + var podNames []string + for _, pod := range podDetails { + podNames = append(podNames, pod.Name) + } + return podNames +} + // TaintScheduledMaintenance adds the scheduled maintenance taint onto a node func (n Node) TaintScheduledMaintenance(nodeName string, eventID string) error { if !n.nthConfig.TaintNode { diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index bb07f4b16..bd6a97e13 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -16,6 +16,7 @@ package node_test import ( "context" "fmt" + "sort" "strconv" "strings" "testing" @@ -314,6 +315,62 @@ func TestFetchPodsNameList(t *testing.T) { h.Equals(t, []string{"myPod"}, podList) } +func TestFetchPodDetails(t *testing.T) { + isController := true + client := fake.NewSimpleClientset( + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "web-1", + Namespace: "team-a", + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"example.com/owner": "team-a"}, + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicaSet", Name: "web", Controller: &isController}, + }, + }, + Spec: v1.PodSpec{NodeName: nodeName}, + Status: v1.PodStatus{Phase: v1.PodRunning}, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-1", + Namespace: "ci", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ConfigMap", Name: "not-a-controller"}, + }, + }, + Spec: v1.PodSpec{NodeName: nodeName}, + Status: v1.PodStatus{Phase: v1.PodSucceeded}, + }, + ) + + tNode := getNode(t, getDrainHelper(client)) + podDetails, err := tNode.FetchPodDetails(nodeName) + h.Ok(t, err) + sort.Slice(podDetails, func(i, j int) bool { return podDetails[i].Name < podDetails[j].Name }) + h.Equals(t, []node.PodDetail{ + { + Name: "agent-1", + Namespace: "ci", + Phase: "Succeeded", + OwnerKind: "", + }, + { + Name: "web-1", + Namespace: "team-a", + Phase: "Running", + OwnerKind: "ReplicaSet", + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"example.com/owner": "team-a"}, + }, + }, podDetails) +} + +func TestPodNames(t *testing.T) { + h.Equals(t, []string(nil), node.PodNames(nil)) + h.Equals(t, []string{"a", "b"}, node.PodNames([]node.PodDetail{{Name: "a"}, {Name: "b"}})) +} + func TestLogPods(t *testing.T) { client := fake.NewSimpleClientset( &v1.Pod{ diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go index 87cb50199..62e303324 100644 --- a/pkg/webhook/webhook_test.go +++ b/pkg/webhook/webhook_test.go @@ -28,6 +28,7 @@ import ( "github.com/aws/aws-node-termination-handler/pkg/config" "github.com/aws/aws-node-termination-handler/pkg/ec2metadata" "github.com/aws/aws-node-termination-handler/pkg/monitor" + "github.com/aws/aws-node-termination-handler/pkg/node" h "github.com/aws/aws-node-termination-handler/pkg/test" "github.com/aws/aws-node-termination-handler/pkg/webhook" "github.com/rs/zerolog/log" @@ -114,6 +115,49 @@ func TestPostSuccess(t *testing.T) { webhook.Post(nodeMetadata, event, nthconfig) } +func TestPostPodDetails(t *testing.T) { + const podDetailsTemplate = `{{ range .PodDetails }}{{ if and (eq .Phase "Running") (ne .OwnerKind "DaemonSet") }}{{ .Namespace }}/{{ .Name }} {{ index .Labels "app" }} {{ index .Annotations "example.com/owner" }};{{ end }}{{ end }}` + + event := &monitor.InterruptionEvent{ + Pods: []string{"web-1", "ds-1", "done-1"}, + PodDetails: []node.PodDetail{ + { + Name: "web-1", + Namespace: "team-a", + Phase: "Running", + OwnerKind: "ReplicaSet", + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"example.com/owner": "alice"}, + }, + {Name: "ds-1", Namespace: "kube-system", Phase: "Running", OwnerKind: "DaemonSet"}, + {Name: "done-1", Namespace: "ci", Phase: "Succeeded"}, + }, + } + + requestBody := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + h.Ok(t, err) + requestBody <- string(body) + })) + defer server.Close() + + nthconfig := config.Config{ + WebhookURL: server.URL, + WebhookHeaders: testWebhookHeaders, + WebhookTemplate: podDetailsTemplate, + } + + webhook.Post(ec2metadata.NodeMetadata{}, event, nthconfig) + + select { + case body := <-requestBody: + h.Equals(t, "team-a/web-1 web alice;", body) + default: + t.Fatal("webhook was not posted") + } +} + func TestPostTemplateParseError(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { t.Error("Request made with invalid webhook") diff --git a/test/e2e/webhook-test b/test/e2e/webhook-test index 298ade73f..04a5648aa 100755 --- a/test/e2e/webhook-test +++ b/test/e2e/webhook-test @@ -74,7 +74,7 @@ anth_helm_args=( --set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" --set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" --set webhookURL="${WEBHOOK_URL}" - --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} - AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\}\"\}" + --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} - AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\} - Pods: \{\{ range \.PodDetails \}\}\{\{ \.Namespace \}\}/\{\{ \.Name \}\} \{\{ \.Phase \}\}/\{\{ \.OwnerKind \}\} \{\{ end \}\}\"\}" --set enableSpotInterruptionDraining="true" --set enableScheduledEventDraining="true" --set daemonsetTolerations="" From 7bb07c58e4af1674d6887c96dc1cfb693cfbfa95 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Thu, 12 Nov 2026 18:27:24 +0800 Subject: [PATCH 2/2] Document webhook pod details and remove FetchPodNameList Add a Webhook Template section to the chart README listing the .PodDetails fields, with a caveat to select specific annotation keys rather than rendering whole maps, since annotations such as last-applied-configuration can hold sensitive data. Remove FetchPodNameList, which no longer has a caller; the handler derives pod names from FetchPodDetails. --- .../aws-node-termination-handler/README.md | 12 +++++++++- pkg/node/node.go | 18 +++------------ pkg/node/node_test.go | 23 ------------------- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/config/helm/aws-node-termination-handler/README.md b/config/helm/aws-node-termination-handler/README.md index 7175cfa65..2b85701d6 100644 --- a/config/helm/aws-node-termination-handler/README.md +++ b/config/helm/aws-node-termination-handler/README.md @@ -91,7 +91,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode | `webhookURLSecretName` | Pass the webhook URL as a Secret using the key `webhookurl`. | `""` | | `webhookHeaders` | Replace the default webhook headers (e.g. `{"Content-type":"application/json"}`). | `""` | | `webhookProxy` | Uses the specified HTTP(S) proxy for sending webhook data. | `""` | -| `webhookTemplate` | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`). | `""` | +| `webhookTemplate` | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`). See [Webhook Template](#webhook-template) for the available fields. | `""` | | `webhookTemplateConfigMapName` | Pass the webhook template file as a configmap. | "``" | | `webhookTemplateConfigMapKey` | Name of the Configmap key storing the template file. | `""` | | `enableSqsTerminationDraining` | If `true`, this turns on queue-processor mode which drains nodes when an SQS termination event is received. | `false` | @@ -177,6 +177,16 @@ The configuration in this table applies to AWS Node Termination Handler testing | `instanceMetadataURL` | (Used for testing) If specified, use the provided metadata URL. | `""` | | `procUptimeFile` | (Used for Testing) Specify the uptime file. | `/proc/uptime` | +## Webhook Template + +The webhook template is a Go [text/template](https://pkg.go.dev/text/template) with [Sprig](https://masterminds.github.io/sprig/) functions. Besides the interruption event fields (e.g. `.EventID`, `.Kind`, `.NodeName`, `.Pods`), it can read `.PodDetails`: one entry per pod on the node, with `Name`, `Namespace`, `Phase`, `OwnerKind` (the kind of the pod's controller, empty for a bare pod), `Labels` and `Annotations`. + +Annotations can hold sensitive data. For example, `kubectl.kubernetes.io/last-applied-configuration` contains the whole applied manifest, including inline environment variables. Select the keys you need with `index` instead of rendering a whole map with `{{ .Annotations }}` or `{{ toJson .Annotations }}`: + +```gotemplate +{"text":"Pods on {{ .NodeName }}:{{ range .PodDetails }}{{ if and (eq .Phase "Running") (ne .OwnerKind "DaemonSet") }} {{ .Namespace }}/{{ .Name }} owner={{ index .Annotations "example.com/owner" }}{{ end }}{{ end }}"} +``` + ## Metrics Endpoint Considerations AWS Node Termination Handler in IMDS mode runs as a DaemonSet with `useHostNetwork: true` by default. If the Prometheus server is enabled with `enablePrometheusServer: true` nothing else will be able to bind to the configured port (by default `prometheusServerPort: 9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint. diff --git a/pkg/node/node.go b/pkg/node/node.go index aeaf61123..7042a2a41 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -95,8 +95,9 @@ type PodDetail struct { Namespace string Phase string // OwnerKind is the kind of the pod's controller (e.g. ReplicaSet, DaemonSet, Job), or "" for a bare pod - OwnerKind string - Labels map[string]string + OwnerKind string + Labels map[string]string + // Annotations can hold sensitive data; templates should select specific keys rather than render the whole map Annotations map[string]string } @@ -526,19 +527,6 @@ func (n Node) LogPods(podList []string, nodeName string) error { return nil } -// FetchPodNameList fetches list of all the pods names running on given nodeName -func (n Node) FetchPodNameList(nodeName string) ([]string, error) { - podList, err := n.fetchAllPods(nodeName) - if err != nil { - return nil, err - } - var podNamesList []string - for _, pod := range podList.Items { - podNamesList = append(podNamesList, pod.Name) - } - return podNamesList, nil -} - // FetchPodDetails fetches details of all the pods on given nodeName func (n Node) FetchPodDetails(nodeName string) ([]PodDetail, error) { podList, err := n.fetchAllPods(nodeName) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index bd6a97e13..5ac03d2ec 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -292,29 +292,6 @@ func TestMarkForUncordonAfterRebootAddActionLabelFailure(t *testing.T) { h.Assert(t, err != nil, "Failed to return error on MarkForUncordonAfterReboot failing to add action Label") } -func TestFetchPodsNameList(t *testing.T) { - client := fake.NewSimpleClientset( - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "myPod", - Labels: map[string]string{ - "spec.nodeName": nodeName, - }, - }, - }, - &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - }, - ) - - tNode := getNode(t, getDrainHelper(client)) - podList, err := tNode.FetchPodNameList(nodeName) - h.Ok(t, err) - h.Equals(t, []string{"myPod"}, podList) -} - func TestFetchPodDetails(t *testing.T) { isController := true client := fake.NewSimpleClientset(