Expose pod details to the webhook template

config/helm/aws-node-termination-handler/README.md

@@ -91,7 +91,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode
 | `webhookURLSecretName`             | Pass the webhook URL as a Secret using the key `webhookurl`.                                                                                                                                                                                                                                                                                                                           | `""`                                                  |
 | `webhookHeaders`                   | Replace the default webhook headers (e.g. `{"Content-type":"application/json"}`).                                                                                                                                                                                                                                                                                                      | `""`                                                  |
 | `webhookProxy`                     | Uses the specified HTTP(S) proxy for sending webhook data.                                                                                                                                                                                                                                                                                                                             | `""`                                                  |
-| `webhookTemplate`                  | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`).                                                                                                                     | `""`                                                  |
+| `webhookTemplate`                  | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`). See [Webhook Template](#webhook-template) for the available fields.                                                 | `""`                                                  |
 | `webhookTemplateConfigMapName`     | Pass the webhook template file as a configmap.                                                                                                                                                                                                                                                                                                                                         | "``"                                                  |
 | `webhookTemplateConfigMapKey`      | Name of the Configmap key storing the template file.                                                                                                                                                                                                                                                                                                                                   | `""`                                                  |
 | `enableSqsTerminationDraining`     | If `true`, this turns on queue-processor mode which drains nodes when an SQS termination event is received.                                                                                                                                                                                                                                                                            | `false`                                               |
@@ -177,6 +177,16 @@ The configuration in this table applies to AWS Node Termination Handler testing
 | `instanceMetadataURL` | (Used for testing) If specified, use the provided metadata URL.                   | `""`           |
 | `procUptimeFile`      | (Used for Testing) Specify the uptime file.                                       | `/proc/uptime` |
 
+## Webhook Template
+
+The webhook template is a Go [text/template](https://pkg.go.dev/text/template) with [Sprig](https://masterminds.github.io/sprig/) functions. Besides the interruption event fields (e.g. `.EventID`, `.Kind`, `.NodeName`, `.Pods`), it can read `.PodDetails`: one entry per pod on the node, with `Name`, `Namespace`, `Phase`, `OwnerKind` (the kind of the pod's controller, empty for a bare pod), `Labels` and `Annotations`.
+
+Annotations can hold sensitive data. For example, `kubectl.kubernetes.io/last-applied-configuration` contains the whole applied manifest, including inline environment variables. Select the keys you need with `index` instead of rendering a whole map with `{{ .Annotations }}` or `{{ toJson .Annotations }}`:
+
+```gotemplate
+{"text":"Pods on {{ .NodeName }}:{{ range .PodDetails }}{{ if and (eq .Phase "Running") (ne .OwnerKind "DaemonSet") }} {{ .Namespace }}/{{ .Name }} owner={{ index .Annotations "example.com/owner" }}{{ end }}{{ end }}"}
+```
+
 ## Metrics Endpoint Considerations
 
 AWS Node Termination Handler in IMDS mode runs as a DaemonSet with `useHostNetwork: true` by default. If the Prometheus server is enabled with `enablePrometheusServer: true` nothing else will be able to bind to the configured port (by default `prometheusServerPort: 9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint.

pkg/interruptionevent/draincordon/handler.go

@@ -95,7 +95,8 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error {
 		}
 	}
 
-	podNameList, err := h.commonHandler.Node.FetchPodNameList(nodeName)
+	podDetails, err := h.commonHandler.Node.FetchPodDetails(nodeName)
+	podNameList := node.PodNames(podDetails)
 	if err != nil {
 		log.Warn().
 			Err(err).
@@ -104,6 +105,7 @@ func (h *Handler) HandleEvent(drainEvent *monitor.InterruptionEvent) error {
 			Msg("Failed to fetch pod names. Proceeding with fallback pod names")
 	} else {
 		drainEvent.Pods = podNameList
+		drainEvent.PodDetails = podDetails
 	}
 
 	err = h.commonHandler.Node.LogPods(podNameList, nodeName)

pkg/interruptionevent/draincordon/handler_test.go

@@ -72,3 +72,62 @@ func TestHandleEvent_PreDrainFailureSkipsCordonAndPostDrainTask(t *testing.T) {
 		t.Fatal("node was cordoned after pre-drain failure")
 	}
 }
+
+func TestHandleEvent_PopulatesPodDetails(t *testing.T) {
+	const nodeName = "test-node"
+
+	client := fake.NewSimpleClientset(
+		&corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{Name: nodeName},
+		},
+		&corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        "agent-1",
+				Namespace:   "ci",
+				Annotations: map[string]string{"example.com/build-url": "https://ci.example.com/job/1"},
+			},
+			Spec:   corev1.PodSpec{NodeName: nodeName},
+			Status: corev1.PodStatus{Phase: corev1.PodRunning},
+		},
+	)
+	testNode, err := node.NewWithValues(config.Config{}, &drain.Helper{
+		Ctx:    context.TODO(),
+		Client: client,
+	}, uptime.Uptime)
+	if err != nil {
+		t.Fatalf("create test node: %v", err)
+	}
+
+	nthConfig := config.Config{CordonOnly: true}
+	handler := New(
+		interruptioneventstore.New(nthConfig),
+		*testNode,
+		nthConfig,
+		ec2metadata.NodeMetadata{},
+		observability.Metrics{},
+		observability.K8sEventRecorder{},
+	)
+	drainEvent := &monitor.InterruptionEvent{
+		EventID:  "pod-details-event",
+		Kind:     monitor.SpotITNKind,
+		NodeName: nodeName,
+	}
+
+	if err := handler.HandleEvent(drainEvent); err != nil {
+		t.Fatalf("handle event: %v", err)
+	}
+
+	if len(drainEvent.Pods) != 1 || drainEvent.Pods[0] != "agent-1" {
+		t.Fatalf("expected Pods [agent-1], got %v", drainEvent.Pods)
+	}
+	if len(drainEvent.PodDetails) != 1 {
+		t.Fatalf("expected 1 pod detail, got %d", len(drainEvent.PodDetails))
+	}
+	pod := drainEvent.PodDetails[0]
+	if pod.Namespace != "ci" || pod.Phase != "Running" || pod.OwnerKind != "" {
+		t.Fatalf("unexpected pod detail: %+v", pod)
+	}
+	if got := pod.Annotations["example.com/build-url"]; got != "https://ci.example.com/job/1" {
+		t.Fatalf("expected build-url annotation, got %q", got)
+	}
+}

pkg/monitor/types.go

@@ -51,6 +51,7 @@ type InterruptionEvent struct {
 	NodeName             string
 	NodeLabels           map[string]string
 	Pods                 []string
+	PodDetails           []node.PodDetail `json:"-"`
 	InstanceID           string
 	ProviderID           string
 	InstanceType         string

pkg/monitor/types_test.go

@@ -14,10 +14,13 @@
 package monitor_test
 
 import (
+	"encoding/json"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-node-termination-handler/pkg/monitor"
+	"github.com/aws/aws-node-termination-handler/pkg/node"
 	h "github.com/aws/aws-node-termination-handler/pkg/test"
 )
 
@@ -64,3 +67,17 @@ func TestIsRebalanceRecommendation_Empty_Failure(t *testing.T) {
 	event := &monitor.InterruptionEvent{}
 	h.Equals(t, false, event.IsRebalanceRecommendation())
 }
+
+func TestInterruptionEventJSONOmitsPodDetails(t *testing.T) {
+	event := monitor.InterruptionEvent{
+		Pods: []string{"agent-1"},
+		PodDetails: []node.PodDetail{
+			{Name: "agent-1", Annotations: map[string]string{"example.com/secret": "value"}},
+		},
+	}
+
+	eventJSON, err := json.Marshal(event)
+	h.Ok(t, err)
+	h.Assert(t, !strings.Contains(string(eventJSON), "PodDetails"), "PodDetails should not be serialized")
+	h.Assert(t, !strings.Contains(string(eventJSON), "example.com/secret"), "pod annotations should not be serialized")
+}

pkg/node/node.go

@@ -89,6 +89,18 @@ type Node struct {
 	uptime      uptime.UptimeFuncType
 }
 
+// PodDetail holds details of a pod on a node, as exposed to the webhook template
+type PodDetail struct {
+	Name      string
+	Namespace string
+	Phase     string
+	// OwnerKind is the kind of the pod's controller (e.g. ReplicaSet, DaemonSet, Job), or "" for a bare pod
+	OwnerKind string
+	Labels    map[string]string
+	// Annotations can hold sensitive data; templates should select specific keys rather than render the whole map
+	Annotations map[string]string
+}
+
 type ZerologWriter struct {
 	logger zerolog.Logger
 }
@@ -515,17 +527,37 @@ func (n Node) LogPods(podList []string, nodeName string) error {
 	return nil
 }
 
-// FetchPodNameList fetches list of all the pods names running on given nodeName
-func (n Node) FetchPodNameList(nodeName string) ([]string, error) {
+// FetchPodDetails fetches details of all the pods on given nodeName
+func (n Node) FetchPodDetails(nodeName string) ([]PodDetail, error) {
 	podList, err := n.fetchAllPods(nodeName)
 	if err != nil {
 		return nil, err
 	}
-	var podNamesList []string
+	var podDetails []PodDetail
 	for _, pod := range podList.Items {
-		podNamesList = append(podNamesList, pod.Name)
+		ownerKind := ""
+		if owner := metav1.GetControllerOf(&pod); owner != nil {
+			ownerKind = owner.Kind
+		}
+		podDetails = append(podDetails, PodDetail{
+			Name:        pod.Name,
+			Namespace:   pod.Namespace,
+			Phase:       string(pod.Status.Phase),
+			OwnerKind:   ownerKind,
+			Labels:      pod.Labels,
+			Annotations: pod.Annotations,
+		})
+	}
+	return podDetails, nil
+}
+
+// PodNames returns the names of the given pods
+func PodNames(podDetails []PodDetail) []string {
+	var podNames []string
+	for _, pod := range podDetails {
+		podNames = append(podNames, pod.Name)
 	}
-	return podNamesList, nil
+	return podNames
 }
 
 // TaintScheduledMaintenance adds the scheduled maintenance taint onto a node

pkg/node/node_test.go

@@ -16,6 +16,7 @@ package node_test
 import (
 	"context"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
@@ -291,27 +292,60 @@ func TestMarkForUncordonAfterRebootAddActionLabelFailure(t *testing.T) {
 	h.Assert(t, err != nil, "Failed to return error on MarkForUncordonAfterReboot failing to add action Label")
 }
 
-func TestFetchPodsNameList(t *testing.T) {
+func TestFetchPodDetails(t *testing.T) {
+	isController := true
 	client := fake.NewSimpleClientset(
 		&v1.Pod{
 			ObjectMeta: metav1.ObjectMeta{
-				Name: "myPod",
-				Labels: map[string]string{
-					"spec.nodeName": nodeName,
+				Name:        "web-1",
+				Namespace:   "team-a",
+				Labels:      map[string]string{"app": "web"},
+				Annotations: map[string]string{"example.com/owner": "team-a"},
+				OwnerReferences: []metav1.OwnerReference{
+					{Kind: "ReplicaSet", Name: "web", Controller: &isController},
 				},
 			},
+			Spec:   v1.PodSpec{NodeName: nodeName},
+			Status: v1.PodStatus{Phase: v1.PodRunning},
 		},
-		&v1.Node{
+		&v1.Pod{
 			ObjectMeta: metav1.ObjectMeta{
-				Name: nodeName,
+				Name:      "agent-1",
+				Namespace: "ci",
+				OwnerReferences: []metav1.OwnerReference{
+					{Kind: "ConfigMap", Name: "not-a-controller"},
+				},
 			},
+			Spec:   v1.PodSpec{NodeName: nodeName},
+			Status: v1.PodStatus{Phase: v1.PodSucceeded},
 		},
 	)
 
 	tNode := getNode(t, getDrainHelper(client))
-	podList, err := tNode.FetchPodNameList(nodeName)
-	h.Ok(t, err)
-	h.Equals(t, []string{"myPod"}, podList)
+	podDetails, err := tNode.FetchPodDetails(nodeName)
+	h.Ok(t, err)
+	sort.Slice(podDetails, func(i, j int) bool { return podDetails[i].Name < podDetails[j].Name })
+	h.Equals(t, []node.PodDetail{
+		{
+			Name:      "agent-1",
+			Namespace: "ci",
+			Phase:     "Succeeded",
+			OwnerKind: "",
+		},
+		{
+			Name:        "web-1",
+			Namespace:   "team-a",
+			Phase:       "Running",
+			OwnerKind:   "ReplicaSet",
+			Labels:      map[string]string{"app": "web"},
+			Annotations: map[string]string{"example.com/owner": "team-a"},
+		},
+	}, podDetails)
+}
+
+func TestPodNames(t *testing.T) {
+	h.Equals(t, []string(nil), node.PodNames(nil))
+	h.Equals(t, []string{"a", "b"}, node.PodNames([]node.PodDetail{{Name: "a"}, {Name: "b"}}))
 }
 
 func TestLogPods(t *testing.T) {

pkg/webhook/webhook_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/aws/aws-node-termination-handler/pkg/config"
 	"github.com/aws/aws-node-termination-handler/pkg/ec2metadata"
 	"github.com/aws/aws-node-termination-handler/pkg/monitor"
+	"github.com/aws/aws-node-termination-handler/pkg/node"
 	h "github.com/aws/aws-node-termination-handler/pkg/test"
 	"github.com/aws/aws-node-termination-handler/pkg/webhook"
 	"github.com/rs/zerolog/log"
@@ -114,6 +115,49 @@ func TestPostSuccess(t *testing.T) {
 	webhook.Post(nodeMetadata, event, nthconfig)
 }
 
+func TestPostPodDetails(t *testing.T) {
+	const podDetailsTemplate = `{{ range .PodDetails }}{{ if and (eq .Phase "Running") (ne .OwnerKind "DaemonSet") }}{{ .Namespace }}/{{ .Name }} {{ index .Labels "app" }} {{ index .Annotations "example.com/owner" }};{{ end }}{{ end }}`
+
+	event := &monitor.InterruptionEvent{
+		Pods: []string{"web-1", "ds-1", "done-1"},
+		PodDetails: []node.PodDetail{
+			{
+				Name:        "web-1",
+				Namespace:   "team-a",
+				Phase:       "Running",
+				OwnerKind:   "ReplicaSet",
+				Labels:      map[string]string{"app": "web"},
+				Annotations: map[string]string{"example.com/owner": "alice"},
+			},
+			{Name: "ds-1", Namespace: "kube-system", Phase: "Running", OwnerKind: "DaemonSet"},
+			{Name: "done-1", Namespace: "ci", Phase: "Succeeded"},
+		},
+	}
+
+	requestBody := make(chan string, 1)
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		body, err := io.ReadAll(req.Body)
+		h.Ok(t, err)
+		requestBody <- string(body)
+	}))
+	defer server.Close()
+
+	nthconfig := config.Config{
+		WebhookURL:      server.URL,
+		WebhookHeaders:  testWebhookHeaders,
+		WebhookTemplate: podDetailsTemplate,
+	}
+
+	webhook.Post(ec2metadata.NodeMetadata{}, event, nthconfig)
+
+	select {
+	case body := <-requestBody:
+		h.Equals(t, "team-a/web-1 web alice;", body)
+	default:
+		t.Fatal("webhook was not posted")
+	}
+}
+
 func TestPostTemplateParseError(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		t.Error("Request made with invalid webhook")

test/e2e/webhook-test

@@ -74,7 +74,7 @@ anth_helm_args=(
   --set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO"
   --set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG"
   --set webhookURL="${WEBHOOK_URL}"
-  --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} -  AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\}\"\}"
+  --set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} -  AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\} - Pods: \{\{ range \.PodDetails \}\}\{\{ \.Namespace \}\}/\{\{ \.Name \}\} \{\{ \.Phase \}\}/\{\{ \.OwnerKind \}\} \{\{ end \}\}\"\}"
   --set enableSpotInterruptionDraining="true"
   --set enableScheduledEventDraining="true"
   --set daemonsetTolerations=""