fix(admin-token-issuer-proxy): retry DNS cold starts

Author: mikeyrcamp

src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache.go

@@ -114,6 +114,14 @@ func isRetryableTransportError(err error) bool {
 		return false
 	}
 
+	// Kubernetes DNS can return NXDOMAIN until a Service is created. Treat DNS
+	// resolution failures as startup-transient so the process remains live and
+	// unready while it waits for api-keys to become discoverable.
+	var dnsErr *net.DNSError
+	if errors.As(err, &dnsErr) {
+		return true
+	}
+
 	var networkErr net.Error
 	if errors.As(err, &networkErr) && (networkErr.Timeout() || networkErr.Temporary()) {
 		return true

src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache_test.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -189,6 +190,40 @@ func TestCacheFetchWithRetryToleratesConnectionRefused(t *testing.T) {
 	}
 }
 
+func TestCacheFetchWithRetryToleratesDNSNotFound(t *testing.T) {
+	var requests atomic.Int32
+	cache := New("http://api-keys.test/v1/services")
+	cache.httpClient.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		if requests.Add(1) < 3 {
+			return nil, &url.Error{
+				Op:  r.Method,
+				URL: r.URL.String(),
+				Err: &net.DNSError{Name: "api-keys.test", IsNotFound: true},
+			}
+		}
+		return &http.Response{
+			StatusCode: http.StatusOK,
+			Header:     make(http.Header),
+			Body: io.NopCloser(newJSONReader(t, models.ServicesResponse{Services: []models.ServiceInfo{{
+				ServiceID:   "test-service-id",
+				ServiceName: "test-service",
+			}}})),
+		}, nil
+	})
+
+	err := cache.FetchWithRetry(context.Background(), RetryPolicy{
+		InitialBackoff: time.Millisecond,
+		MaxBackoff:     2 * time.Millisecond,
+	})
+
+	if err != nil {
+		t.Fatalf("expected DNS-not-found retries to recover, got %v", err)
+	}
+	if got := requests.Load(); got != 3 {
+		t.Fatalf("expected 3 metadata requests, got %d", got)
+	}
+}
+
 func TestCacheFetchWithRetryStopsOnPermanentTransportConfiguration(t *testing.T) {
 	testCases := []struct {
 		name string