Add http debugging

When developing things (like
prometheus#962), it's useful to be able
to debug the http roundtrips. This adds a an --http.debug flag which
logs the HTTP requests while redacting credentials.

Signed-off-by: Phil Dibowitz <phil@ipom.com>

Author: jaymzh

config/http_debug.go

@@ -0,0 +1,113 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptrace"
+	"strings"
+	"time"
+)
+
+const debugBodyPreviewLimit = 512
+
+var redactedDebugRequestHeaders = map[string]struct{}{
+	"authorization":           {},
+	"cf-access-client-id":     {},
+	"cf-access-client-secret": {},
+	"cf-access-token":         {},
+	"proxy-authorization":     {},
+}
+
+// NewDebugRoundTripper returns a RoundTripper that writes outgoing HTTP
+// requests and their responses to out. Credential-bearing request headers are
+// redacted, and response body previews are limited to 512 bytes.
+func NewDebugRoundTripper(out io.Writer, next http.RoundTripper) http.RoundTripper {
+	return &debugRoundTripper{out: out, next: next}
+}
+
+type debugRoundTripper struct {
+	out  io.Writer
+	next http.RoundTripper
+}
+
+// RoundTrip implements http.RoundTripper.
+func (rt *debugRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	fmt.Fprintf(rt.out, "--> %s %s\n", req.Method, req.URL)
+
+	trace := &httptrace.ClientTrace{
+		WroteHeaderField: func(key string, values []string) {
+			fmt.Fprintf(rt.out, "    %s: %s\n", key, redactDebugHeader(key, values))
+		},
+	}
+	req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
+
+	start := time.Now()
+	resp, err := rt.next.RoundTrip(req)
+	elapsed := time.Since(start)
+	if err != nil {
+		fmt.Fprintf(rt.out, "<-- error after %s: %v\n", elapsed, err)
+		return resp, err
+	}
+
+	fmt.Fprintf(rt.out, "<-- %s in %s (content-type: %s)\n", resp.Status, elapsed, resp.Header.Get("Content-Type"))
+
+	if resp.Body != nil {
+		preview := make([]byte, debugBodyPreviewLimit)
+		n, _ := io.ReadFull(resp.Body, preview)
+		resp.Body = &previewedBody{
+			preview: preview[:n],
+			rest:    resp.Body,
+		}
+		if n > 0 {
+			fmt.Fprintf(rt.out, "    body preview: %q\n", preview[:n])
+		}
+	}
+
+	return resp, nil
+}
+
+func (rt *debugRoundTripper) CloseIdleConnections() {
+	if ci, ok := rt.next.(closeIdler); ok {
+		ci.CloseIdleConnections()
+	}
+}
+
+func redactDebugHeader(name string, values []string) string {
+	if _, ok := redactedDebugRequestHeaders[strings.ToLower(name)]; ok {
+		return "<redacted>"
+	}
+	return strings.Join(values, ", ")
+}
+
+type previewedBody struct {
+	preview []byte
+	off     int
+	rest    io.ReadCloser
+}
+
+func (b *previewedBody) Read(p []byte) (int, error) {
+	if b.off < len(b.preview) {
+		n := copy(p, b.preview[b.off:])
+		b.off += n
+		return n, nil
+	}
+	return b.rest.Read(p)
+}
+
+func (b *previewedBody) Close() error {
+	return b.rest.Close()
+}

config/http_debug_test.go

@@ -0,0 +1,78 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDebugRoundTripper(t *testing.T) {
+	const responseBody = "complete response body"
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, err := io.WriteString(w, responseBody)
+		require.NoError(t, err)
+	}))
+	t.Cleanup(server.Close)
+
+	var output bytes.Buffer
+	client := &http.Client{Transport: NewDebugRoundTripper(&output, http.DefaultTransport)}
+	req, err := http.NewRequest(http.MethodGet, server.URL+"/alerts", http.NoBody)
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "Bearer authorization-secret")
+	req.Header.Set("Cf-Access-Token", "cf-access-secret")
+	req.Header.Set("X-Debug-Test", "visible")
+
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, resp.Body.Close()) })
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, responseBody, string(body))
+
+	log := output.String()
+	require.Contains(t, log, "--> GET "+server.URL+"/alerts")
+	require.Contains(t, log, "Authorization: <redacted>")
+	require.Contains(t, log, "Cf-Access-Token: <redacted>")
+	require.Contains(t, log, "X-Debug-Test: visible")
+	require.Contains(t, log, "<-- 200 OK")
+	require.Contains(t, log, `body preview: "complete response body"`)
+	require.NotContains(t, log, "authorization-secret")
+	require.NotContains(t, log, "cf-access-secret")
+}
+
+func TestDebugRoundTripperError(t *testing.T) {
+	expectedErr := errors.New("request failed")
+	next := NewRoundTripCheckRequest(func(*http.Request) {}, nil, expectedErr)
+	var output bytes.Buffer
+	rt := NewDebugRoundTripper(&output, next)
+	req, err := http.NewRequest(http.MethodGet, "https://example.com/alerts", http.NoBody)
+	require.NoError(t, err)
+
+	_, err = rt.RoundTrip(req)
+	require.ErrorIs(t, err, expectedErr)
+	require.True(t, strings.HasPrefix(output.String(), "--> GET https://example.com/alerts\n"))
+	require.Contains(t, output.String(), "<-- error after ")
+	require.Contains(t, output.String(), expectedErr.Error())
+}