Proxy: Preserve HTTP response headers and safely rewrite HLS. v8.0.20 (#4718)

## Summary

- Preserve end-to-end backend response headers for HTTP-FLV, HTTP-TS,
and HLS responses.
- Remove standard hop-by-hop headers and additional fields nominated by
`Connection`.
- Rewrite HLS segment URLs with a canonical single `&` when the original
URL already has query parameters.
- Repair response metadata after modifying an m3u8 body, and add
wire-level regression coverage.

## Problem

The HTTP proxy called `WriteHeader` before copying headers from the
backend. Go commits the response at that point, so later additions such
as `Content-Type`, `Cache-Control`, `ETag`, `Last-Modified`, `Vary`, and
custom origin metadata never reached the downstream client.

HLS playlist rewriting also generated segment URLs containing an
unnecessary empty query component:

```text
segment.ts?spbhid=xxx&&token=abc
```

Moving header copying before `WriteHeader` is not sufficient by itself.
An m3u8 playlist is modified when the proxy inserts `spbhid`, so the
backend's original `Content-Length`, strong `ETag`, digest, content
encoding, and range metadata no longer describe the downstream
representation. Forwarding the stale length can make Go reject the
enlarged response with `http: wrote more than the declared
Content-Length`.

A proxy must also avoid forwarding connection-specific headers to the
next hop.

## Changes

- Copy backend end-to-end headers before committing HTTP-FLV, HTTP-TS,
and HLS responses.
- Strip `Connection`, its nominated fields, and the standard
proxy/connection-specific header set.
- Keep byte-transparent FLV and TS representation metadata unchanged.
- Read and rewrite m3u8 playlists before sending response headers.
- Recompute `Content-Length` and remove stale validators, digests,
encodings, and range metadata when the playlist body changes.
- Generate existing-query segment URLs as:

  ```text
  segment.ts?spbhid=xxx&token=abc
  ```

- Add real HTTP-boundary tests for header preservation, hop-by-hop
removal, query rewriting, complete playlist delivery, and rewritten-body
metadata.

---------

Co-authored-by: chatgpt-codex-connector[bot] <199175422+chatgpt-codex-connector[bot]@users.noreply.github.com>

Author: winlinvip

internal/proxy/http.go

@@ -44,6 +44,60 @@ func buildBackendHTTPURL(ip string, port int, path string) string {
 	return fmt.Sprintf("http://%v:%v%s", ip, port, path)
 }
 
+// copyBackendResponseHeaders copies end-to-end response headers while dropping
+// fields that apply only to the backend connection. Connection can nominate
+// additional hop-by-hop fields, so collect those names before copying.
+func copyBackendResponseHeaders(dst, src http.Header) {
+	hopByHop := map[string]bool{
+		"Connection":          true,
+		"Proxy-Connection":    true,
+		"Keep-Alive":          true,
+		"Proxy-Authenticate":  true,
+		"Proxy-Authorization": true,
+		"Te":                  true,
+		"Trailer":             true,
+		"Transfer-Encoding":   true,
+		"Upgrade":             true,
+	}
+	for _, value := range src.Values("Connection") {
+		for _, name := range strings.Split(value, ",") {
+			if name = strings.TrimSpace(name); name != "" {
+				hopByHop[http.CanonicalHeaderKey(name)] = true
+			}
+		}
+	}
+
+	for name, values := range src {
+		name = http.CanonicalHeaderKey(name)
+		if hopByHop[name] {
+			continue
+		}
+
+		dst.Del(name)
+		for _, value := range values {
+			dst.Add(name, value)
+		}
+	}
+}
+
+// repairRewrittenResponseHeaders removes backend representation metadata that
+// no longer describes a rewritten response body, then publishes its new size.
+func repairRewrittenResponseHeaders(header http.Header, contentLength int) {
+	for _, name := range []string{
+		"Accept-Ranges",
+		"Content-Digest",
+		"Content-Encoding",
+		"Content-MD5",
+		"Content-Range",
+		"Digest",
+		"ETag",
+		"Repr-Digest",
+	} {
+		header.Del(name)
+	}
+	header.Set("Content-Length", strconv.Itoa(contentLength))
+}
+
 type httpStreamProxyServer struct {
 	// The environment interface.
 	environment env.ProxyEnvironment
@@ -347,13 +401,10 @@ func (v *httpFlvTsConnection) serveByBackend(ctx context.Context, w http.Respons
 		return errors.Errorf("proxy stream to %v failed, status=%v", backendURL, resp.Status)
 	}
 
-	// Copy all headers from backend to client.
+	// Copy end-to-end headers before committing the response. Headers changed
+	// after WriteHeader are not sent by net/http.
+	copyBackendResponseHeaders(w.Header(), resp.Header)
 	w.WriteHeader(resp.StatusCode)
-	for k, v := range resp.Header {
-		for _, vv := range v {
-			w.Header().Add(k, vv)
-		}
-	}
 
 	logger.Debug(ctx, "HTTP start streaming")
 
@@ -476,16 +527,11 @@ func (v *hlsPlayStream) serveByBackend(ctx context.Context, w http.ResponseWrite
 		return errors.Errorf("proxy stream to %v failed, status=%v", backendURL, resp.Status)
 	}
 
-	// Copy all headers from backend to client.
-	w.WriteHeader(resp.StatusCode)
-	for k, v := range resp.Header {
-		for _, vv := range v {
-			w.Header().Add(k, vv)
-		}
-	}
-
 	// For TS file, directly copy it.
 	if !strings.HasSuffix(r.URL.Path, ".m3u8") {
+		copyBackendResponseHeaders(w.Header(), resp.Header)
+		w.WriteHeader(resp.StatusCode)
+
 		if _, err := io.Copy(w, resp.Body); err != nil {
 			return errors.Wrapf(err, "copy stream to client, backend=%v", backendURL)
 		}
@@ -500,13 +546,20 @@ func (v *hlsPlayStream) serveByBackend(ctx context.Context, w http.ResponseWrite
 		return errors.Wrapf(err, "read stream from %v", backendURL)
 	}
 
-	m3u8 := string(b)
+	backendM3U8 := string(b)
+	m3u8 := backendM3U8
 	if strings.Contains(m3u8, ".ts?") {
-		m3u8 = strings.ReplaceAll(m3u8, ".ts?", fmt.Sprintf(".ts?spbhid=%v&&", v.SRSProxyBackendHLSID))
+		m3u8 = strings.ReplaceAll(m3u8, ".ts?", fmt.Sprintf(".ts?spbhid=%v&", v.SRSProxyBackendHLSID))
 	} else {
 		m3u8 = strings.ReplaceAll(m3u8, ".ts", fmt.Sprintf(".ts?spbhid=%v", v.SRSProxyBackendHLSID))
 	}
 
+	copyBackendResponseHeaders(w.Header(), resp.Header)
+	if m3u8 != backendM3U8 {
+		repairRewrittenResponseHeaders(w.Header(), len(m3u8))
+	}
+	w.WriteHeader(resp.StatusCode)
+
 	if _, err := io.Copy(w, strings.NewReader(m3u8)); err != nil {
 		return errors.Wrapf(err, "proxy m3u8 client to %v", backendURL)
 	}