feat(objectstorage): retry on HTTP 429 rate limits

relates to #1764

Author: rubenhoenle

stackit/internal/services/objectstorage/utils/util.go

@@ -1,9 +1,13 @@
 package utils
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"math/rand/v2"
 	"net/http"
+	"strconv"
 	"time"
 
 	objectstorage "github.com/stackitcloud/stackit-sdk-go/services/objectstorage/v2api"
@@ -35,14 +39,124 @@ func EnableProject(ctx context.Context, projectId, region string, client objects
 	return nil
 }
 
+// RetryTransport wraps an underlying RoundTripper to retry on HTTP 429 rate limit errors with jitter.
+type RetryTransport struct {
+	Base        http.RoundTripper
+	MaxRetries  int
+	BaseBackoff time.Duration
+	MaxJitter   time.Duration
+}
+
+func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	base := t.Base
+	if base == nil {
+		base = http.DefaultTransport
+	}
+
+	// Preserve request body for retries if present
+	var bodyBytes []byte
+	if req.Body != nil && req.Body != http.NoBody {
+		var err error
+		bodyBytes, err = io.ReadAll(req.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		err = req.Body.Close()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var resp *http.Response
+	var err error
+
+	for attempt := 0; attempt <= t.MaxRetries; attempt++ {
+		// Re-hydrate the request body on each attempt
+		if bodyBytes != nil {
+			req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+		}
+
+		resp, err = base.RoundTrip(req)
+
+		// If success or non-429 error, return immediately
+		if err != nil || resp.StatusCode != http.StatusTooManyRequests {
+			return resp, err
+		}
+
+		// Stop if max retries reached
+		if attempt == t.MaxRetries {
+			break
+		}
+
+		// Calculate base sleep duration (value of Retry-After Header, if header isn't present falls back to exponential backoff)
+		wait := t.getWaitDuration(resp, attempt)
+
+		// Always add random jitter regardless of Retry-After header presence. Else all resource / datasource
+		// goroutines would try again in parallel after exactly the same interval.
+		jitter := time.Duration(rand.Int64N(int64(t.MaxJitter))) //nolint:gosec // only used for jitter
+		totalWait := wait + jitter
+
+		// Drain and close response body before retrying to reuse TCP connections
+		_, err = io.Copy(io.Discard, resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		err = resp.Body.Close()
+		if err != nil {
+			return nil, err
+		}
+
+		select {
+		case <-req.Context().Done():
+			return nil, req.Context().Err()
+		case <-time.After(totalWait):
+		}
+	}
+
+	return resp, err
+}
+
+func (t *RetryTransport) getWaitDuration(resp *http.Response, attempt int) time.Duration {
+	if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" {
+		// Try parsing as integer seconds
+		if seconds, err := strconv.Atoi(retryAfter); err == nil {
+			return time.Duration(seconds) * time.Second
+		}
+		// Try parsing as HTTP-Date string
+		if date, err := http.ParseTime(retryAfter); err == nil {
+			if d := time.Until(date); d > 0 {
+				return d
+			}
+		}
+	}
+
+	// Fallback to exponential backoff
+	return t.BaseBackoff * (1 << attempt)
+}
+
 func ConfigureClient(ctx context.Context, providerData *core.ProviderData, diags *diag.Diagnostics) *objectstorage.APIClient {
+	// Add middleware to retry on HTTP 429 rate limits.
+	// This solution is **not** intended to be copied to each and every service (!!).
+	// This should be rolled out centrally instead, should be easily doable after
+	// this refactoring: https://github.com/stackitcloud/terraform-provider-stackit/pull/1663
+	retryRoundTripper := &RetryTransport{
+		Base:        providerData.RoundTripper,
+		MaxRetries:  3,
+		BaseBackoff: 1 * time.Second,
+		MaxJitter:   500 * time.Millisecond, // Always added to wait time
+	}
+
 	apiClientConfigOptions := []config.ConfigurationOption{
-		config.WithCustomAuth(providerData.RoundTripper),
+		config.WithCustomAuth(retryRoundTripper),
 		utils.UserAgentConfigOption(providerData.Version),
 	}
+
 	if providerData.ObjectStorageCustomEndpoint != "" {
 		apiClientConfigOptions = append(apiClientConfigOptions, config.WithEndpoint(providerData.ObjectStorageCustomEndpoint))
 	}
+
 	apiClient, err := objectstorage.NewAPIClient(apiClientConfigOptions...)
 	if err != nil {
 		core.LogAndAddError(ctx, diags, "Error configuring API client", fmt.Sprintf("Configuring client: %v. This is an error related to the provider configuration, not to the resource configuration", err))

stackit/internal/services/objectstorage/utils/util_test.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"os"
@@ -33,6 +34,10 @@ func TestConfigureClient(t *testing.T) {
 		t.Errorf("error setting env variable: %v", err)
 	}
 
+	var roundTripper http.RoundTripper = &http.Transport{
+		TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13},
+	}
+
 	type args struct {
 		providerData *core.ProviderData
 	}
@@ -46,13 +51,21 @@ func TestConfigureClient(t *testing.T) {
 			name: "default endpoint",
 			args: args{
 				providerData: &core.ProviderData{
-					Version: testVersion,
+					Version:      testVersion,
+					RoundTripper: roundTripper,
 				},
 			},
 			expected: func() *objectstorage.APIClient {
 				apiClient, err := objectstorage.NewAPIClient(
 					utils.UserAgentConfigOption(testVersion),
+					config.WithCustomAuth(&RetryTransport{
+						Base:        roundTripper,
+						MaxRetries:  3,
+						BaseBackoff: 1 * time.Second,
+						MaxJitter:   500 * time.Millisecond,
+					}),
 				)
+
 				if err != nil {
 					t.Errorf("error configuring client: %v", err)
 				}
@@ -65,13 +78,20 @@ func TestConfigureClient(t *testing.T) {
 			args: args{
 				providerData: &core.ProviderData{
 					Version:                     testVersion,
+					RoundTripper:                roundTripper,
 					ObjectStorageCustomEndpoint: testCustomEndpoint,
 				},
 			},
 			expected: func() *objectstorage.APIClient {
 				apiClient, err := objectstorage.NewAPIClient(
 					utils.UserAgentConfigOption(testVersion),
 					config.WithEndpoint(testCustomEndpoint),
+					config.WithCustomAuth(&RetryTransport{
+						Base:        roundTripper,
+						MaxRetries:  3,
+						BaseBackoff: 1 * time.Second,
+						MaxJitter:   500 * time.Millisecond,
+					}),
 				)
 				if err != nil {
 					t.Errorf("error configuring client: %v", err)
@@ -91,6 +111,7 @@ func TestConfigureClient(t *testing.T) {
 				t.Errorf("ConfigureClient() error = %v, want %v", diags.HasError(), tt.wantErr)
 			}
 
+			// fails because reflect.DeepEqual can't compare functions properly -> false
 			if !reflect.DeepEqual(actual, tt.expected) {
 				t.Errorf("ConfigureClient() = %v, want %v", actual, tt.expected)
 			}