|
| 1 | +package metering |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "encoding/json" |
| 6 | + "net/http" |
| 7 | + "net/http/httptest" |
| 8 | + "sync/atomic" |
| 9 | + "testing" |
| 10 | + "time" |
| 11 | + |
| 12 | + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" |
| 13 | +) |
| 14 | + |
| 15 | +func TestReservationRenewsBeyondOriginalExpiryAndStops(t *testing.T) { |
| 16 | + var calls atomic.Int32 |
| 17 | + var lost atomic.Bool |
| 18 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 19 | + if calls.Add(1) == 1 { |
| 20 | + w.WriteHeader(503) |
| 21 | + return |
| 22 | + } |
| 23 | + json.NewEncoder(w).Encode(reserveResponse{ReservationID: "lease", LeaseMS: 300}) |
| 24 | + })) |
| 25 | + defer server.Close() |
| 26 | + client := NewClient(server.URL, "fixture", time.Second) |
| 27 | + client.reservations.Store("lease", newLease(time.Now(), 300)) |
| 28 | + stop := client.KeepReservation(context.Background(), "lease", func() { lost.Store(true) }) |
| 29 | + time.Sleep(600 * time.Millisecond) |
| 30 | + stop() |
| 31 | + if lost.Load() || calls.Load() < 3 { |
| 32 | + t.Fatalf("renewal did not preserve hold: lost=%v calls=%d", lost.Load(), calls.Load()) |
| 33 | + } |
| 34 | + before := calls.Load() |
| 35 | + time.Sleep(150 * time.Millisecond) |
| 36 | + if calls.Load() != before { |
| 37 | + t.Fatal("renewal continued after stop") |
| 38 | + } |
| 39 | +} |
| 40 | + |
| 41 | +func TestLostReservationStopsOnlyWhenLeaseExpires(t *testing.T) { |
| 42 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(503) })) |
| 43 | + defer server.Close() |
| 44 | + client := NewClient(server.URL, "fixture", time.Second) |
| 45 | + client.reservations.Store("lease", newLease(time.Now(), 120)) |
| 46 | + lost := make(chan struct{}) |
| 47 | + stop := client.KeepReservation(context.Background(), "lease", func() { close(lost) }) |
| 48 | + defer stop() |
| 49 | + select { |
| 50 | + case <-lost: |
| 51 | + t.Fatal("temporary error canceled a valid lease") |
| 52 | + case <-time.After(50 * time.Millisecond): |
| 53 | + } |
| 54 | + select { |
| 55 | + case <-lost: |
| 56 | + case <-time.After(time.Second): |
| 57 | + t.Fatal("lost hold did not stop inference") |
| 58 | + } |
| 59 | +} |
| 60 | + |
| 61 | +// Metering refusing a renewal means the hold is gone; retrying cannot help. |
| 62 | +func TestRejectedRenewalIsLostImmediately(t *testing.T) { |
| 63 | + var calls atomic.Int32 |
| 64 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 65 | + calls.Add(1) |
| 66 | + w.WriteHeader(http.StatusConflict) |
| 67 | + json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"code": "conflict", "message": "reservation is no longer active"}}) |
| 68 | + })) |
| 69 | + defer server.Close() |
| 70 | + client := NewClient(server.URL, "fixture", time.Second) |
| 71 | + client.reservations.Store("lease", newLease(time.Now(), 3000)) |
| 72 | + lost := make(chan struct{}) |
| 73 | + stop := client.KeepReservation(context.Background(), "lease", func() { close(lost) }) |
| 74 | + defer stop() |
| 75 | + select { |
| 76 | + case <-lost: |
| 77 | + case <-time.After(1500 * time.Millisecond): |
| 78 | + t.Fatal("rejected renewal waited for the lease to run out") |
| 79 | + } |
| 80 | + if calls.Load() != 1 { |
| 81 | + t.Fatalf("retried a rejected renewal: %d calls", calls.Load()) |
| 82 | + } |
| 83 | +} |
| 84 | + |
| 85 | +// During a Metering outage each request renews a bounded number of times |
| 86 | +// instead of an accelerating burst as the lease runs out. |
| 87 | +func TestRenewalRetriesAreSpacedDuringOutage(t *testing.T) { |
| 88 | + var calls atomic.Int32 |
| 89 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 90 | + calls.Add(1) |
| 91 | + w.WriteHeader(503) |
| 92 | + })) |
| 93 | + defer server.Close() |
| 94 | + client := NewClient(server.URL, "fixture", time.Second) |
| 95 | + client.reservations.Store("lease", newLease(time.Now(), 1000)) |
| 96 | + lost := make(chan struct{}) |
| 97 | + stop := client.KeepReservation(context.Background(), "lease", func() { close(lost) }) |
| 98 | + defer stop() |
| 99 | + select { |
| 100 | + case <-lost: |
| 101 | + case <-time.After(3 * time.Second): |
| 102 | + t.Fatal("lease never expired") |
| 103 | + } |
| 104 | + // One tenth of a 1 s lease is 100 ms, so at most about ten attempts. |
| 105 | + if got := calls.Load(); got > 11 { |
| 106 | + t.Fatalf("renewal attempts were not spaced: %d", got) |
| 107 | + } |
| 108 | +} |
| 109 | + |
| 110 | +// The lease is measured on the Gateway's clock from before the request, so a |
| 111 | +// skewed Metering or enclave clock cannot shorten or extend it. |
| 112 | +func TestReserveStoresLeaseRelativeToGatewayClock(t *testing.T) { |
| 113 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 114 | + json.NewEncoder(w).Encode(reserveResponse{ReservationID: "lease", LeaseMS: 60_000}) |
| 115 | + })) |
| 116 | + defer server.Close() |
| 117 | + client := NewClient(server.URL, "fixture", time.Second) |
| 118 | + before := time.Now() |
| 119 | + id, err := client.Reserve(context.Background(), domain.AuthContext{Account: domain.Account{ID: "a"}, APIKey: domain.APIKey{ID: "k"}}, "chat/completions", domain.GenerateRequest{}, domain.PublicModel{PublicModelID: "m"}, "request") |
| 120 | + if err != nil || id != "lease" { |
| 121 | + t.Fatalf("reserve = %q, %v", id, err) |
| 122 | + } |
| 123 | + value, ok := client.reservations.Load("lease") |
| 124 | + if !ok { |
| 125 | + t.Fatal("lease not stored") |
| 126 | + } |
| 127 | + held := value.(lease) |
| 128 | + if held.length != time.Minute || held.expires.Before(before.Add(time.Minute)) || held.expires.After(time.Now().Add(time.Minute)) { |
| 129 | + t.Fatalf("lease = %+v", held) |
| 130 | + } |
| 131 | +} |
| 132 | + |
| 133 | +func TestRenewalMetricsDoNotIncludeReservationIDs(t *testing.T) { |
| 134 | + if got := meteringOperation(http.MethodPost, "/internal/metering/reservations/private-id/renew"); got != "renew" { |
| 135 | + t.Fatalf("renewal metric operation = %q", got) |
| 136 | + } |
| 137 | +} |
0 commit comments