Skip to content

Commit 388e775

Browse files
committed
fix: transient unhealthy state during capacity doubling
1 parent 66519d7 commit 388e775

2 files changed

Lines changed: 101 additions & 2 deletions

File tree

pkg/salud/salud.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -242,9 +242,9 @@ func (s *service) salud(mode string, durPercentile float64, connsPercentile floa
242242
}
243243

244244
selfHealth := true
245-
if nHoodRadius == networkRadius && s.reserve.CommittedDepth() != networkRadius {
245+
if nHoodRadius == networkRadius && s.reserve.StorageRadius() > networkRadius {
246246
selfHealth = false
247-
s.logger.Warning("node is unhealthy due to storage radius discrepancy", "self_radius", s.reserve.CommittedDepth(), "network_radius", networkRadius)
247+
s.logger.Warning("node is unhealthy due to storage radius discrepancy", "self_storage_radius", s.reserve.StorageRadius(), "network_radius", networkRadius)
248248
}
249249

250250
s.isSelfHealthy.Store(selfHealth)

pkg/salud/salud_test.go

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,105 @@ func TestUnsub(t *testing.T) {
233233
}
234234
}
235235

236+
// TestTransientHealthy validates that a node remains healthy when its CommittedDepth
237+
// exceeds the NetworkRadius due to CapacityDoubling, provided its StorageRadius
238+
// is still sufficient (<= NetworkRadius).
239+
//
240+
// Scenario:
241+
// - Network Radius: 9
242+
// - Local Storage Radius: 9
243+
// - Capacity Doubling: 1
244+
// - Local Committed Depth: 10 (9 + 1)
245+
//
246+
// Expectation: Healthy
247+
func TestTransientHealthy(t *testing.T) {
248+
t.Parallel()
249+
peers := []peer{
250+
{swarm.RandAddress(t), &status.Snapshot{ConnectedPeers: 100, StorageRadius: 9, BeeMode: "full", CommittedDepth: 9}, 0, true},
251+
{swarm.RandAddress(t), &status.Snapshot{ConnectedPeers: 100, StorageRadius: 9, BeeMode: "full", CommittedDepth: 9}, 0, true},
252+
}
253+
254+
statusM := &statusMock{make(map[string]peer)}
255+
addrs := make([]swarm.Address, 0, len(peers))
256+
for _, p := range peers {
257+
addrs = append(addrs, p.addr)
258+
statusM.peers[p.addr.ByteString()] = p
259+
}
260+
261+
topM := topMock.NewTopologyDriver(topMock.WithPeers(addrs...))
262+
263+
reserve := mockstorer.NewReserve(
264+
mockstorer.WithRadius(9), // Same as network
265+
mockstorer.WithReserveSize(100),
266+
mockstorer.WithCapacityDoubling(1), // Adds 1 to committed depth -> 10
267+
)
268+
269+
service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8)
270+
testutil.CleanupCloser(t, service)
271+
272+
// Wait for peer health sync
273+
err := spinlock.Wait(time.Minute, func() bool {
274+
return len(topM.PeersHealth()) == len(peers)
275+
})
276+
if err != nil {
277+
t.Fatal(err)
278+
}
279+
280+
// Verify self health
281+
// With old logic: CommittedDepth (10) != NetworkRadius (9) -> Unhealthy
282+
// With new logic: StorageRadius (9) > NetworkRadius (9) (False) -> Healthy
283+
if !service.IsHealthy() {
284+
t.Fatalf("self should be healthy in transient state")
285+
}
286+
}
287+
288+
// TestActuallyUnhealthy validates that we still catch cases where StorageRadius
289+
// is too high (node storing subset of required data), even with CapacityDoubling.
290+
//
291+
// Scenario:
292+
// - Network Radius: 9
293+
// - Local Storage Radius: 10 (Too specific!)
294+
// - Capacity Doubling: 1
295+
// - Local Committed Depth: 11
296+
//
297+
// Expectation: Unhealthy
298+
func TestActuallyUnhealthy(t *testing.T) {
299+
t.Parallel()
300+
peers := []peer{
301+
{swarm.RandAddress(t), &status.Snapshot{ConnectedPeers: 100, StorageRadius: 9, BeeMode: "full", CommittedDepth: 9}, 0, true},
302+
{swarm.RandAddress(t), &status.Snapshot{ConnectedPeers: 100, StorageRadius: 9, BeeMode: "full", CommittedDepth: 9}, 0, true},
303+
}
304+
305+
statusM := &statusMock{make(map[string]peer)}
306+
addrs := make([]swarm.Address, 0, len(peers))
307+
for _, p := range peers {
308+
addrs = append(addrs, p.addr)
309+
statusM.peers[p.addr.ByteString()] = p
310+
}
311+
312+
topM := topMock.NewTopologyDriver(topMock.WithPeers(addrs...))
313+
314+
reserve := mockstorer.NewReserve(
315+
mockstorer.WithRadius(10), // Higher than network! Missing data.
316+
mockstorer.WithReserveSize(100),
317+
mockstorer.WithCapacityDoubling(1),
318+
)
319+
320+
service := salud.New(statusM, topM, reserve, log.Noop, stabilmock.NewSubscriber(true), "full", 0.8, 0.8)
321+
testutil.CleanupCloser(t, service)
322+
323+
err := spinlock.Wait(time.Minute, func() bool {
324+
return len(topM.PeersHealth()) == len(peers)
325+
})
326+
if err != nil {
327+
t.Fatal(err)
328+
}
329+
330+
if service.IsHealthy() {
331+
t.Fatalf("self should NOT be healthy when storage radius is too high")
332+
}
333+
}
334+
236335
type statusMock struct {
237336
peers map[string]peer
238337
}

0 commit comments

Comments
 (0)