Skip to content

Commit 2b41a67

Browse files
authored
Merge pull request #498 from kaisoz/kaisoz/bug-router-relay-limits
feat(router): configurable relay circuit limits
2 parents 340d926 + a4470ea commit 2b41a67

10 files changed

Lines changed: 229 additions & 2 deletions

File tree

‎cmd/sam-one/main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,8 @@ func main() {
233233
rootCmd.Flags().IntVar(&routerTunables.ConnsPerSourceIP, "router-conns-per-source-ip", 0, "Per-source-IP connection budget (0 follows the high watermark; proxied peers share source IPs)")
234234
rootCmd.Flags().DurationVar(&routerTunables.DHTProviderAddrTTL, "router-dht-provider-addr-ttl", 0, "DHT provider address TTL (0 keeps the library default)")
235235
rootCmd.Flags().DurationVar(&routerTunables.DHTMaxRecordAge, "router-dht-max-record-age", 0, "DHT record max age (0 keeps the library default)")
236+
rootCmd.Flags().DurationVar(&routerTunables.RelayLimitDuration, "router-relay-limit-duration", 0, "Relayed connection lifetime (0 keeps the component default of 1h; use e.g. 24h for longer)")
237+
rootCmd.Flags().Var(&routerTunables.RelayLimitData, "router-relay-limit-data", "Bytes relayed per direction per connection, e.g. 512MiB (0 keeps the component default: no limit)")
236238
rootCmd.Flags().BoolVar(&routerAllowLoopback, "router-allow-loopback", true, "Advertise loopback addresses (disable on public deployments)")
237239

238240
rootCmd.AddCommand(newAdminSubcommands()...)

‎cmd/sam-router/main.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ var (
4646
lowWaterMark int
4747
highWaterMark int
4848
metricsAddr string
49+
relayLimitDuration time.Duration
50+
relayLimitData router.ByteSize
4951
)
5052

5153
var logger = golog.Logger("sam-router-cli")
@@ -86,6 +88,8 @@ func main() {
8688
LowWaterMark: lowWaterMark,
8789
HighWaterMark: highWaterMark,
8890
MetricsAddr: metricsAddr,
91+
RelayLimitDuration: relayLimitDuration,
92+
RelayLimitData: int64(relayLimitData),
8993
}
9094

9195
r, err := router.NewRouter(cmd.Context(), opts)
@@ -125,6 +129,8 @@ func main() {
125129
rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 1000, "Connection manager low watermark limit")
126130
rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", 4000, "Connection manager high watermark limit")
127131
rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")
132+
rootCmd.Flags().DurationVar(&relayLimitDuration, "relay-limit-duration", router.DefaultRelayLimitDuration, "Lifetime of each relayed connection (0 = no limit)")
133+
rootCmd.Flags().Var(&relayLimitData, "relay-limit-data", "Bytes relayed per direction on each relayed connection, e.g. 512MiB (0 = no limit)")
128134

129135
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
130136
defer cancel()

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/a2aproject/a2a-go/v2 v2.5.0
77
github.com/biscuit-auth/biscuit-go/v2 v2.2.0
88
github.com/coreos/go-oidc/v3 v3.21.0
9+
github.com/dustin/go-humanize v1.0.1
910
github.com/golang-jwt/jwt/v5 v5.3.1
1011
github.com/hashicorp/golang-lru/v2 v2.0.7
1112
github.com/ipfs/go-cid v0.6.2
@@ -48,7 +49,6 @@ require (
4849
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
4950
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect
5051
github.com/dunglas/httpsfv v1.1.0 // indirect
51-
github.com/dustin/go-humanize v1.0.1 // indirect
5252
github.com/filecoin-project/go-clock v0.1.0 // indirect
5353
github.com/flynn/noise v1.1.0 // indirect
5454
github.com/go-jose/go-jose/v4 v4.1.4 // indirect

‎internal/router/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ type Options struct {
4343
DHTMaxRecordAge time.Duration
4444
LowWaterMark int
4545
HighWaterMark int
46+
// RelayLimitDuration / RelayLimitData cap each relayed connection; 0 is
47+
// no limit, so Default() leaves them alone.
48+
RelayLimitDuration time.Duration
49+
RelayLimitData int64
4650
// RequiredRole restricts enrollment and startup to only accept tokens containing this role.
4751
RequiredRole string
4852
// HTTPFallbackHandler, when set, serves ordinary (non-WebSocket-upgrade)

‎internal/router/relay_limit.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package router
16+
17+
import (
18+
"fmt"
19+
"math"
20+
"time"
21+
22+
"github.com/dustin/go-humanize"
23+
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
24+
)
25+
26+
// DefaultRelayLimitDuration is how long a relayed connection lives unless the operator sets it.
27+
const DefaultRelayLimitDuration = time.Hour
28+
29+
// relayLimit maps operator limits (0 = none) onto go-libp2p, where only a nil
30+
// limit is unlimited and a zero field cuts every circuit at once.
31+
func relayLimit(duration time.Duration, data int64) *relay.RelayLimit {
32+
if duration <= 0 && data <= 0 {
33+
return nil
34+
}
35+
limit := &relay.RelayLimit{Duration: duration, Data: data}
36+
if duration <= 0 {
37+
// The relay advertises seconds as uint32; anything larger wraps.
38+
limit.Duration = math.MaxUint32 * time.Second
39+
}
40+
if data <= 0 {
41+
limit.Data = math.MaxInt64
42+
}
43+
return limit
44+
}
45+
46+
// ByteSize is a flag value that accepts human sizes such as 128MiB.
47+
type ByteSize int64
48+
49+
func (b *ByteSize) String() string { return humanize.IBytes(uint64(*b)) }
50+
51+
func (b *ByteSize) Type() string { return "size" }
52+
53+
func (b *ByteSize) Set(s string) error {
54+
n, err := humanize.ParseBytes(s)
55+
if err != nil {
56+
return err
57+
}
58+
if n > math.MaxInt64 {
59+
return fmt.Errorf("size %q is too large", s)
60+
}
61+
*b = ByteSize(n)
62+
return nil
63+
}
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package router
16+
17+
import (
18+
"context"
19+
"io"
20+
"math"
21+
"reflect"
22+
"testing"
23+
"time"
24+
25+
"github.com/libp2p/go-libp2p"
26+
"github.com/libp2p/go-libp2p/core/host"
27+
"github.com/libp2p/go-libp2p/core/network"
28+
"github.com/libp2p/go-libp2p/core/peer"
29+
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client"
30+
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
31+
"github.com/multiformats/go-multiaddr"
32+
)
33+
34+
func TestRelayLimit(t *testing.T) {
35+
unlimitedDuration := math.MaxUint32 * time.Second
36+
for _, tc := range []struct {
37+
name string
38+
duration time.Duration
39+
data int64
40+
want *relay.RelayLimit
41+
}{
42+
{"no limits", 0, 0, nil},
43+
{"duration only", time.Hour, 0, &relay.RelayLimit{Duration: time.Hour, Data: math.MaxInt64}},
44+
{"data only", 0, 4096, &relay.RelayLimit{Duration: unlimitedDuration, Data: 4096}},
45+
{"both", time.Minute, 4096, &relay.RelayLimit{Duration: time.Minute, Data: 4096}},
46+
} {
47+
t.Run(tc.name, func(t *testing.T) {
48+
if got := relayLimit(tc.duration, tc.data); !reflect.DeepEqual(got, tc.want) {
49+
t.Errorf("relayLimit(%v, %d) = %+v, want %+v", tc.duration, tc.data, got, tc.want)
50+
}
51+
})
52+
}
53+
}
54+
55+
func TestByteSizeSet(t *testing.T) {
56+
for _, tc := range []struct {
57+
in string
58+
want ByteSize
59+
wantErr bool
60+
}{
61+
{"128MiB", 128 << 20, false},
62+
{"1GB", 1_000_000_000, false},
63+
{"0", 0, false},
64+
{"abc", 0, true},
65+
{"20EiB", 0, true},
66+
} {
67+
var got ByteSize
68+
err := got.Set(tc.in)
69+
if (err != nil) != tc.wantErr || got != tc.want {
70+
t.Errorf("Set(%q) = %d, %v; want %d, error %v", tc.in, got, err, tc.want, tc.wantErr)
71+
}
72+
}
73+
}
74+
75+
// A real circuit: the data cap truncates, and a duration-only limit must not cut.
76+
func TestRelayLimitOnCircuit(t *testing.T) {
77+
const payload = 8192
78+
for _, tc := range []struct {
79+
name string
80+
limit *relay.RelayLimit
81+
complete bool
82+
}{
83+
{"data cap cuts the circuit", relayLimit(0, 4096), false},
84+
{"duration only passes everything", relayLimit(time.Hour, 0), true},
85+
} {
86+
t.Run(tc.name, func(t *testing.T) {
87+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
88+
defer cancel()
89+
90+
relayHost := newLoopbackHost(t, libp2p.DisableRelay())
91+
if _, err := relay.New(relayHost, relay.WithLimit(tc.limit)); err != nil {
92+
t.Fatal(err)
93+
}
94+
relayInfo := peer.AddrInfo{ID: relayHost.ID(), Addrs: relayHost.Addrs()}
95+
96+
dst := newLoopbackHost(t, libp2p.EnableRelay())
97+
dst.SetStreamHandler("/test", func(s network.Stream) {
98+
_, _ = s.Write(make([]byte, payload))
99+
_ = s.Close()
100+
})
101+
if err := dst.Connect(ctx, relayInfo); err != nil {
102+
t.Fatal(err)
103+
}
104+
if _, err := client.Reserve(ctx, dst, relayInfo); err != nil {
105+
t.Fatal(err)
106+
}
107+
108+
src := newLoopbackHost(t, libp2p.EnableRelay())
109+
if err := src.Connect(ctx, relayInfo); err != nil {
110+
t.Fatal(err)
111+
}
112+
circuit, err := multiaddr.NewMultiaddr("/p2p/" + relayHost.ID().String() + "/p2p-circuit")
113+
if err != nil {
114+
t.Fatal(err)
115+
}
116+
if err := src.Connect(ctx, peer.AddrInfo{ID: dst.ID(), Addrs: []multiaddr.Multiaddr{relayHost.Addrs()[0].Encapsulate(circuit)}}); err != nil {
117+
t.Fatal(err)
118+
}
119+
120+
s, err := src.NewStream(network.WithAllowLimitedConn(ctx, "test"), dst.ID(), "/test")
121+
if err != nil {
122+
t.Fatal(err)
123+
}
124+
defer func() { _ = s.Close() }()
125+
got, _ := io.Copy(io.Discard, s)
126+
if (got == payload) != tc.complete {
127+
t.Errorf("read %d of %d bytes through the relay, want complete=%v", got, payload, tc.complete)
128+
}
129+
})
130+
}
131+
}
132+
133+
func newLoopbackHost(t *testing.T, opts ...libp2p.Option) host.Host {
134+
t.Helper()
135+
h, err := libp2p.New(append(opts, libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))...)
136+
if err != nil {
137+
t.Fatal(err)
138+
}
139+
t.Cleanup(func() { _ = h.Close() })
140+
return h
141+
}

‎internal/router/router.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,8 @@ func (r *Router) Start() error {
415415
}
416416

417417
// Setup Relay
418-
_, err = relay.New(hostNode, relay.WithACL(&relayACL{r: r}))
418+
_, err = relay.New(hostNode, relay.WithACL(&relayACL{r: r}),
419+
relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)))
419420
if err != nil {
420421
_ = hostNode.Close()
421422
return err

‎internal/standalone/standalone.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ type RouterTunables struct {
150150
// DHTProviderAddrTTL / DHTMaxRecordAge tune DHT record lifetimes.
151151
DHTProviderAddrTTL time.Duration
152152
DHTMaxRecordAge time.Duration
153+
// RelayLimitDuration / RelayLimitData cap each relayed connection.
154+
RelayLimitDuration time.Duration
155+
RelayLimitData router.ByteSize
153156
// DisallowLoopback stops advertising loopback addresses (useful on
154157
// public deployments; the default keeps local development working).
155158
DisallowLoopback bool
@@ -180,6 +183,9 @@ func (o *Options) Default() {
180183
if o.Router.ConnsPerSourceIP == 0 {
181184
o.Router.ConnsPerSourceIP = o.Router.HighWaterMark
182185
}
186+
if o.Router.RelayLimitDuration == 0 {
187+
o.Router.RelayLimitDuration = router.DefaultRelayLimitDuration
188+
}
183189
}
184190

185191
// Validate rejects option combinations Start could not honor.
@@ -350,6 +356,8 @@ func (s *Server) Start(ctx context.Context) error {
350356
HighWaterMark: s.opts.Router.HighWaterMark,
351357
DHTProviderAddrTTL: s.opts.Router.DHTProviderAddrTTL,
352358
DHTMaxRecordAge: s.opts.Router.DHTMaxRecordAge,
359+
RelayLimitDuration: s.opts.Router.RelayLimitDuration,
360+
RelayLimitData: int64(s.opts.Router.RelayLimitData),
353361
// Single-port deployments typically sit behind a TLS-terminating
354362
// proxy (Cloud Run, L7 LBs) or NAT where every peer shares a few
355363
// source IPs; libp2p's default 8-conns-per-IP cap would throttle

‎site/content/docs/reference/router.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ replica count.
4343
| `--conns-per-source-ip` | `8` (libp2p default) | Inbound connections accepted per source IP. Raise it behind a TLS-terminating proxy or a NAT that puts many peers on one address. |
4444
| `--low-watermark`, `--high-watermark` | `1000`, `4000` | Connection manager limits. Above the high mark, connections are trimmed down to the low mark. |
4545
| `--dht-provider-addr-ttl`, `--dht-max-record-age` | library defaults | DHT record lifetimes. |
46+
| `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |
4647
| `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |
4748
| `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |
4849

‎site/content/docs/reference/sam-one.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ of zero leaves the default of that component unchanged.
6464
| `--router-low-watermark`, `--router-high-watermark` | `--low-watermark`, `--high-watermark` |
6565
| `--router-conns-per-source-ip` | `--conns-per-source-ip`. `0` follows the high watermark, because peers behind a proxy share source IPs. |
6666
| `--router-dht-provider-addr-ttl`, `--router-dht-max-record-age` | The DHT record lifetimes. |
67+
| `--router-relay-limit-duration`, `--router-relay-limit-data` | `--relay-limit-duration`, `--relay-limit-data`. `0` keeps `1h` and no data limit, so set a long duration such as `24h` instead of an unlimited one. |
6768
| `--router-allow-loopback` | `--allow-loopback`. Defaults to `true` here, for a laptop. Disable it on a public deployment. |
6869

6970
## The banner

0 commit comments

Comments
 (0)