diff --git a/.gitignore b/.gitignore index 143198a..b40c1f6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ vendor/ # Internal planning docs (not OSS-ready) /PRD-*.md dist/ + +# Local benchmark data, profiles, and Go caches +/.cache/ diff --git a/Makefile b/Makefile index 691b817..56551e7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-verify run test docker lint clean +.PHONY: build build-verify run test docker lint clean bench-e2e bench-e2e-open-loop build: CGO_ENABLED=0 go build -o bin/agentgate ./cmd/agentgw @@ -12,6 +12,17 @@ run: test: CGO_ENABLED=0 go test -v -count=1 ./... +BENCHTIME ?= 20000x +BENCH_FILTER ?= ^BenchmarkHandleActE2E$$ + +bench-e2e-open-loop: BENCH_FILTER = ^BenchmarkHandleActOpenLoop$$ + +bench-e2e bench-e2e-open-loop: + mkdir -p .cache/tmp .cache/go-tmp .cache/go-build + TMPDIR="$(CURDIR)/.cache/tmp" GOTMPDIR="$(CURDIR)/.cache/go-tmp" \ + GOCACHE="$(CURDIR)/.cache/go-build" GOMAXPROCS=4 \ + go test -tags=bench ./bench -run '^$$' -bench '$(BENCH_FILTER)' -benchtime=$(BENCHTIME) -count=1 + docker: docker-compose up --build diff --git a/bench/RESULTS.md b/bench/RESULTS.md new file mode 100644 index 0000000..dea317e --- /dev/null +++ b/bench/RESULTS.md @@ -0,0 +1,366 @@ +# Phase 0: end-to-end act overhead + +**Verdict: FAIL overall. The `<1 ms p99` exit gate is not met across the tested load levels.** + +Only the original concurrency-1 run with a 0 ms upstream passes. +All 6 fixed-arrival-rate scenarios also miss the target. +No gateway production code or tuning changed. +These results don't establish performance on an actual 4-vCPU machine. + +## Verdict by load level + +| Load level | 0 ms upstream | 5 ms upstream | +| --- | --- | --- | +| Original closed-loop c1 | PASS, p99 0.3410 ms | FAIL, p99 2.078 ms | +| Open-loop 25%, 5,403.5 requests/s | FAIL, p99 6.996 ms | FAIL, p99 2.329 ms | +| Open-loop 50%, 10,807 requests/s | FAIL, p99 111.029 ms | FAIL, p99 17.977 ms | +| Open-loop 75%, 16,210.5 requests/s | FAIL, p99 37.918 ms | FAIL, p99 298.834 ms | +| Original closed-loop c16 | FAIL, p99 3.418 ms | FAIL, p99 4.277 ms | +| Original closed-loop c64 | FAIL, p99 6.217 ms | FAIL, p99 4.006 ms | + +The original c16 and c64 runs are saturation probes, not measurements of isolated incremental processing cost. +Their loaded tail latency includes queueing. +The fixed-rate runs retain queueing but stop coupling the next arrival to the previous completion. +No tested fixed rate establishes a passing load range on this host. +The percentage denominator is the highest original c64 throughput, 21,614 requests/s. +It isn't a measured CPU-utilization percentage or a guaranteed capacity under a different workload. +In particular, the original 5 ms c64 run reached only 10,270 requests/s. +The common 50% and 75% offered rates exceed that separate closed-loop throughput. + +## Host and revision + +Measured on 2026-10-18. + +| Item | Value | +| --- | --- | +| Gateway base | `origin/main`, `2f81c85a2364622fb462f1cf5a646db476028eb7` | +| Benchmark branch | `perf/phase0-exit-bench` | +| `sysctl machdep.cpu.brand_string` | `Apple M4` | +| `sysctl hw.ncpu` | `10` | +| `sysctl hw.memsize` | `17179869184` bytes, 16 GiB | +| `sw_vers` | macOS `27.0.1`, build `26A434` | +| `go version` | `go version go1.26.0 darwin/arm64` | +| `GOMAXPROCS` | `4` | +| Storage | Real SQLite files on the host filesystem, under `.cache/tmp` | + +`GOMAXPROCS=4` emulates 4 concurrent Go execution slots on Apple M4. +It doesn't reserve CPUs, impose VM quotas, or isolate external processes. +The client and mock share those slots with the gateway. +This is a local development host, not a dedicated Linux performance runner. +The original closed-loop table is unchanged from commit `a7a1a8c588029a47ee0ad85ebc9d927eedd28e27`. +The open-loop and wakeup comparison were added afterward on the same host. +Only 1 complete measured run was recorded per final scenario. No repeated-run confidence intervals are available. + +## Measurement + +`BenchmarkHandleActE2E` uses the real `gateway.Server` over an `httptest.NewServer` loopback listener. +It exercises authentication and authorization, then proxies to a second loopback listener. +The stack uses: + +- A fresh on-disk SQLite database and every real migration. +- A real HMAC `auth.KeyStore`, with its default 30s cache TTL. +- A real encrypted SQLite vault behind `vault.CachedStore`, with its default 60s cache TTL. +- A real signing-key store and receipt `Ledger`. +- Default group commit: 256 maximum receipts per transaction, with no collection delay. +- The production upstream pool, with no custom client or transport injected. +- Active key, tenant, and service rate buckets, each configured for 1,000,000 requests/s and a 1,000,000 burst. +- A tenant inflight limit of 1,024. + +There is 1 scoped agent key, 1 bearer token, and 1 service with a POST action. +The request is about 100 bytes. The upstream response is `{"ok":true}`. +The token is unexpired. OAuth refresh and delegation aren't part of this workload. +Info logging still performs JSON encoding and discards its output. +The added benchmark harness retains error-level logs for failure diagnosis. +Terminal, file, and remote log-sink costs are excluded. +The gateway's default upstream idle-per-host limit remains 32, including at concurrency 64. + +Each scenario runs 256 warmup requests, then 20,000 measured requests. +Workers issue requests without think time, with at most 1 outstanding request per worker. +This is closed-loop load. It doesn't model fixed arrival rates or avoid coordinated omission. +Warmup establishes credentials and connections. Cache TTL refreshes can still occur during longer runs. +All measured requests succeeded. Each completed request had an acknowledged signed receipt before its response. +Checks confirm receipt count, sequential head, successful status, policy, and signature length. + +The per-request client timer starts immediately before `http.Client.Do`. +It ends after reading and closing the entire gateway response body. +Request creation and client-side response JSON decoding are outside that timer. +Each upstream handler measures its actual elapsed duration through its response write. +A buffered channel per worker associates this duration with that worker's outstanding request. +The channel operation itself isn't subtracted. + +```text +overhead[i] = client_elapsed[i] - measured_upstream_handler_elapsed[i] +``` + +Subtraction happens before sorting. Percentiles use the nearest-rank definition. +Upstream sleep overshoot is measured, not replaced with the requested 5 ms delay. +Client overhead, both loopback paths, and gateway queueing remain in the estimate. +Response flushing after the upstream handler returns also remains in the estimate. +These numbers are conservative end-to-end overhead estimates, not isolated handler CPU times. +Throughput is successful measured requests divided by measured wall time, including upstream time. +Go's `ns/op` is aggregate wall time per operation. It isn't per-request latency under concurrency. + +## Original closed-loop results + +All latency values are milliseconds. Throughput is requests/s. +The sample count is 20,000 in every row. + +| Upstream delay | Concurrency | Overhead p50 | Overhead p90 | Overhead p99 | Overhead p99.9 | Throughput | `<1 ms p99` | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 0 ms | 1 | 0.1453 | 0.1849 | 0.3410 | 2.210 | 5,918 | PASS | +| 0 ms | 16 | 0.7516 | 1.198 | 3.418 | 4.315 | 17,832 | FAIL | +| 0 ms | 64 | 2.716 | 3.857 | 6.217 | 7.084 | 21,614 | FAIL | +| 5 ms | 1 | 0.5777 | 0.8749 | 2.078 | 6.938 | 146.4 | FAIL | +| 5 ms | 16 | 1.139 | 1.698 | 4.277 | 5.216 | 2,505 | FAIL | +| 5 ms | 64 | 0.8710 | 2.166 | 4.006 | 5.186 | 10,270 | FAIL | + +The measured upstream means show why subtracting a nominal delay would be misleading. + +| Upstream delay | Concurrency | Measured upstream mean, ms | Total request p99, ms | +| --- | ---: | ---: | ---: | +| 0 ms | 1 | 0.002202 | 0.3458 | +| 0 ms | 16 | 0.001793 | 3.419 | +| 0 ms | 64 | 0.001480 | 6.222 | +| 5 ms | 1 | 6.158 | 8.678 | +| 5 ms | 16 | 5.194 | 9.424 | +| 5 ms | 64 | 5.059 | 9.091 | + +The total request p99 isn't used to calculate overhead p99. +Different requests can occupy those percentile ranks. +At 20,000 requests, p99.9 describes about 20 tail samples. +Treat it as a tail indicator, not a precise service-level guarantee. + +## Fixed-arrival-rate results + +`BenchmarkHandleActOpenLoop` offers evenly spaced arrivals, independent of earlier request completion. +Both delays use the same reference throughput: 21,614 requests/s from the original 0 ms c64 run. +The offered rates are exactly 5,403.5, 10,807, and 16,210.5 requests/s. +Each row has 256 warmup requests and 20,000 measured requests. +Every measured request completed successfully, with its signed receipt acknowledged. +No scheduled requests were dropped. + +```text +scheduled[i] = epoch + i / offered_rate +overhead[i] = response_body_completed[i] - scheduled[i] - measured_upstream_handler_elapsed[i] +``` + +Each request gets a unique upstream timing slot. +Late arrivals keep their original timestamp. The schedule never resets or skips work to catch up. +This avoids coordinated omission from waiting for previous completions. +Client request creation and connection-pool waiting also remain in scheduled latency. +The load generator yields with `runtime.Gosched` while waiting for the next deadline. +It avoids sub-millisecond sleeps, but competes within the same `GOMAXPROCS=4` budget. +This isn't an isolated external load generator. + +All latency values below are milliseconds. +Goodput includes the final queue drain, so it can fall below the offered rate. +Every row fails the `<1 ms p99` gate. + +| Delay | Reference load | Offered requests/s | Overhead p50 | Overhead p90 | Overhead p99 | Overhead p99.9 | Goodput, requests/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 ms | 25% | 5,403.5 | 0.2388 | 1.490 | 6.996 | 18.072 | 5,403 | +| 0 ms | 50% | 10,807 | 1.014 | 73.218 | 111.029 | 115.567 | 10,459 | +| 0 ms | 75% | 16,210.5 | 2.116 | 22.192 | 37.918 | 41.142 | 15,787 | +| 5 ms | 25% | 5,403.5 | 0.1698 | 0.5189 | 2.329 | 2.998 | 5,393 | +| 5 ms | 50% | 10,807 | 0.5364 | 3.153 | 17.977 | 29.768 | 10,777 | +| 5 ms | 75% | 16,210.5 | 198.188 | 285.943 | 298.834 | 302.655 | 13,004 | + +The generator records lateness through the actual `http.Client.Do` invocation. +Connection-pool waiting occurs after that point. It remains in overhead, not in the lateness column below. + +| Delay | Reference load | Client-start lateness p99, ms | Measured upstream mean, ms | Total scheduled latency p99, ms | +| --- | ---: | ---: | ---: | ---: | +| 0 ms | 25% | 0.2471 | 0.002873 | 6.999 | +| 0 ms | 50% | 0.2990 | 0.003080 | 111.044 | +| 0 ms | 75% | 0.5686 | 0.002674 | 37.921 | +| 5 ms | 25% | 0.01906 | 5.009 | 7.332 | +| 5 ms | 50% | 0.2614 | 5.081 | 23.886 | +| 5 ms | 75% | 1.523 | 5.343 | 303.992 | + +The client permits up to 128 TCP connections. Pending requests aren't capped or dropped. +The original 64-worker limit doesn't constrain arrivals. +Client-pool and gateway queues remain visible in each request's scheduled latency. +This connection limit can affect congested rows, so these aren't isolated server-service-time measurements. +The gateway's upstream pool and admission settings remain unchanged. + +An initial harness allowed 1,024 client connections and encountered loopback connection resets during bursts. +Those incomplete runs aren't valid percentile measurements and aren't used above. +Bounding connection creation avoids that local failure without reducing the scheduled request count. +The completed run took 15.030s across the 6 scenarios. +The non-monotonic tails show run sensitivity. They don't justify interpolating a passing load level. + +## The 5 ms c1 tail + +`BenchmarkHandleActWakeup` compares sleeping and busy-spinning inside the otherwise identical upstream handler. +Both variants use concurrency 1, a 5 ms requested delay, and 20,000 measured requests. +Actual upstream elapsed time is still subtracted per request. +Neither variant changes gateway code, receipt behavior, or rate limits. + +All latency values below are milliseconds. + +| Upstream wait | Overhead p50 | Overhead p90 | Overhead p99 | Overhead p99.9 | Upstream mean | Throughput, requests/s | Gate | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| Sleep, repeated diagnostic | 0.5247 | 0.8235 | 1.855 | 4.459 | 6.175 | 147.6 | FAIL | +| Busy spin | 0.3280 | 0.4522 | 1.371 | 4.945 | 5.005 | 185.7 | FAIL | + +The comparison took 249.651s. The original sleeping c1 row remains visible above. +Busy spin lowers p99 by about 26%, but it still misses the gate. +Its p99.9 is worse, so this isn't evidence that spinning fixes tail latency. +The actual sleep overshoot was already subtracted. It doesn't directly account for the remaining overhead. +macOS idle wakeup, scheduling, or power-state changes remain hypotheses, not established causes. +Busy spin keeps a Go execution slot active and changes the host's execution pattern. +That prevents this comparison from isolating idle wakeup alone. +Execution traces and power-state measurements are needed before attributing the remaining tail. +Don't subtract an assumed platform penalty from the reported gate result. + +## Commands + +Run from the repository root. +Scratch files, Go caches, and profiles stay inside ignored `.cache/`. + +```sh +sysctl machdep.cpu.brand_string hw.ncpu hw.memsize +sw_vers +go version + +make bench-e2e +``` + +The Makefile target defaults to 20,000 requests per scenario and `GOMAXPROCS=4`. +It sets `TMPDIR`, `GOTMPDIR`, and `GOCACHE` inside `.cache/`. +Its Go command is: + +```sh +GOMAXPROCS=4 go test -tags=bench ./bench -run '^$' \ + -bench '^BenchmarkHandleActE2E$' -benchtime=20000x -count=1 +``` + +The benchmark passed its functional checks in 156.182s. +Passing those checks doesn't mean the latency target passed. +For a quick functional run, use `make bench-e2e BENCHTIME=64x`. +Don't use that small sample for tail claims. +Select a scenario with `BENCH_FILTER`, for example: + +```sh +make bench-e2e BENCH_FILTER='^BenchmarkHandleActE2E/delay=0s/c=64$$' +``` + +The doubled dollar sign preserves the regex anchor through Make expansion. + +For the new fixed-arrival matrix: + +```sh +make bench-e2e-open-loop +``` + +This retains the existing `bench-e2e` target and its closed-loop defaults. +The new target uses the same cache paths, `GOMAXPROCS=4`, and 20,000-request default. +Its Go command is: + +```sh +GOMAXPROCS=4 go test -tags=bench ./bench -run '^$' \ + -bench '^BenchmarkHandleActOpenLoop$' -benchtime=20000x -count=1 +``` + +For the sleep versus spin comparison: + +```sh +make bench-e2e BENCH_FILTER='^BenchmarkHandleActWakeup$$' +``` + +Its Go command is: + +```sh +GOMAXPROCS=4 go test -tags=bench ./bench -run '^$' \ + -bench '^BenchmarkHandleActWakeup$' -benchtime=20000x -count=1 +``` + +## Failure profile + +The profile targets the failed 0 ms, concurrency-64 scenario. +It uses 200,000 requests to collect more samples. +These instrumented measurements aren't used in the results table. +Setup, warmup, and untimed benchmark checks also appear in the process-wide CPU profile. +The larger run also grows the receipt database further. + +```sh +mkdir -p .cache/tmp .cache/go-tmp .cache/go-build +export TMPDIR="$PWD/.cache/tmp" +export GOTMPDIR="$PWD/.cache/go-tmp" +export GOCACHE="$PWD/.cache/go-build" +export GOMAXPROCS=4 + +go test -tags=bench ./bench -run '^$' \ + -bench '^BenchmarkHandleActE2E/delay=0s/c=64$' \ + -benchtime=200000x -count=1 \ + -cpuprofile .cache/act-c64.cpu -o .cache/bench.test +go tool pprof -top -nodecount=15 .cache/bench.test .cache/act-c64.cpu +go tool pprof -top -cum -nodecount=80 .cache/bench.test .cache/act-c64.cpu +``` + +Profile duration was 12.34s, with 23.08s of attributed samples. +The top 5 flat costs were: + +| Function | Flat samples | Share | What to investigate | +| --- | ---: | ---: | --- | +| `syscall.rawsyscalln` | 20.63s | 89.38% | Socket and SQLite system calls | +| `runtime.kevent` | 0.75s | 3.25% | Network poller activity | +| `runtime.pthread_kill` | 0.31s | 1.34% | Runtime signals and preemption | +| `runtime.usleep` | 0.30s | 1.30% | Runtime scheduling waits | +| `runtime.pthread_cond_wait` | 0.23s | 1.00% | Runtime thread parking | + +The cumulative view attributes 37.05% to `receipt.(*Ledger).commitBatch`, including its children. +SQLite execution accounts for 36.66%, including 26.30% under `receipt.insertReceipt`. +Socket writes account for 32.58%. Socket reads account for 20.62%. +SQLite page reads account for 19.41% under `_unixRead`. +These cumulative paths overlap. Their percentages must not be added. +The signing callback encloses database work. Its cumulative share doesn't isolate Ed25519 signing cost. +macOS syscall attribution doesn't prove where wall-clock tail latency originates. +The profile includes the in-process client and upstream, not just gateway CPU work. + +### Follow-ups, without tuning this change + +1. Repeat on a dedicated 4-vCPU Linux runner, then run k6 with a separate load generator. +2. Capture execution traces and block profiles to separate receipt queueing from scheduler stalls. +3. Inspect group-commit batch sizes and SQLite page I/O under sustained ledger growth. +4. Measure socket and connection-pool costs, including the default 32 idle connections at concurrency 64. +5. Investigate timer and scheduler jitter in the 5 ms workload. Preserve measured upstream subtraction. + +Separate client-pool waits from gateway and receipt queueing before attributing the open-loop tails to one component. +Keep synchronous receipt acknowledgment and durability guarantees unchanged during any later tuning. + +## k6 status + +`which k6` found no executable. k6 wasn't installed. +**k6 performance numbers are pending.** +`node --input-type=module --check < bench/k6/act.js` passed. +That validates JavaScript syntax only, not k6 execution. +The script includes a `<1 ms p99` overhead threshold and failure checks. +See [local setup](k6/README.md) for the mock upstream and real gateway provisioning commands. +An HTTP smoke check verified that provisioning flow against both local server binaries. +Both servers were stopped afterward. + +## Validation + +With the cache variables above and `GOMAXPROCS=4`: + +| Command | Result | +| --- | --- | +| `go build ./...` | PASS | +| `go vet ./...` | PASS | +| `go vet -tags=bench ./bench` | PASS | +| `go test -race ./...` | PASS, all normal packages | +| `go test -tags=bench ./bench ./bench/mockupstream -run Test -count=1` | PASS | +| `make bench-e2e BENCHTIME=64x` | PASS, 6 functional scenarios | +| `make bench-e2e` | PASS, 120,000 measured requests; latency target FAIL | +| `make bench-e2e-open-loop` | PASS, 120,000 measured requests; all 6 latency gates FAIL | +| `make bench-e2e BENCH_FILTER='^BenchmarkHandleActWakeup$$'` | PASS, 40,000 measured requests; both latency gates FAIL | +| Profile command above | PASS | +| `node --input-type=module --check < bench/k6/act.js` | PASS, syntax only | + +The `bench` build tag excludes the heavy benchmark from normal race-test runs. +Mock-upstream tests remain included in `go test -race ./...`. +Build, both vet commands, race tests, and tagged helper tests were rerun after the open-loop addition. +The existing closed-loop 64-request smoke matrix was rerun and passed. +The full closed-loop table and CPU profile remain the original measurements, not new reruns. +No `.cache/` files belong in the commit. diff --git a/bench/e2e_test.go b/bench/e2e_test.go new file mode 100644 index 0000000..bbf8edd --- /dev/null +++ b/bench/e2e_test.go @@ -0,0 +1,359 @@ +//go:build bench + +package bench + +import ( + "bytes" + "context" + "crypto/rand" + "database/sql" + "encoding/json" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/Clawdlinux/agentgate/internal/auth" + agentgatedb "github.com/Clawdlinux/agentgate/internal/db" + "github.com/Clawdlinux/agentgate/internal/gateway" + "github.com/Clawdlinux/agentgate/internal/ratelimit" + "github.com/Clawdlinux/agentgate/internal/receipt" + "github.com/Clawdlinux/agentgate/internal/registry" + "github.com/Clawdlinux/agentgate/internal/signer" + "github.com/Clawdlinux/agentgate/internal/vault" +) + +const warmupRequests = 256 + +type stack struct { + db *sql.DB + ledger *receipt.Ledger + client *http.Client + url string + key string + upstream []chan time.Duration +} + +type stackOptions struct { + timingSlots int + spinUpstream bool +} + +type benchmarkLogWriter struct { + b *testing.B +} + +func (w benchmarkLogWriter) Write(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"level":"ERROR"`)) { + w.b.Logf("gateway error: %s", data) + } + return len(data), nil +} + +func newStack(b *testing.B, concurrency int, delay time.Duration) *stack { + b.Helper() + return newStackWithOptions(b, concurrency, delay, stackOptions{}) +} + +func newStackWithOptions(b *testing.B, concurrency int, delay time.Duration, options stackOptions) *stack { + b.Helper() + database, err := agentgatedb.Open(filepath.Join(b.TempDir(), "agentgate.db")) + must(b, err) + b.Cleanup(func() { database.Close() }) + must(b, agentgatedb.RunMigrations(database)) + + masterKey := make([]byte, 32) + _, err = rand.Read(masterKey) + must(b, err) + pepper, err := auth.DeriveKeyPepper(masterKey) + must(b, err) + keys, err := auth.NewKeyStore(database, pepper) + must(b, err) + _, plaintext, err := keys.Create(context.Background(), "e2e-bench", []string{"mock"}, []string{"bench-user"}) + must(b, err) + store, err := vault.NewSQLiteStore(database, masterKey) + must(b, err) + cached, err := vault.NewCachedStore(store) + must(b, err) + token := make([]byte, 32) + _, err = rand.Read(token) + must(b, err) + credential := fmt.Sprintf("%x", token) + must(b, cached.Put("bench-user", "mock", vault.Token{AccessToken: credential, TokenType: "Bearer"})) + + signers, err := signer.NewStore(database, masterKey) + must(b, err) + b.Cleanup(signers.Close) + _, signingKey, err := signers.LoadOrCreateActive(1) + clear(signingKey) + must(b, err) + ledger := receipt.NewLedger(database, signers) + b.Cleanup(ledger.Close) + + timings := make([]chan time.Duration, max(concurrency, options.timingSlots)) + for i := range timings { + timings[i] = make(chan time.Duration, 1) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + var params struct { + Worker int `json:"worker"` + } + if r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer "+credential { + http.Error(w, "unexpected upstream request", http.StatusBadRequest) + return + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil || params.Worker < 0 || params.Worker >= len(timings) { + http.Error(w, "invalid worker", http.StatusBadRequest) + return + } + // Closed-loop workers serialize requests. Open-loop requests each have their own slot. + defer func() { timings[params.Worker] <- time.Since(start) }() + if delay > 0 { + if options.spinUpstream { + until := time.Now().Add(delay) + for time.Now().Before(until) { + } + } else { + time.Sleep(delay) + } + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"ok":true}`) + })) + b.Cleanup(upstream.Close) + + reg := registry.New() + must(b, reg.LoadBytes([]byte(fmt.Sprintf(`services: + mock: + base_url: %s + auth: + type: bearer + rate_limit: + requests_per_second: 1000000 + burst: 1000000 + actions: + echo: + method: POST + path: /echo + params: + worker: int +`, upstream.URL)))) + limiter := ratelimit.New(reg.RateLimits(), ratelimit.WithSettings(ratelimit.Settings{ + KeyRPS: 1e6, KeyBurst: 1e6, TenantRPS: 1e6, TenantBurst: 1e6, TenantMaxInflight: 1024, + })) + handler := gateway.New(gateway.Config{ + Registry: reg, Vault: cached, Authorizer: keys, Receipts: ledger, Limiter: limiter, + Logger: slog.New(slog.NewJSONHandler(benchmarkLogWriter{b}, &slog.HandlerOptions{Level: slog.LevelInfo})), + }) + b.Cleanup(handler.CloseIdleConnections) + server := httptest.NewServer(handler) + b.Cleanup(server.Close) + transport := &http.Transport{ + MaxIdleConns: concurrency, MaxIdleConnsPerHost: concurrency, MaxConnsPerHost: concurrency, + } + b.Cleanup(transport.CloseIdleConnections) + return &stack{ + db: database, ledger: ledger, client: &http.Client{Transport: transport, Timeout: 10 * time.Second}, + url: server.URL + "/v1/act", key: plaintext, upstream: timings, + } +} + +type sample struct { + total time.Duration + upstream time.Duration + dispatchLag time.Duration +} + +func (s *stack) act(ctx context.Context, worker int, body string) (sample, error) { + return s.actAt(ctx, worker, body, time.Time{}) +} + +func (s *stack) actAt(ctx context.Context, worker int, body string, scheduled time.Time) (sample, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.url, strings.NewReader(body)) + if err != nil { + return sample{}, err + } + req.Header.Set("Authorization", "Bearer "+s.key) + req.Header.Set("Content-Type", "application/json") + start := time.Now() + resp, err := s.client.Do(req) + if err != nil { + return sample{}, err + } + data, err := io.ReadAll(resp.Body) + resp.Body.Close() + finished := time.Now() + total := finished.Sub(start) + if err != nil { + return sample{}, err + } + if resp.StatusCode != http.StatusOK { + return sample{}, fmt.Errorf("act status %d: %s", resp.StatusCode, data) + } + var envelope struct { + Status int `json:"status"` + Body struct { + OK bool `json:"ok"` + } `json:"body"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return sample{}, err + } + if envelope.Status != http.StatusOK || !envelope.Body.OK { + return sample{}, fmt.Errorf("unexpected act response: %s", data) + } + select { + case duration := <-s.upstream[worker]: + if duration > total { + return sample{}, fmt.Errorf("upstream time %s exceeds total %s", duration, total) + } + return timedSample(scheduled, start, finished, duration), nil + case <-ctx.Done(): + return sample{}, ctx.Err() + } +} + +func timedSample(scheduled, started, finished time.Time, upstream time.Duration) sample { + if scheduled.IsZero() { + scheduled = started + } + return sample{ + total: finished.Sub(scheduled), upstream: upstream, dispatchLag: started.Sub(scheduled), + } +} + +func (s *stack) run(count, concurrency int, samples []sample) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errors := make(chan error, concurrency) + start := make(chan struct{}) + var ready, done sync.WaitGroup + for worker := range min(count, concurrency) { + ready.Add(1) + done.Add(1) + go func() { + defer done.Done() + body := fmt.Sprintf(`{"service":"mock","action":"echo","on_behalf_of":"bench-user","params":{"worker":%d}}`, worker) + ready.Done() + <-start + for i := worker; i < count; i += concurrency { + result, err := s.act(ctx, worker, body) + if err != nil { + errors <- err + cancel() + return + } + if samples != nil { + samples[i] = result + } + } + }() + } + ready.Wait() + close(start) + done.Wait() + close(errors) + for err := range errors { + return err + } + return nil +} + +func BenchmarkHandleActE2E(b *testing.B) { + if runtime.GOMAXPROCS(0) != 4 { + b.Fatal("run with GOMAXPROCS=4 for the Phase 0 comparison") + } + for _, delay := range []time.Duration{0, 5 * time.Millisecond} { + b.Run("delay="+delay.String(), func(b *testing.B) { + for _, concurrency := range []int{1, 16, 64} { + b.Run(fmt.Sprintf("c=%d", concurrency), func(b *testing.B) { + benchmarkAct(b, concurrency, delay) + }) + } + }) + } +} + +func benchmarkAct(b *testing.B, concurrency int, delay time.Duration) { + s := newStack(b, concurrency, delay) + must(b, s.run(warmupRequests, concurrency, nil)) + samples := make([]sample, b.N) + b.ResetTimer() + start := time.Now() + err := s.run(b.N, concurrency, samples) + elapsed := time.Since(start) + b.StopTimer() + must(b, err) + s.checkReceipts(b, warmupRequests+b.N) + reportSamples(b, samples, elapsed) +} + +func (s *stack) checkReceipts(b *testing.B, expected int) { + b.Helper() + var count, invalid int + must(b, s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM( + CASE WHEN status_code != 200 OR policy_decision != 'allow' OR error_code != '' + OR length(signature) != 64 THEN 1 ELSE 0 END), 0) FROM receipts`).Scan(&count, &invalid)) + head, _, err := s.ledger.Head(context.Background()) + must(b, err) + if count != expected || head != uint64(count) || invalid != 0 { + b.Fatalf("receipt mismatch: count=%d head=%d invalid=%d expected=%d", count, head, invalid, expected) + } +} + +func reportSamples(b *testing.B, samples []sample, elapsed time.Duration) { + b.Helper() + overheads := make([]time.Duration, b.N) + totals := make([]time.Duration, b.N) + var upstreamTotal time.Duration + for i, result := range samples { + overheads[i] = result.total - result.upstream + totals[i] = result.total + upstreamTotal += result.upstream + } + slices.Sort(overheads) + slices.Sort(totals) + for _, p := range []float64{50, 90, 99, 99.9} { + b.ReportMetric(float64(percentile(overheads, p))/float64(time.Microsecond), fmt.Sprintf("p%g-overhead-us", p)) + } + b.ReportMetric(float64(percentile(totals, 99))/float64(time.Microsecond), "p99-total-us") + b.ReportMetric(float64(upstreamTotal)/float64(b.N)/float64(time.Microsecond), "upstream-mean-us") + b.ReportMetric(float64(b.N)/elapsed.Seconds(), "req/s") +} + +// percentile uses the nearest rank. Subtract upstream time before sorting, never between independently sorted percentiles. +func percentile(sorted []time.Duration, p float64) time.Duration { + return sorted[max(0, int(math.Ceil(p*float64(len(sorted))/100))-1)] +} + +func must(b *testing.B, err error) { + b.Helper() + if err != nil { + b.Fatal(err) + } +} + +func TestPercentile(t *testing.T) { + samples := make([]time.Duration, 1000) + for i := range samples { + samples[i] = time.Duration(i + 1) + } + for _, p := range []float64{50, 90, 99, 99.9} { + if got, want := percentile(samples, p), time.Duration(math.Round(p*10)); got != want { + t.Errorf("p%g: got %d, want %d", p, got, want) + } + } + if got := percentile([]time.Duration{42}, 99.9); got != 42 { + t.Errorf("single sample: got %d, want 42", got) + } +} diff --git a/bench/k6/README.md b/bench/k6/README.md new file mode 100644 index 0000000..93e5b9e --- /dev/null +++ b/bench/k6/README.md @@ -0,0 +1,111 @@ +# Local act load test + +Run only against your local gateway. These limits are unsafe for production. +The script tests 1 service, 1 HMAC agent key, and 1 cached bearer token. +Every successful action still commits a signed receipt. + +## Start the mock upstream + +Run commands from the repository root. Keep each server in its own terminal. +Keep caches and scratch data inside the worktree: + +```sh +mkdir -p .cache/tmp .cache/go-tmp .cache/go-build +export TMPDIR="$PWD/.cache/tmp" +export GOTMPDIR="$PWD/.cache/go-tmp" +export GOCACHE="$PWD/.cache/go-build" +export GOMAXPROCS=4 +go run ./bench/mockupstream -addr 127.0.0.1:9090 -delay 0 +``` + +Check readiness from another terminal: + +```sh +curl --fail http://127.0.0.1:9090/healthz +``` + +Restart with `-delay 5ms` for the second scenario. +The mock returns measured handler nanoseconds inside its JSON body. +The gateway wraps this body without changing those measurements. +Sleep overshoot is measured rather than assumed. + +## Start the gateway + +Set the same cache variables in this terminal. +Generate local secrets. Don't commit them. + +```sh +export AGENTGATE_VAULT_KEY="$(openssl rand -hex 16)" +export AGENTGATE_ADMIN_SECRET="$(openssl rand -hex 24)" +export AGENTGATE_RATE_KEY_RPS=1000000 +export AGENTGATE_RATE_KEY_BURST=1000000 +export AGENTGATE_RATE_TENANT_RPS=1000000 +export AGENTGATE_RATE_TENANT_BURST=1000000 +export AGENTGATE_TENANT_MAX_INFLIGHT=1024 +GOMAXPROCS=4 go run ./cmd/agentgw \ + --config bench/services.yaml --db .cache/k6-agentgate.db --addr 127.0.0.1:8080 +``` + +Save `AGENTGATE_ADMIN_SECRET` for the next terminal. +The first startup prints a bootstrap agent key. You can ignore it and create a scoped benchmark key. +Keep logs private. The startup message contains that bootstrap credential. +SQLite uses real files, migrations, and the default receipt group-commit settings. +The gateway's upstream pool keeps its production defaults. +Info logging remains enabled. Your terminal or log sink can affect these results. + +## Provision credentials + +Export the same `AGENTGATE_ADMIN_SECRET` in this terminal. +The following commands need `curl`, `openssl`, and `python3`. + +```sh +export MOCK_TOKEN="$(openssl rand -hex 16)" +curl --fail-with-body -sS http://127.0.0.1:8080/admin/tokens \ + -H "X-Admin-Secret: $AGENTGATE_ADMIN_SECRET" \ + -H 'Content-Type: application/json' \ + -d "{\"user_id\":\"bench-user\",\"service\":\"mock\",\"access_token\":\"$MOCK_TOKEN\"}" + +export AGENTGATE_AGENT_KEY="$( + curl --fail-with-body -sS http://127.0.0.1:8080/admin/keys \ + -H "X-Admin-Secret: $AGENTGATE_ADMIN_SECRET" \ + -H 'Content-Type: application/json' \ + -d '{"name":"bench","allowed_services":["mock"],"allowed_users":["bench-user"]}' \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["key"])' +)" +``` + +## Run k6 + +```sh +which k6 +for vus in 1 16 64; do + VUS="$vus" DURATION=60s k6 run \ + --summary-export ".cache/k6-delay0-c${vus}.json" bench/k6/act.js +done +``` + +Restart the mock with `-delay 5ms`. Repeat with summary filenames containing `delay5ms`. +The `<1 ms p99` threshold returns a nonzero exit status when missed. +The script also fails on unsuccessful requests or missing timing samples. +No deliberate client sleep occurs between requests. This is closed-loop concurrency, not fixed arrival rate. +Use 20,000 or more successful requests per scenario before interpreting p99.9. + +`gateway_overhead_ms` subtracts each measured upstream duration before aggregating percentiles. +It uses k6 `http_req_duration`: request sending, response waiting, and response receiving. +Connection setup and blocked time are separate k6 metrics. They aren't included in this estimate. +The mock timestamp excludes response encoding and writing. Those costs stay in the overhead estimate. +These are conservative estimates, not exact replicas of the Go benchmark's handler-completion timestamps. +The setup request validates credentials. Initial per-VU connections remain in the measured k6 run. +Keep the load generator separate when benchmarking an actual 4-vCPU server. + +## Validate without k6 + +Don't install k6 just to validate syntax. +Node can parse the module without resolving k6 imports: + +```sh +node --input-type=module --check < bench/k6/act.js +``` + +This checks syntax only. It doesn't execute k6 imports, scenarios, or thresholds. +See [the recorded benchmark](../RESULTS.md) for measured Go results and the k6 status. diff --git a/bench/k6/act.js b/bench/k6/act.js new file mode 100644 index 0000000..3f592cd --- /dev/null +++ b/bench/k6/act.js @@ -0,0 +1,81 @@ +import http from 'k6/http'; +import { check, fail } from 'k6'; +import { Trend } from 'k6/metrics'; + +const overhead = new Trend('gateway_overhead_ms', true); +const upstream = new Trend('upstream_handler_ms', true); +const baseURL = (__ENV.BASE_URL || 'http://127.0.0.1:8080').replace(/\/$/, ''); +const key = __ENV.AGENTGATE_AGENT_KEY; +const vus = Number(__ENV.VUS || 16); + +if (!key) { + throw new Error('AGENTGATE_AGENT_KEY is required'); +} +if (!Number.isInteger(vus) || vus < 1) { + throw new Error('VUS must be a positive integer'); +} + +export const options = { + scenarios: { + act: { + executor: 'constant-vus', + vus, + duration: __ENV.DURATION || '30s', + gracefulStop: '10s', + }, + }, + summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(99)', 'p(99.9)', 'max'], + thresholds: { + http_req_failed: ['rate==0'], + checks: ['rate==1'], + gateway_overhead_ms: ['p(99)<1'], + }, +}; + +function act(worker) { + return http.post(`${baseURL}/v1/act`, JSON.stringify({ + service: 'mock', + action: 'echo', + on_behalf_of: 'bench-user', + params: { worker }, + }), { + headers: { + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json', + }, + timeout: '10s', + tags: { name: 'POST /v1/act' }, + }); +} + +export function setup() { + const response = act(0); + if (response.status !== 200) { + fail(`warmup failed: status ${response.status}, body ${response.body}`); + } +} + +export default function () { + const response = act(__VU); + let body; + try { + body = response.json(); + } catch (_) { + check(response, { 'valid JSON envelope': () => false }); + return; + } + const valid = check(response, { + 'act and upstream succeeded': (r) => r.status === 200 && body.status === 200 && body.body?.ok === true, + 'upstream timing present': () => Number.isFinite(body.body?.upstream_ns) && body.body.upstream_ns >= 0, + }); + if (!valid) { + return; + } + const upstreamMS = body.body.upstream_ns / 1e6; + const overheadMS = response.timings.duration - upstreamMS; + if (!check(overheadMS, { 'non-negative overhead': (value) => value >= 0 })) { + return; + } + upstream.add(upstreamMS); + overhead.add(overheadMS); +} diff --git a/bench/mockupstream/main.go b/bench/mockupstream/main.go new file mode 100644 index 0000000..6c5f591 --- /dev/null +++ b/bench/mockupstream/main.go @@ -0,0 +1,56 @@ +// Command mockupstream serves a local upstream for the act load test. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "time" +) + +func main() { + addr := flag.String("addr", "127.0.0.1:9090", "loopback listen address") + delay := flag.Duration("delay", 0, "fixed upstream delay, for example 5ms") + flag.Parse() + if *delay < 0 { + log.Fatal("delay must be non-negative") + } + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "ok\n") + }) + mux.HandleFunc("POST /echo", echo(*delay)) + log.Printf("mock upstream on %s, delay %s", *addr, *delay) + server := &http.Server{Addr: *addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + log.Fatal(server.ListenAndServe()) +} + +func echo(delay time.Duration) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + var params json.RawMessage + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(¶ms); err != nil { + http.Error(w, "invalid JSON params", http.StatusBadRequest) + return + } + if r.Header.Get("Authorization") == "" { + http.Error(w, "missing injected token", http.StatusUnauthorized) + return + } + if delay > 0 { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-r.Context().Done(): + return + } + } + w.Header().Set("Content-Type", "application/json") + // Exclude response encoding and writing from this timestamp. Their cost stays in the overhead estimate. + fmt.Fprintf(w, `{"ok":true,"upstream_ns":%d}`, time.Since(start).Nanoseconds()) + } +} diff --git a/bench/mockupstream/main_test.go b/bench/mockupstream/main_test.go new file mode 100644 index 0000000..894fbde --- /dev/null +++ b/bench/mockupstream/main_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestEcho(t *testing.T) { + for _, delay := range []time.Duration{0, 5 * time.Millisecond} { + t.Run(delay.String(), func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/echo", strings.NewReader(`{"worker":1}`)) + req.Header.Set("Authorization", "Bearer test-only") + response := httptest.NewRecorder() + echo(delay).ServeHTTP(response, req) + var body struct { + OK bool `json:"ok"` + UpstreamNS int64 `json:"upstream_ns"` + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if response.Code != http.StatusOK || !body.OK || body.UpstreamNS < delay.Nanoseconds() { + t.Fatalf("unexpected response: code=%d body=%+v", response.Code, body) + } + }) + } +} + +func TestEchoRejectsInvalidRequests(t *testing.T) { + for _, tc := range []struct { + name, body, auth string + status int + }{ + {"invalid JSON", "{", "Bearer test-only", http.StatusBadRequest}, + {"missing token", "{}", "", http.StatusUnauthorized}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/echo", strings.NewReader(tc.body)) + req.Header.Set("Authorization", tc.auth) + response := httptest.NewRecorder() + echo(0).ServeHTTP(response, req) + if response.Code != tc.status { + t.Fatalf("status=%d, want %d", response.Code, tc.status) + } + }) + } +} diff --git a/bench/open_loop_test.go b/bench/open_loop_test.go new file mode 100644 index 0000000..6024a1e --- /dev/null +++ b/bench/open_loop_test.go @@ -0,0 +1,144 @@ +//go:build bench + +package bench + +import ( + "context" + "fmt" + "runtime" + "slices" + "sync" + "testing" + "time" +) + +// measuredC64MaxRPS is the highest c64 throughput from the original unprofiled run. +// Keep this fixed so reruns don't silently change their offered load. +const measuredC64MaxRPS = 21614 + +func arrivalOffset(index int, rate float64) time.Duration { + return time.Duration(float64(index) * float64(time.Second) / rate) +} + +func (s *stack) runOpenLoop(count int, rate float64, samples []sample) (time.Duration, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Every request gets a fixed deadline. Neither completion nor lateness shifts subsequent arrivals. + epoch := time.Now().Add(10 * time.Millisecond) + errors := make(chan error, count) + var done sync.WaitGroup + for i := range count { + scheduled := epoch.Add(arrivalOffset(i, rate)) + // Yield instead of sleeping at sub-millisecond intervals. Dispatch lag remains part of the measured latency. + for time.Now().Before(scheduled) { + runtime.Gosched() + } + done.Add(1) + go func() { + defer done.Done() + body := fmt.Sprintf(`{"service":"mock","action":"echo","on_behalf_of":"bench-user","params":{"worker":%d}}`, i) + result, err := s.actAt(ctx, i, body, scheduled) + if err != nil { + errors <- err + cancel() + return + } + samples[i] = result + }() + } + done.Wait() + elapsed := time.Since(epoch) + close(errors) + for err := range errors { + return elapsed, err + } + return elapsed, nil +} + +func BenchmarkHandleActOpenLoop(b *testing.B) { + if runtime.GOMAXPROCS(0) != 4 { + b.Fatal("run with GOMAXPROCS=4 for the Phase 0 comparison") + } + for _, delay := range []time.Duration{0, 5 * time.Millisecond} { + b.Run("delay="+delay.String(), func(b *testing.B) { + for _, load := range []int{25, 50, 75} { + rate := float64(measuredC64MaxRPS) * float64(load) / 100 + b.Run(fmt.Sprintf("load=%d/rps=%g", load, rate), func(b *testing.B) { + // Bound TCP connection bursts, not arrivals. Client-pool queueing remains in scheduled latency. + s := newStackWithOptions(b, 128, delay, stackOptions{timingSlots: b.N}) + must(b, s.run(warmupRequests, 64, nil)) + samples := make([]sample, b.N) + b.ResetTimer() + elapsed, err := s.runOpenLoop(b.N, rate, samples) + b.StopTimer() + must(b, err) + s.checkReceipts(b, warmupRequests+b.N) + reportSamples(b, samples, elapsed) + lags := make([]time.Duration, b.N) + for i, result := range samples { + lags[i] = result.dispatchLag + } + slices.Sort(lags) + b.ReportMetric(float64(percentile(lags, 99))/float64(time.Microsecond), "p99-dispatch-lag-us") + b.ReportMetric(rate, "offered-req/s") + }) + } + }) + } +} + +func BenchmarkHandleActWakeup(b *testing.B) { + if runtime.GOMAXPROCS(0) != 4 { + b.Fatal("run with GOMAXPROCS=4 for the Phase 0 comparison") + } + for _, mode := range []string{"sleep", "spin"} { + b.Run(mode, func(b *testing.B) { + s := newStackWithOptions(b, 1, 5*time.Millisecond, stackOptions{spinUpstream: mode == "spin"}) + must(b, s.run(warmupRequests, 1, nil)) + samples := make([]sample, b.N) + b.ResetTimer() + start := time.Now() + err := s.run(b.N, 1, samples) + elapsed := time.Since(start) + b.StopTimer() + must(b, err) + s.checkReceipts(b, warmupRequests+b.N) + reportSamples(b, samples, elapsed) + }) + } +} + +func TestArrivalOffset(t *testing.T) { + for _, tc := range []struct { + index int + rate float64 + want time.Duration + }{ + {0, 5403.5, 0}, + {54035, 5403.5, 10 * time.Second}, + {10807, 10807, time.Second}, + {32421, 16210.5, 2 * time.Second}, + } { + if got := arrivalOffset(tc.index, tc.rate); got != tc.want { + t.Errorf("index=%d rate=%g: got %s, want %s", tc.index, tc.rate, got, tc.want) + } + } + // Offsets are computed from the epoch, without accumulating a truncated interval. + rate := 16210.5 + if got := arrivalOffset(1000000, rate); got != time.Duration(1000000*float64(time.Second)/rate) { + t.Errorf("accumulated interval rounding: %s", got) + } +} + +func TestScheduledSampleIncludesDispatchDelay(t *testing.T) { + epoch := time.Unix(0, 0) + result := timedSample(epoch, epoch.Add(20*time.Millisecond), epoch.Add(27*time.Millisecond), 5*time.Millisecond) + if result.total != 27*time.Millisecond || result.total-result.upstream != 22*time.Millisecond || + result.dispatchLag != 20*time.Millisecond { + t.Fatalf("scheduled latency omitted delay: %+v", result) + } + closed := timedSample(time.Time{}, epoch.Add(20*time.Millisecond), epoch.Add(27*time.Millisecond), 5*time.Millisecond) + if closed.total != 7*time.Millisecond || closed.dispatchLag != 0 { + t.Fatalf("closed-loop timing changed: %+v", closed) + } +} diff --git a/bench/services.yaml b/bench/services.yaml new file mode 100644 index 0000000..4cc316c --- /dev/null +++ b/bench/services.yaml @@ -0,0 +1,14 @@ +services: + mock: + base_url: http://127.0.0.1:9090 + auth: + type: bearer + rate_limit: + requests_per_second: 1000000 + burst: 1000000 + actions: + echo: + method: POST + path: /echo + params: + worker: int