From aa34be8af8484b37beb0baffbfc86698df548f35 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Fri, 23 Oct 2026 08:16:09 +0300 Subject: [PATCH] fix(storage): never hold a read transaction across a StreamActivities send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StreamActivities walked the activity bucket inside one BBolt read transaction and sent each record into a buffered channel. #1737 made the send cancel-aware, but a consumer that stops reading without cancelling (an activity export to a client that keeps the connection open but no longer reads) still parked the producer inside db.View. The open read transaction pinned the mmap: the next write that had to grow the file waited on the remap lock for ever, and every later reader — agent-token lookups included — queued behind that writer, hanging all authenticated requests while /healthz kept answering. The producer now collects a bounded batch (at most 100 matches, at most 1000 keys examined) inside a short db.View, leaves the transaction, then sends, and resumes the walk strictly before the last examined key in a new transaction. Offset, limit and newest-first order are unchanged; records written after the stream starts are not guaranteed to appear, and records pruned between batches are skipped. Tests: the no-cancel test that asserted the leak as the contract is replaced by one that asserts a stalled consumer holds no transaction, a remap test (a 16 MiB write must not block behind a stalled stream), and batch-boundary coverage for offset/limit/order and a sparse filter past the per-transaction scan cap. --- internal/storage/activity.go | 133 ++++++++++++------ .../storage/activity_stream_cancel_test.go | 100 ++++++++++++- 2 files changed, 187 insertions(+), 46 deletions(-) diff --git a/internal/storage/activity.go b/internal/storage/activity.go index c36316d2c..0d8ecb5db 100644 --- a/internal/storage/activity.go +++ b/internal/storage/activity.go @@ -1,9 +1,9 @@ package storage import ( + "bytes" "context" "encoding/json" - "errors" "fmt" "time" @@ -289,72 +289,125 @@ func (m *Manager) AggregateToolUsage(since time.Time) (map[string]ToolUsageStat, return stats, err } +// Batch bounds for StreamActivities: each read transaction collects at most +// streamBatchSize matching records and examines at most streamScanPerTx keys, +// so a transaction stays short even when the filter matches little. +const ( + streamBatchSize = 100 + streamScanPerTx = 1000 +) + // StreamActivities returns a channel that yields activity records matching the filter. // The channel is closed when all matching records have been sent, or as soon as // ctx is done. This is useful for streaming large exports without loading all // records into memory. Respects filter.Limit and filter.Offset for bounded streaming. // -// The producer walks the activity bucket inside a BBolt read transaction and -// sends into a bounded channel. A consumer that stops reading early MUST cancel -// ctx: that is what lets the producer give up its pending send, return from -// db.View and release the read transaction (an open read transaction pins the -// mmap and blocks db.Close and remaps on later writes). Without cancellation an -// abandoned stream holds that transaction for the life of the process. +// The producer never holds a BBolt read transaction across a channel send: it +// collects a bounded batch inside a short db.View, leaves the transaction, then +// sends, and resumes the walk from the last key it examined in a new +// transaction. A consumer that stalls without cancelling therefore parks only +// the producer goroutine, not the database: an open read transaction would pin +// the mmap, so the next write that grows the file would wait on the remap for +// ever and every later reader would queue behind that writer (#1742). A +// consumer that stops reading early should still cancel ctx so the goroutine +// exits. Records written after the stream starts are not guaranteed to appear, +// and records pruned between batches are skipped. func (m *Manager) StreamActivities(ctx context.Context, filter ActivityFilter) <-chan *ActivityRecord { - ch := make(chan *ActivityRecord, 100) + ch := make(chan *ActivityRecord, streamBatchSize) go func() { defer close(ch) - err := m.db.db.View(func(tx *bbolt.Tx) error { - bucket := tx.Bucket([]byte(ActivityRecordsBucket)) - if bucket == nil { - return nil - } + var resume []byte // last key examined; the next batch starts strictly before it + skipped := 0 + collected := 0 - cursor := bucket.Cursor() - skipped := 0 - sent := 0 + for { + if ctx.Err() != nil { + return + } - for k, v := cursor.Last(); k != nil; k, v = cursor.Prev() { - var record ActivityRecord - if err := record.UnmarshalBinary(v); err != nil { - continue + batch := make([]*ActivityRecord, 0, streamBatchSize) + finished := false + err := m.db.db.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket([]byte(ActivityRecordsBucket)) + if bucket == nil { + finished = true + return nil } - if !filter.Matches(&record) { - continue + cursor := bucket.Cursor() + var k, v []byte + if resume == nil { + k, v = cursor.Last() + } else { + // Seek lands on the first key >= resume (or nil past the + // end); step back to the first key strictly before it. + k, v = cursor.Seek(resume) + if k == nil { + k, v = cursor.Last() + } + for k != nil && bytes.Compare(k, resume) >= 0 { + k, v = cursor.Prev() + } } - // Handle offset - if filter.Offset > 0 && skipped < filter.Offset { - skipped++ - continue + for examined := 0; k != nil; k, v = cursor.Prev() { + if examined >= streamScanPerTx || len(batch) >= streamBatchSize { + return nil + } + examined++ + resume = append(resume[:0], k...) + + var record ActivityRecord + if err := record.UnmarshalBinary(v); err != nil { + continue + } + + if !filter.Matches(&record) { + continue + } + + // Handle offset + if filter.Offset > 0 && skipped < filter.Offset { + skipped++ + continue + } + + // Handle limit — stop after collecting limit records + if filter.Limit > 0 && collected >= filter.Limit { + finished = true + return nil + } + + batch = append(batch, &record) + collected++ } - // Handle limit — stop after sending limit records - if filter.Limit > 0 && sent >= filter.Limit { - return nil - } + finished = true + return nil + }) + if err != nil { + m.logger.Errorw("Error streaming activities", "error", err) + return + } + for _, record := range batch { // A done context wins even when the buffer has room: select // picks a ready case at random, so check first. - if err := ctx.Err(); err != nil { - return err + if ctx.Err() != nil { + return } select { - case ch <- &record: - sent++ + case ch <- record: case <-ctx.Done(): - return ctx.Err() + return } } - return nil - }) - - if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - m.logger.Errorw("Error streaming activities", "error", err) + if finished { + return + } } }() diff --git a/internal/storage/activity_stream_cancel_test.go b/internal/storage/activity_stream_cancel_test.go index 01c357358..25b18fd50 100644 --- a/internal/storage/activity_stream_cancel_test.go +++ b/internal/storage/activity_stream_cancel_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -95,23 +96,68 @@ func TestStreamActivities_CancelReleasesReadTransaction(t *testing.T) { } } -// A consumer that walks away WITHOUT cancelling still leaks the transaction -// (that is the contract: the context is the release). This guards the test -// above against passing vacuously if the buffer ever grows past the seed. -func TestStreamActivities_AbandonedWithoutCancelHoldsTransaction(t *testing.T) { +// A consumer that stops reading WITHOUT cancelling (an export to an HTTP +// client that keeps the connection open but no longer reads) must not pin a +// read transaction either: the producer parks on the channel send outside +// db.View (#1742). +func TestStreamActivities_StalledConsumerHoldsNoTransaction(t *testing.T) { manager := newStreamTestManager(t) seedStreamRecords(t, manager, 300) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() ch := manager.StreamActivities(ctx, ActivityFilter{Types: []string{string(ActivityTypeToolCall)}}) _, ok := <-ch require.True(t, ok) time.Sleep(200 * time.Millisecond) - require.Equal(t, 1, manager.GetDB().Stats().OpenTxN, "producer should be parked on a full channel inside db.View") + require.Equal(t, 0, manager.GetDB().Stats().OpenTxN, "producer must not hold a read transaction while blocked on a send") cancel() - waitForOpenTx(t, manager, 0) + closeWithin(t, manager, 5*time.Second) +} + +// The #1742 hang: a stalled stream plus a write that must grow (remap) the +// file. With the read transaction held across the send, the writer waits for +// the mmap lock for ever and every later reader queues behind it. +func TestStreamActivities_StalledConsumerDoesNotBlockRemap(t *testing.T) { + manager := newStreamTestManager(t) + seedStreamRecords(t, manager, 300) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch := manager.StreamActivities(ctx, ActivityFilter{Types: []string{string(ActivityTypeToolCall)}}) + _, ok := <-ch + require.True(t, ok) + time.Sleep(100 * time.Millisecond) // let the producer fill the buffer and park + + db := manager.GetDB() + grow := make([]byte, 16<<20) // larger than the current mmap: forces a remap + done := make(chan error, 1) + go func() { + done <- db.Update(func(tx *bbolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("remap-probe")) + if err != nil { + return err + } + return b.Put([]byte("k"), grow) + }) + }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + cancel() // unwedge the writer so the package run can finish + <-done + t.Fatal("a write needing a remap blocked behind a stalled activity stream") + } + + // The stream resumes where it stopped and loses nothing across batches. + n := 1 + for range ch { + n++ + } + require.Equal(t, 300, n, "stream must deliver every record across batches") closeWithin(t, manager, 5*time.Second) } @@ -145,3 +191,45 @@ func TestStreamActivities_LiveContextStreamsAll(t *testing.T) { waitForOpenTx(t, manager, 0) closeWithin(t, manager, 5*time.Second) } + +// Offset, limit and order hold across batch boundaries, and a filter that +// matches rarely still finds every match past the per-transaction scan cap. +func TestStreamActivities_BatchBoundaries(t *testing.T) { + manager := newStreamTestManager(t) + const total = 2500 // > streamScanPerTx, > several streamBatchSize + seedStreamRecords(t, manager, total) + + collect := func(f ActivityFilter) []string { + var ids []string + for rec := range manager.StreamActivities(context.Background(), f) { + ids = append(ids, rec.ID) + } + return ids + } + id := func(i int) string { return fmt.Sprintf("stream-cancel-%04d", i) } + + all := collect(ActivityFilter{}) + require.Len(t, all, total) + for i, got := range all { + require.Equal(t, id(total-1-i), got, "newest first, no gaps or repeats") + } + + page := collect(ActivityFilter{Offset: 150, Limit: 230}) + require.Len(t, page, 230) + require.Equal(t, id(total-1-150), page[0]) + require.Equal(t, id(total-1-150-229), page[229]) + + // One match near each end of a 2500-key walk: the scan cap must not end + // the stream early. + sparse := collect(ActivityFilter{Types: []string{string(ActivityTypeToolCall)}, RequestID: "none"}) + require.Empty(t, sparse) + require.NoError(t, manager.SaveActivity(&ActivityRecord{ + ID: "oldest-match", Type: ActivityTypeToolCall, ServerName: "rare", ToolName: "t", + Status: ActivityStatusSuccess, Timestamp: time.Now().UTC().Add(-2 * time.Hour), + })) + rare := collect(ActivityFilter{Server: "rare"}) + require.Equal(t, []string{"oldest-match"}, rare) + + waitForOpenTx(t, manager, 0) + closeWithin(t, manager, 5*time.Second) +}