Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 93 additions & 40 deletions internal/storage/activity.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package storage

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"time"

Expand Down Expand Up @@ -289,72 +289,125 @@ func (m *Manager) AggregateToolUsage(since time.Time) (map[string]ToolUsageStat,
return stats, err
}

// Batch bounds for StreamActivities: each read transaction collects at most
// streamBatchSize matching records and examines at most streamScanPerTx keys,
// so a transaction stays short even when the filter matches little.
const (
streamBatchSize = 100
streamScanPerTx = 1000
)

// StreamActivities returns a channel that yields activity records matching the filter.
// The channel is closed when all matching records have been sent, or as soon as
// ctx is done. This is useful for streaming large exports without loading all
// records into memory. Respects filter.Limit and filter.Offset for bounded streaming.
//
// The producer walks the activity bucket inside a BBolt read transaction and
// sends into a bounded channel. A consumer that stops reading early MUST cancel
// ctx: that is what lets the producer give up its pending send, return from
// db.View and release the read transaction (an open read transaction pins the
// mmap and blocks db.Close and remaps on later writes). Without cancellation an
// abandoned stream holds that transaction for the life of the process.
// The producer never holds a BBolt read transaction across a channel send: it
// collects a bounded batch inside a short db.View, leaves the transaction, then
// sends, and resumes the walk from the last key it examined in a new
// transaction. A consumer that stalls without cancelling therefore parks only
// the producer goroutine, not the database: an open read transaction would pin
// the mmap, so the next write that grows the file would wait on the remap for
// ever and every later reader would queue behind that writer (#1742). A
// consumer that stops reading early should still cancel ctx so the goroutine
// exits. Records written after the stream starts are not guaranteed to appear,
// and records pruned between batches are skipped.
func (m *Manager) StreamActivities(ctx context.Context, filter ActivityFilter) <-chan *ActivityRecord {
ch := make(chan *ActivityRecord, 100)
ch := make(chan *ActivityRecord, streamBatchSize)

go func() {
defer close(ch)

err := m.db.db.View(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(ActivityRecordsBucket))
if bucket == nil {
return nil
}
var resume []byte // last key examined; the next batch starts strictly before it
skipped := 0
collected := 0

cursor := bucket.Cursor()
skipped := 0
sent := 0
for {
if ctx.Err() != nil {
return
}

for k, v := cursor.Last(); k != nil; k, v = cursor.Prev() {
var record ActivityRecord
if err := record.UnmarshalBinary(v); err != nil {
continue
batch := make([]*ActivityRecord, 0, streamBatchSize)
finished := false
err := m.db.db.View(func(tx *bbolt.Tx) error {
bucket := tx.Bucket([]byte(ActivityRecordsBucket))
if bucket == nil {
finished = true
return nil
}

if !filter.Matches(&record) {
continue
cursor := bucket.Cursor()
var k, v []byte
if resume == nil {
k, v = cursor.Last()
} else {
// Seek lands on the first key >= resume (or nil past the
// end); step back to the first key strictly before it.
k, v = cursor.Seek(resume)
if k == nil {
k, v = cursor.Last()
}
for k != nil && bytes.Compare(k, resume) >= 0 {
k, v = cursor.Prev()
}
}

// Handle offset
if filter.Offset > 0 && skipped < filter.Offset {
skipped++
continue
for examined := 0; k != nil; k, v = cursor.Prev() {
if examined >= streamScanPerTx || len(batch) >= streamBatchSize {
return nil
}
examined++
resume = append(resume[:0], k...)

var record ActivityRecord
if err := record.UnmarshalBinary(v); err != nil {
continue
}

if !filter.Matches(&record) {
continue
}

// Handle offset
if filter.Offset > 0 && skipped < filter.Offset {
skipped++
continue
}

// Handle limit — stop after collecting limit records
if filter.Limit > 0 && collected >= filter.Limit {
finished = true
return nil
}

batch = append(batch, &record)
collected++
}

// Handle limit — stop after sending limit records
if filter.Limit > 0 && sent >= filter.Limit {
return nil
}
finished = true
return nil
})
if err != nil {
m.logger.Errorw("Error streaming activities", "error", err)
return
}

for _, record := range batch {
// A done context wins even when the buffer has room: select
// picks a ready case at random, so check first.
if err := ctx.Err(); err != nil {
return err
if ctx.Err() != nil {
return
}
select {
case ch <- &record:
sent++
case ch <- record:
case <-ctx.Done():
return ctx.Err()
return
}
}

return nil
})

if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
m.logger.Errorw("Error streaming activities", "error", err)
if finished {
return
}
}
}()

Expand Down
100 changes: 94 additions & 6 deletions internal/storage/activity_stream_cancel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

"github.com/stretchr/testify/require"
"go.etcd.io/bbolt"
"go.uber.org/zap"
)

Expand Down Expand Up @@ -95,23 +96,68 @@ func TestStreamActivities_CancelReleasesReadTransaction(t *testing.T) {
}
}

// A consumer that walks away WITHOUT cancelling still leaks the transaction
// (that is the contract: the context is the release). This guards the test
// above against passing vacuously if the buffer ever grows past the seed.
func TestStreamActivities_AbandonedWithoutCancelHoldsTransaction(t *testing.T) {
// A consumer that stops reading WITHOUT cancelling (an export to an HTTP
// client that keeps the connection open but no longer reads) must not pin a
// read transaction either: the producer parks on the channel send outside
// db.View (#1742).
func TestStreamActivities_StalledConsumerHoldsNoTransaction(t *testing.T) {
manager := newStreamTestManager(t)
seedStreamRecords(t, manager, 300)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ch := manager.StreamActivities(ctx, ActivityFilter{Types: []string{string(ActivityTypeToolCall)}})
_, ok := <-ch
require.True(t, ok)

time.Sleep(200 * time.Millisecond)
require.Equal(t, 1, manager.GetDB().Stats().OpenTxN, "producer should be parked on a full channel inside db.View")
require.Equal(t, 0, manager.GetDB().Stats().OpenTxN, "producer must not hold a read transaction while blocked on a send")

cancel()
waitForOpenTx(t, manager, 0)
closeWithin(t, manager, 5*time.Second)
}

// The #1742 hang: a stalled stream plus a write that must grow (remap) the
// file. With the read transaction held across the send, the writer waits for
// the mmap lock for ever and every later reader queues behind it.
func TestStreamActivities_StalledConsumerDoesNotBlockRemap(t *testing.T) {
manager := newStreamTestManager(t)
seedStreamRecords(t, manager, 300)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ch := manager.StreamActivities(ctx, ActivityFilter{Types: []string{string(ActivityTypeToolCall)}})
_, ok := <-ch
require.True(t, ok)
time.Sleep(100 * time.Millisecond) // let the producer fill the buffer and park

db := manager.GetDB()
grow := make([]byte, 16<<20) // larger than the current mmap: forces a remap
done := make(chan error, 1)
go func() {
done <- db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("remap-probe"))
if err != nil {
return err
}
return b.Put([]byte("k"), grow)
})
}()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(5 * time.Second):
cancel() // unwedge the writer so the package run can finish
<-done
t.Fatal("a write needing a remap blocked behind a stalled activity stream")
}

// The stream resumes where it stopped and loses nothing across batches.
n := 1
for range ch {
n++
}
require.Equal(t, 300, n, "stream must deliver every record across batches")
closeWithin(t, manager, 5*time.Second)
}

Expand Down Expand Up @@ -145,3 +191,45 @@ func TestStreamActivities_LiveContextStreamsAll(t *testing.T) {
waitForOpenTx(t, manager, 0)
closeWithin(t, manager, 5*time.Second)
}

// Offset, limit and order hold across batch boundaries, and a filter that
// matches rarely still finds every match past the per-transaction scan cap.
func TestStreamActivities_BatchBoundaries(t *testing.T) {
manager := newStreamTestManager(t)
const total = 2500 // > streamScanPerTx, > several streamBatchSize
seedStreamRecords(t, manager, total)

collect := func(f ActivityFilter) []string {
var ids []string
for rec := range manager.StreamActivities(context.Background(), f) {
ids = append(ids, rec.ID)
}
return ids
}
id := func(i int) string { return fmt.Sprintf("stream-cancel-%04d", i) }

all := collect(ActivityFilter{})
require.Len(t, all, total)
for i, got := range all {
require.Equal(t, id(total-1-i), got, "newest first, no gaps or repeats")
}

page := collect(ActivityFilter{Offset: 150, Limit: 230})
require.Len(t, page, 230)
require.Equal(t, id(total-1-150), page[0])
require.Equal(t, id(total-1-150-229), page[229])

// One match near each end of a 2500-key walk: the scan cap must not end
// the stream early.
sparse := collect(ActivityFilter{Types: []string{string(ActivityTypeToolCall)}, RequestID: "none"})
require.Empty(t, sparse)
require.NoError(t, manager.SaveActivity(&ActivityRecord{
ID: "oldest-match", Type: ActivityTypeToolCall, ServerName: "rare", ToolName: "t",
Status: ActivityStatusSuccess, Timestamp: time.Now().UTC().Add(-2 * time.Hour),
}))
rare := collect(ActivityFilter{Server: "rare"})
require.Equal(t, []string{"oldest-match"}, rare)

waitForOpenTx(t, manager, 0)
closeWithin(t, manager, 5*time.Second)
}
Loading