Skip to content

Commit d1277dc

Browse files
tamirmsclaude
andcommitted
runcontainer: count or test before allocating in run-bitmap intersections
runContainer16.andBitmapContainer materialized the run as an 8 KiB bitmap container and then intersected, so an empty or array-sized result paid for a bitmap it never returned. It now counts the result with the existing andBitmapContainerCardinality and builds only that: an empty array container when the intersection is empty, an array of the right size when it is small, and the bitmap only when the result is one. A run with more than 64 intervals is instead copied into a scratch bitmap on the stack, counted there, and extracted in one pass, if it covers more than 4096 values or the CPU has the vector fill that makes extracting cheap; otherwise counting interval by interval is cheaper. An array result of 1024 values or more is extracted with that vector fill where there is one, as bitmapContainer.andBitmap already does. bitmapContainer.iandRun16 did the same conversion on the argument side. It now counts the same way, returns a small result as an array built from the run's intervals, and otherwise clears the gaps between intervals in place; past the same bounds the run is built in stack scratch and ANDed in one flat pass. resetBitmapRange, which clears those gaps, now clears whole words with the clear builtin rather than one at a time. bitmapContainer.iand returns the bitmap itself for a full run instead of a copy, and runContainer16.iand returns the run itself for a full argument instead of intersecting with it. runContainer16.intersects computed rc.and(a) and tested the result for emptiness, allocating a container to answer a boolean; every mixed pairing with a run reached it through the array and bitmap dispatchers. It now walks the run's intervals against the other container, each returning at the first shared value: a masked word scan for bitmaps, andArray's merge for arrays, or a search per value when the intervals far outnumber the values, and an interval overlap test for runs. BenchmarkRunAndBitmap (added): And, in-place And and Intersects between a run-optimized bitmap and a dense one over a single key. The in-place cases intersect a fresh copy each time, made outside the timer, and the clone+ cases include the copy. Xeon 8375C: master this change And, empty result 2.8 µs 8328 B/op 4 allocs 594 ns 104 B/op 2 allocs And, array result 4.0 µs 14504 B/op 8 allocs 2.3 µs 6280 B/op 6 allocs And, bitmap result 3.7 µs 16560 B/op 8 allocs 2.0 µs 8336 B/op 6 allocs And, 1024-interval run, array 10.0 µs 12456 B/op 8 allocs 8.0 µs 4232 B/op 6 allocs And, 1024-interval run, bitmap 9.2 µs 16560 B/op 8 allocs 7.1 µs 8336 B/op 6 allocs in-place And, bitmap result 2.2 µs 8224 B/op 2 allocs 475 ns 0 B/op 0 allocs in-place And, 1024-interval run 8.2 µs 8224 B/op 2 allocs 5.2 µs 0 B/op 0 allocs copy + in-place And, bitmap result 3.7 µs 16628 B/op 9 allocs 2.1 µs 8404 B/op 7 allocs copy + in-place And, 1024 intervals 9.0 µs 16628 B/op 9 allocs 6.8 µs 8404 B/op 7 allocs Intersects, disjoint 2.8 µs 8248 B/op 3 allocs 118 ns 0 B/op 0 allocs Intersects, overlapping 3.8 µs 14392 B/op 4 allocs 9 ns 0 B/op 0 allocs Intersects, few values, 1024 intervals 1.2 µs 32 B/op 2 allocs 60 ns 0 B/op 0 allocs Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0198cmGsvonq19wTyzKBsw2Z
1 parent 142f290 commit d1277dc

4 files changed

Lines changed: 494 additions & 12 deletions

File tree

‎bitmapcontainer.go‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,11 @@ func bitmapEquals(a, b []uint64) bool {
318318
// is the array-to-bitmap conversion point and leaves a comfortable margin.
319319
const bitmapContainerVectorFillMinCardinality = 4096
320320

321+
// arrayVectorFillMinCardinality is where decoding a whole key with the vector
322+
// fill beats decoding only the words that hold the values. Measured crossover
323+
// is near 1000 values.
324+
const arrayVectorFillMinCardinality = 1024
325+
321326
func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
322327
if useVectorFill && bc.cardinality >= bitmapContainerVectorFillMinCardinality {
323328
return fillLeastSignificant16bitsVector(bc.bitmap, x, i, mask)
@@ -809,16 +814,28 @@ func (bc *bitmapContainer) iand(a container) container {
809814
return bc.iandBitmap(x)
810815
case *runContainer16:
811816
if x.isFull() {
812-
return bc.clone()
817+
return bc
813818
}
814819
return bc.iandRun16(x)
815820
}
816821
panic("unsupported container type")
817822
}
818823

819824
func (bc *bitmapContainer) iandRun16(rc *runContainer16) container {
820-
rcb := newBitmapContainerFromRun(rc)
821-
return bc.iandBitmap(rcb)
825+
if rc.andInScratch() {
826+
var scratch [bitmapContainerSize]uint64
827+
for i := range rc.iv {
828+
setBitmapRange(scratch[:], int(rc.iv[i].start), int(rc.iv[i].last())+1)
829+
}
830+
return bc.iandBitmap(&bitmapContainer{bitmap: scratch[:]})
831+
}
832+
card := rc.andBitmapContainerCardinality(bc)
833+
if card <= arrayDefaultMaxSize {
834+
return rc.andBitmapArray(bc, card)
835+
}
836+
clearBitmapGaps(bc.bitmap, rc.iv, 0, maxCapacity)
837+
bc.cardinality = card
838+
return bc
822839
}
823840

824841
func (bc *bitmapContainer) iandArray(ac *arrayContainer) container {
@@ -850,6 +867,26 @@ func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int {
850867
return pos
851868
}
852869

870+
func (bc *bitmapContainer) intersectsRange(start, end uint) bool {
871+
if start >= end {
872+
return false
873+
}
874+
firstword, endword := start/64, (end-1)/64
875+
lo, hi := ^uint64(0)<<(start%64), ^uint64(0)>>((64-end)&63)
876+
if firstword == endword {
877+
return bc.bitmap[firstword]&lo&hi != 0
878+
}
879+
if bc.bitmap[firstword]&lo != 0 || bc.bitmap[endword]&hi != 0 {
880+
return true
881+
}
882+
for _, w := range bc.bitmap[firstword+1 : endword] {
883+
if w != 0 {
884+
return true
885+
}
886+
}
887+
return false
888+
}
889+
853890
func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int {
854891
if start >= end {
855892
return 0

‎runbitmap_and_test.go‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
package roaring
2+
3+
import (
4+
"math/rand"
5+
"testing"
6+
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
// randomRun builds a canonical run container: up to n sorted intervals of
11+
// at most maxLen values separated by gaps of at most maxGap, optionally
12+
// touching 0 and 65535.
13+
func randomRun(rng *rand.Rand, n, maxLen, maxGap int, touchEdges bool) *runContainer16 {
14+
rc := &runContainer16{}
15+
pos := 0
16+
if !touchEdges {
17+
pos = rng.Intn(100)
18+
}
19+
for i := 0; i < n && pos < 65536; i++ {
20+
length := min(1+rng.Intn(1+rng.Intn(maxLen)), 65536-pos)
21+
rc.iv = append(rc.iv, interval16{start: uint16(pos), length: uint16(length - 1)})
22+
pos += length + 1 + rng.Intn(maxGap)
23+
}
24+
if last := int(rc.maximum()); touchEdges && last != 65535 {
25+
if start := 65535 - rng.Intn(500); last+2 <= start {
26+
rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(65535 - start)})
27+
}
28+
}
29+
return rc
30+
}
31+
32+
// randomBitmapContainer ANDs ands random words per position, so the density
33+
// is 1/2^ands.
34+
func randomBitmapContainer(rng *rand.Rand, ands int) *bitmapContainer {
35+
bc := newBitmapContainer()
36+
for i := range bc.bitmap {
37+
w := rng.Uint64()
38+
for k := 1; k < ands; k++ {
39+
w &= rng.Uint64()
40+
}
41+
bc.bitmap[i] = w
42+
}
43+
bc.computeCardinality()
44+
return bc
45+
}
46+
47+
func requireAndResult(t *testing.T, want, got container) {
48+
t.Helper()
49+
require.True(t, want.equals(got), "want %d values, got %d", want.getCardinality(), got.getCardinality())
50+
_, isArray := got.(*arrayContainer)
51+
require.Equal(t, got.getCardinality() <= arrayDefaultMaxSize, isArray, "container kind must follow its cardinality")
52+
}
53+
54+
// The reference is the old path: materialize the run, then the unchanged
55+
// bitmap-bitmap intersection. Runs of one to hundreds of intervals check
56+
// both the per-interval and the scratch paths, against every density.
57+
func TestRunAndBitmapContainer(t *testing.T) {
58+
rng := rand.New(rand.NewSource(1))
59+
regular := func(stride, length int) *runContainer16 {
60+
rc := &runContainer16{}
61+
for start := 0; start+length <= 65536; start += stride {
62+
rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(length - 1)})
63+
}
64+
return rc
65+
}
66+
runs := []*runContainer16{regular(32, 8), regular(64, 1), regular(3, 2), regular(200, 100)}
67+
for i := 0; i < 120; i++ {
68+
runs = append(runs, randomRun(rng, 1+rng.Intn(40), 3000, 2000, i%5 == 0), randomRun(rng, 65+rng.Intn(400), 60, 120, false))
69+
}
70+
// Only values in the gaps of regular(32, 8), so that run's result is empty.
71+
gaps := newBitmapContainer()
72+
for v := 0; v < 65536; v++ {
73+
if v%32 >= 8 {
74+
gaps.iadd(uint16(v))
75+
}
76+
}
77+
for i, rc := range runs {
78+
for _, bc := range []*bitmapContainer{randomBitmapContainer(rng, 6), randomBitmapContainer(rng, 2), randomBitmapContainer(rng, 1), newBitmapContainerwithRange(0, 65535), gaps} {
79+
want := newBitmapContainerFromRun(rc).andBitmap(bc)
80+
requireAndResult(t, want, rc.and(bc))
81+
requireAndResult(t, want, bc.and(rc))
82+
requireAndResult(t, want, bc.clone().iand(rc))
83+
require.Equal(t, !want.isEmpty(), rc.intersects(bc), "run %d", i)
84+
}
85+
}
86+
}
87+
88+
func TestRunAndBitmapContainerEdges(t *testing.T) {
89+
empty := newBitmapContainer()
90+
full := newBitmapContainerwithRange(0, 65535)
91+
single := &runContainer16{iv: []interval16{{start: 100, length: 9}}}
92+
edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}}
93+
atThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize - 1}}}
94+
pastThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize}}}
95+
for _, rc := range []*runContainer16{single, edges, atThreshold, pastThreshold} {
96+
for _, bc := range []*bitmapContainer{empty, full} {
97+
want := newBitmapContainerFromRun(rc).andBitmap(bc)
98+
requireAndResult(t, want, rc.and(bc))
99+
requireAndResult(t, want, bc.clone().iand(rc))
100+
}
101+
}
102+
// Exactly 4096 values through the scratch path: every even value below
103+
// 16384 against the values below 8192.
104+
evens := &runContainer16{}
105+
for v := 0; v < 4*arrayDefaultMaxSize; v += 2 {
106+
evens.iv = append(evens.iv, interval16{start: uint16(v)})
107+
}
108+
half := newBitmapContainerwithRange(0, 2*arrayDefaultMaxSize-1)
109+
requireAndResult(t, newBitmapContainerFromRun(evens).andBitmap(half), evens.and(half))
110+
}
111+
112+
// An empty intersection allocates only its empty array container, and an
113+
// in-place And with a full run and Intersects allocate nothing; the old paths
114+
// allocated an 8 KiB bitmap first.
115+
func TestRunAndBitmapContainerAllocations(t *testing.T) {
116+
rc := &runContainer16{iv: []interval16{{start: 0, length: 999}, {start: 30000, length: 999}}}
117+
bc := newBitmapContainerwithRange(2000, 20000)
118+
require.LessOrEqual(t, testing.AllocsPerRun(100, func() {
119+
if !rc.and(bc).isEmpty() {
120+
t.Fatal("expected empty")
121+
}
122+
}), 1.0)
123+
// The clone costs two allocations, the empty result one.
124+
require.LessOrEqual(t, testing.AllocsPerRun(100, func() {
125+
_ = bc.clone().iand(rc)
126+
}), 3.0)
127+
full := newRunContainer16Range(0, MaxUint16)
128+
require.Zero(t, testing.AllocsPerRun(100, func() { _ = bc.iand(full) }))
129+
require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.iand(full) }))
130+
for _, c := range []container{bc, newArrayContainerRange(2000, 3000), &runContainer16{iv: []interval16{{start: 5000, length: 100}}}} {
131+
require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.intersects(c) }), "%T", c)
132+
}
133+
}
134+
135+
// The reference is the old definition, !rc.and(c).isEmpty().
136+
func TestRunContainerIntersects(t *testing.T) {
137+
rng := rand.New(rand.NewSource(11))
138+
for trial := 0; trial < 400; trial++ {
139+
rc := randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%7 == 0)
140+
ac := newArrayContainer()
141+
for range rng.Intn(300) {
142+
ac.iadd(uint16(rng.Intn(65536)))
143+
}
144+
others := []container{randomBitmapContainer(rng, 1+trial%3*3), ac, randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%5 == 0)}
145+
for _, c := range others {
146+
want := !rc.and(c).isEmpty()
147+
require.Equal(t, want, rc.intersects(c), "trial %d run vs %T", trial, c)
148+
require.Equal(t, want, c.intersects(rc), "trial %d %T vs run", trial, c)
149+
}
150+
}
151+
// A few values against many intervals take the search branch; the
152+
// reference is the same definition.
153+
many := randomRun(rng, 300, 60, 120, false)
154+
require.Greater(t, len(many.iv), 200)
155+
for trial := 0; trial < 200; trial++ {
156+
few := newArrayContainer()
157+
for range 1 + rng.Intn(6) {
158+
few.iadd(uint16(rng.Intn(65536)))
159+
}
160+
want := !many.and(few).isEmpty()
161+
require.Equal(t, want, many.intersects(few), "trial %d few vs many", trial)
162+
require.Equal(t, want, few.intersects(many), "trial %d many vs few", trial)
163+
}
164+
// Edges: empty array, touching intervals, single values at both ends.
165+
empty := newArrayContainer()
166+
edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}}
167+
require.False(t, edges.intersects(empty))
168+
one := newArrayContainer()
169+
one.iadd(65535)
170+
require.True(t, edges.intersects(one))
171+
touching := &runContainer16{iv: []interval16{{start: 1, length: 65533}}}
172+
require.False(t, edges.intersects(touching))
173+
require.True(t, edges.intersects(newBitmapContainerwithRange(0, 0)))
174+
}
175+
176+
// runBitmap holds one run container at key 0 covering n intervals of the
177+
// given length, spaced stride apart.
178+
func runBitmap(n, length, stride int) *Bitmap {
179+
bm := New()
180+
for i := 0; i < n; i++ {
181+
bm.AddRange(uint64(i*stride), uint64(i*stride+length))
182+
}
183+
bm.RunOptimize()
184+
return bm
185+
}
186+
187+
// BenchmarkRunAndBitmap intersects a run-optimized bitmap with a dense one
188+
// over a single key through the public API, one case per result kind, plus
189+
// the in-place form, alone and with the copy it needs, and the Intersects test.
190+
func BenchmarkRunAndBitmap(b *testing.B) {
191+
rng := rand.New(rand.NewSource(7))
192+
short := runBitmap(20, 300, 3000) // 6000 values
193+
long := runBitmap(4, 10000, 16000) // 40000 values
194+
many := runBitmap(1024, 8, 64) // 8192 values in 1024 intervals
195+
gaps, quarter, half, dense := New(), New(), New(), New()
196+
for v := uint32(0); v < 65536; v++ {
197+
if !short.Contains(v) && rng.Intn(2) == 0 {
198+
gaps.Add(v)
199+
}
200+
if rng.Intn(4) == 0 {
201+
quarter.Add(v)
202+
}
203+
if rng.Intn(2) == 0 {
204+
half.Add(v)
205+
}
206+
if rng.Intn(10) != 0 {
207+
dense.Add(v)
208+
}
209+
}
210+
for _, tc := range []struct {
211+
name string
212+
x, y *Bitmap
213+
}{
214+
{"and/empty", short, gaps},
215+
{"and/array", short, half},
216+
{"and/bitmap", long, dense},
217+
{"and/many-intervals/array", many, quarter},
218+
{"and/many-intervals/bitmap", many, dense},
219+
} {
220+
b.Run(tc.name, func(b *testing.B) {
221+
b.ReportAllocs()
222+
for b.Loop() {
223+
_ = And(tc.x, tc.y)
224+
}
225+
})
226+
}
227+
for _, tc := range []struct {
228+
name string
229+
x *Bitmap
230+
}{
231+
{"iand/bitmap", long},
232+
{"iand/many-intervals", many},
233+
} {
234+
b.Run(tc.name, func(b *testing.B) {
235+
b.ReportAllocs()
236+
var xs []*Bitmap
237+
for b.Loop() {
238+
if len(xs) == 0 {
239+
// Fresh copies, made outside the timer: one And turns a copy into its result.
240+
b.StopTimer()
241+
for range 64 {
242+
xs = append(xs, dense.Clone())
243+
}
244+
b.StartTimer()
245+
}
246+
xs[len(xs)-1].And(tc.x)
247+
xs = xs[:len(xs)-1]
248+
}
249+
})
250+
b.Run("clone+"+tc.name, func(b *testing.B) {
251+
b.ReportAllocs()
252+
for b.Loop() {
253+
dense.Clone().And(tc.x)
254+
}
255+
})
256+
}
257+
few := New()
258+
few.AddMany([]uint32{100, 30000, 60000})
259+
for _, tc := range []struct {
260+
name string
261+
x, y *Bitmap
262+
}{
263+
{"intersects/disjoint", short, gaps},
264+
{"intersects/overlapping", short, half},
265+
{"intersects/few-values-many-intervals", many, few},
266+
} {
267+
b.Run(tc.name, func(b *testing.B) {
268+
b.ReportAllocs()
269+
for b.Loop() {
270+
_ = tc.x.Intersects(tc.y)
271+
}
272+
})
273+
}
274+
}

0 commit comments

Comments
 (0)