Document that RunOptimize recovers space after modifying run containers

Author: lemire

BitSliceIndexing/bsi.go

@@ -54,7 +54,9 @@ func NewDefaultBSI() *BSI {
 	return NewBSI(int64(0), int64(0))
 }
 
-// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
+// RunOptimize converts the containers of the underlying bitmaps to their most compact
+// encoding. Calling it again after modifying the BSI can recover space: see
+// Bitmap.RunOptimize.
 func (b *BSI) RunOptimize() {
 	b.eBM.RunOptimize()
 	for i := 0; i < len(b.bA); i++ {

README.md

@@ -280,6 +280,15 @@ That is, given a fixed overhead for the universe size (x), Roaring
 bitmaps never use more than 2 bytes per integer. You can call
 ``BoundSerializedSizeInBytes`` for a more precise estimate.
 
+Call ``RunOptimize`` to convert containers to their most compact encoding,
+including run-length encoding where it helps. Some operations that modify a
+bitmap, such as ``AddRange`` and ``RemoveRange``, do not always convert a run
+container back to an array or bitmap container when the runs stop being the
+most compact encoding. In the worst case, the runs of one 65536-value chunk can
+take about 128 KiB where a bitmap container takes 8 KiB. Calling ``RunOptimize``
+again after such modifications recovers that space, in memory and once
+serialized. This applies to 64-bit bitmaps (``roaring64``) as well.
+
 ### 64-bit Roaring
 
 By default, roaring is used to stored unsigned 32-bit integers. However, we also offer

roaring.go

@@ -394,7 +394,16 @@ func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) {
 	return
 }
 
-// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
+// RunOptimize converts each container to its most compact encoding: run-length encoding
+// where it is smaller, and an array or bitmap container where it is not.
+//
+// Some operations that modify a bitmap, such as AddRange and RemoveRange, do not always
+// convert a run container back to an array or bitmap container when the runs stop being
+// the most compact encoding. Such a container can then use more memory and serialized
+// space than needed: in the worst case, the runs of one 65536-value chunk can take about
+// 128 KiB where a bitmap container takes 8 KiB. Calling RunOptimize again after such
+// modifications recovers that space. Consider calling it before serializing a bitmap that
+// was modified since its last call.
 func (rb *Bitmap) RunOptimize() {
 	rb.highlowcontainer.runOptimize()
 }

roaring64/bsi64.go

@@ -51,7 +51,9 @@ func NewDefaultBSI() *BSI {
 	return NewBSI(int64(0), int64(0))
 }
 
-// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
+// RunOptimize converts the containers of the underlying bitmaps to their most compact
+// encoding. Calling it again after modifying the BSI can recover space: see
+// Bitmap.RunOptimize.
 func (b *BSI) RunOptimize() {
 	b.eBM.RunOptimize()
 	for i := 0; i < len(b.bA); i++ {

roaring64/roaring64.go

@@ -187,7 +187,16 @@ func (rb *Bitmap) UnmarshalBinary(data []byte) error {
 	return err
 }
 
-// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
+// RunOptimize converts each container to its most compact encoding: run-length encoding
+// where it is smaller, and an array or bitmap container where it is not.
+//
+// Some operations that modify a bitmap, such as AddRange and RemoveRange, do not always
+// convert a run container back to an array or bitmap container when the runs stop being
+// the most compact encoding. Such a container can then use more memory and serialized
+// space than needed: in the worst case, the runs of one 65536-value chunk can take about
+// 128 KiB where a bitmap container takes 8 KiB. Calling RunOptimize again after such
+// modifications recovers that space. Consider calling it before serializing a bitmap that
+// was modified since its last call.
 func (rb *Bitmap) RunOptimize() {
 	rb.highlowcontainer.runOptimize()
 }

roaring64/roaring64_test.go

@@ -2141,3 +2141,17 @@ func TestRoaringArray64SortOrder(t *testing.T) {
 		assert.True(t, bm.highlowcontainer.checkKeysSorted())
 	})
 }
+
+func TestRunOptimizeRecoversSpaceAfterRemoveRange(t *testing.T) {
+	bm := New()
+	bm.AddRange(0, 65536)
+	for i := uint64(0); i < 65536; i += 2 {
+		bm.RemoveRange(i, i+1) // each remaining value becomes a run of length one
+	}
+	expected := bm.Clone()
+	before := bm.GetSerializedSizeInBytes()
+	bm.RunOptimize()
+	assert.Less(t, bm.GetSerializedSizeInBytes(), before/10)
+	assert.False(t, bm.HasRunCompression())
+	assert.True(t, expected.Equals(bm))
+}

roaring_test.go

@@ -4211,3 +4211,18 @@ func TestValidateAcceptsRunContainersThatAreNotTheSmallestEncoding(t *testing.T)
 	assert.NoError(t, deserialized.Validate())
 	assert.True(t, expected.Equals(deserialized))
 }
+
+func TestRunOptimizeRecoversSpaceAfterRemoveRange(t *testing.T) {
+	bm := New()
+	bm.AddRange(0, 65536)
+	for i := uint64(0); i < 65536; i += 2 {
+		bm.RemoveRange(i, i+1) // each remaining value becomes a run of length one
+	}
+	expected := bm.Clone()
+	before := bm.GetSerializedSizeInBytes()
+	bm.RunOptimize()
+	assert.Less(t, bm.GetSerializedSizeInBytes(), before/10)
+	assert.False(t, bm.HasRunCompression())
+	assert.True(t, expected.Equals(bm))
+	require.NoError(t, bm.Validate())
+}