Fix NextAbsentValue and PreviousAbsentValue

arraycontainer.go

@@ -1193,7 +1193,7 @@ func (ac *arrayContainer) nextAbsentValue(target uint16) int {
 	}
 
 	if low == cardinality-1 {
-		return int(ac.content[cardinality-1] + 1)
+		return int(ac.content[cardinality-1]) + 1
 	}
 
 	return int(ac.content[low] + 1)

arraycontainer_test.go

@@ -642,6 +642,28 @@ func TestPreviousValueArray(t *testing.T) {
 	})
 }
 
+// uint16(65535)+1 wraps to 0.
+func TestArrayContainerNextAbsentValueAtEnd(t *testing.T) {
+	t.Run("One value at the end", func(t *testing.T) {
+		ac := newArrayContainer()
+		ac.iadd(65535)
+		assert.Equal(t, 1<<16, ac.nextAbsentValue(65535))
+	})
+
+	t.Run("Two values at the end", func(t *testing.T) {
+		ac := newArrayContainer()
+		ac.iadd(65534)
+		ac.iadd(65535)
+		assert.Equal(t, 1<<16, ac.nextAbsentValue(65534))
+		assert.Equal(t, 1<<16, ac.nextAbsentValue(65535))
+	})
+
+	t.Run("In a bitmap", func(t *testing.T) {
+		bmp := BitmapOf(1, 65535)
+		assert.Equal(t, int64(65536), bmp.NextAbsentValue(65535))
+	})
+}
+
 func TestPreviousAbsentValueArray(t *testing.T) {
 	t.Run("Java Port 1", func(t *testing.T) {
 		// [Example 1] https://github.com/RoaringBitmap/RoaringBitmap/blob/5235aa62c32fa3bf7fae40a562e3edc75f61be4e/RoaringBitmap/src/test/java/org/roaringbitmap/TestArrayContainer.java#L793

bitmapcontainer.go

@@ -1390,53 +1390,28 @@ func (bc *bitmapContainer) nextValue(target uint16) int {
 	return bc.NextSetBit(uint(target))
 }
 
-// nextAbsentValue returns the next absent value.
-// if the target is out of bounds a -1 is returned
+// nextAbsentValue returns the smallest absent value >= target.
+// The result is in 0..65535, or 1<<16 when every such value is present.
 func (bc *bitmapContainer) nextAbsentValue(target uint16) int {
-	if bc.cardinality == 0 {
-		return -1
+	x := uint(target >> 6)
+	length := uint(len(bc.bitmap))
+	if x >= length {
+		return 1 << 16
 	}
 
-	var (
-		x      = target >> 6
-		length = uint(len(bc.bitmap))
-	)
-	if uint(x) >= length {
-		return -1
-	}
-	w := bc.bitmap[x]
-	w = w >> uint(target%64)
-	if w == 0 {
-		return int(target)
+	w := (^bc.bitmap[x]) >> uint(target%64)
+	if w != 0 {
+		return int(target) + bits.TrailingZeros64(w)
 	}
 
-	// Check if all 1's
-	// if statement - we skip the if we have all ones [1,1,1,1...1]
-	if ^w != 0 {
-
-		if bits.TrailingZeros64(w) > 0 {
-			// we have something like [X,Y,Z, 0,0,0]. This means the target bit is zero
-			return int(target)
-		}
-
-		// other wise something like [X,Y,0,1,1,1..1], where x and y can be either 1 or 0.
-
-		trailing := bits.TrailingZeros64(^w)
-		return int(target) + trailing
-
-	}
 	x++
-	for ; uint(x) < length; x++ {
-		if bc.bitmap[x] == 0 {
-			return int(x * 64)
-		}
-		if ^bc.bitmap[x] != 0 {
-			trailing := bits.TrailingZeros64(^bc.bitmap[x])
-			return int(x*64) + trailing
+	for ; x < length; x++ {
+		if inv := ^bc.bitmap[x]; inv != 0 {
+			return int(x*64) + bits.TrailingZeros64(inv)
 		}
-
 	}
-	return -1
+
+	return 1 << 16
 }
 
 // previousValue returns either the `target` if found or the previous largest value.
@@ -1458,64 +1433,26 @@ func (bc *bitmapContainer) previousValue(target uint16) int {
 	return bc.uPrevSetBit(uint(target))
 }
 
-// previousAbsentValue returns the next absent value.
+// previousAbsentValue returns the largest absent value <= target,
+// or -1 when every such value is present.
 func (bc *bitmapContainer) previousAbsentValue(target uint16) int {
-	if bc.cardinality == 0 {
+	x := int(target >> 6)
+	if x >= len(bc.bitmap) {
 		return -1
 	}
 
-	var (
-		x      = target >> 6
-		length = uint(len(bc.bitmap))
-	)
-	if uint(x) >= length {
-		return -1
-	}
-	w := bc.bitmap[x]
-	shifted := w >> uint(target%64)
-	if shifted == 0 {
-		return int(target)
+	// Shift left by 63-(target%64) so LeadingZeros ignores bits shifted in.
+	w := (^bc.bitmap[x]) << (63 - uint(target%64))
+	if w != 0 {
+		return int(target) - bits.LeadingZeros64(w)
 	}
 
-	// Check if all 1's
-	// if statement - we skip if we have all ones [1,1,1,1...1] as no value is absent
-	if ^shifted != 0 {
-
-		if bits.TrailingZeros64(shifted) > 0 {
-			// we have something like shifted=[X,Y,Z,..., 0,0,0]. This means the target bit is zero
-			return int(target)
-		}
-
-		// The rotate will rotate the target bit into the leading position.
-		// We know the target bit is not zero because of the TrailingZeros64 check above
-		// We then shift the target bit out of the way.
-		// Assume a structure like an original structure like [X,Y,Z,..., Target, A, B,C...]
-		// shifted will be [X,Y,Z...Target]
-		// shiftedRotated will be [A,B,C....]
-		// If LeadingZeros64 > 0 then A is zero, if not at least A is 1 return
-		// Else count the number of ones's until a 0
-		shiftedRotated := bits.RotateLeft64(w, int(64-uint(target%64))-1) << 1
-		leadingZeros := bits.LeadingZeros64(shiftedRotated)
-		if leadingZeros > 0 {
-			return int(target) - 1
+	for x--; x >= 0; x-- {
+		if inv := ^bc.bitmap[x]; inv != 0 {
+			return x*64 + 63 - bits.LeadingZeros64(inv)
 		}
-		leadingOnes := bits.LeadingZeros64(^shiftedRotated)
-		if leadingOnes > 0 {
-			return int(target) - leadingOnes - 1
-		}
-
 	}
-	x++
-	for ; uint(x) < length; x++ {
-		if bc.bitmap[x] == 0 {
-			return int(x * 64)
-		}
-		if ^bc.bitmap[x] != 0 {
-			trailing := bits.TrailingZeros64(^bc.bitmap[x])
-			return int(x*64) + trailing
-		}
 
-	}
 	return -1
 }
 

bitmapcontainer_test.go

@@ -412,6 +412,35 @@ func TestNextAbsent(t *testing.T) {
 	assert.Equal(t, v, 65536)
 }
 
+func TestBitmapContainerAbsentValueAcrossWords(t *testing.T) {
+	t.Run("Next absent value after a run of ones that reaches the end of a word", func(t *testing.T) {
+		bc := newBitmapContainer()
+		for i := 0; i < 5000; i++ {
+			bc.iadd(uint16(i))
+		}
+		assert.Equal(t, 5000, bc.nextAbsentValue(10))
+		assert.Equal(t, 5000, bc.nextAbsentValue(70))
+	})
+
+	t.Run("Next absent value of a full container", func(t *testing.T) {
+		bc := newBitmapContainer()
+		for i := 0; i < 1<<16; i++ {
+			bc.iadd(uint16(i))
+		}
+		assert.Equal(t, 1<<16, bc.nextAbsentValue(0))
+		assert.Equal(t, 1<<16, bc.nextAbsentValue(64))
+	})
+
+	t.Run("Previous absent value goes down through the earlier words", func(t *testing.T) {
+		bc := newBitmapContainer()
+		for i := 1; i <= 5000; i++ {
+			bc.iadd(uint16(i))
+		}
+		assert.Equal(t, 0, bc.previousAbsentValue(5000))
+		assert.Equal(t, 0, bc.previousAbsentValue(70))
+	})
+}
+
 func TestBitMapContainerValidate(t *testing.T) {
 	bc := newBitmapContainer()
 

roaring.go

@@ -2320,28 +2320,30 @@ func (rb *Bitmap) NextAbsentValue(target uint32) int64 {
 		return int64(target)
 	}
 
+	// keyspace is already key<<16. 1<<16 means the container is exhausted.
 	container := rb.highlowcontainer.getContainer(containerKey)
 	nextValue = int64(container.nextAbsentValue(query))
 	for {
 		if nextValue != (1 << 16) {
-			return int64(combineLoHi32(uint32(nextValue), keyspace))
+			return int64(uint32(nextValue) | keyspace)
 		}
 
 		if containerIndex == rb.highlowcontainer.size()-1 {
-			val, err := container.safeMaximum()
-			if err == nil {
+			if containerKey == 0xFFFF {
 				return -1
 			}
-			return int64(val) + 1
+			return int64(containerKey+1) << 16
 		}
 		containerIndex++
 		nextContainerKey := rb.highlowcontainer.getKeyAtIndex(containerIndex)
-		if containerKey < nextContainerKey {
+		if containerKey+1 < nextContainerKey {
 			// There is a gap between keys
 			// Just increment the current key and shift to get HoB
 			return int64(containerKey+1) << 16
 		}
+
 		containerKey = nextContainerKey
+		keyspace = uint32(containerKey) << 16
 		container = rb.highlowcontainer.getContainer(containerKey)
 		nextValue = int64(container.nextAbsentValue(0))
 	}
@@ -2377,20 +2379,20 @@ func (rb *Bitmap) PreviousAbsentValue(target uint32) int64 {
 		return int64(target)
 	}
 
+	// keyspace is already key<<16. -1 means the container is exhausted.
 	container := rb.highlowcontainer.getContainer(containerKey)
 	prevValue = int64(container.previousAbsentValue(query))
 	for {
 		if prevValue != -1 {
-			return int64(combineLoHi32(uint32(prevValue), keyspace))
+			return int64(uint32(prevValue) | keyspace)
 		}
 
 		if containerIndex == 0 {
-			val, err := container.safeMinimum()
-			if err == nil {
+			if containerKey == 0 {
 				// OR panic, Java panics
 				return -1
 			}
-			return int64(val) - 1
+			return (int64(containerKey) << 16) - 1
 		}
 		containerIndex--
 		nextContainerKey := rb.highlowcontainer.getKeyAtIndex(containerIndex)
@@ -2399,7 +2401,9 @@ func (rb *Bitmap) PreviousAbsentValue(target uint32) int64 {
 			// Just decrement the current key and shift to get HoB of the missing container
 			return (int64(containerKey) << 16) - 1
 		}
+
 		containerKey = nextContainerKey
+		keyspace = uint32(containerKey) << 16
 		container = rb.highlowcontainer.getContainer(containerKey)
 		highestPossible16 := (1 << 16) - 1
 		prevValue = int64(container.previousAbsentValue(uint16(highestPossible16)))

roaring_test.go

@@ -3708,6 +3708,46 @@ func TestNextAndPreviousValue(t *testing.T) {
 	})
 }
 
+func TestNextAndPreviousAbsentValueContainerKeys(t *testing.T) {
+	t.Run("High bits of the result for a key other than 0", func(t *testing.T) {
+		bmp := BitmapOf(65537)
+		assert.Equal(t, int64(65536), bmp.NextAbsentValue(65536))
+		assert.Equal(t, int64(65538), bmp.NextAbsentValue(65537))
+		assert.Equal(t, int64(65536), bmp.PreviousAbsentValue(65537))
+		assert.Equal(t, int64(65538), bmp.PreviousAbsentValue(65538))
+	})
+
+	t.Run("Key follows the search into the previous container", func(t *testing.T) {
+		// Keys 1 and 2: key 0 would hide a missing high-half update.
+		bmp := New()
+		bmp.Add(65536)
+		bmp.Add(131071)
+		bmp.AddRange(131072, 196608)
+		assert.Equal(t, int64(131070), bmp.PreviousAbsentValue(196607))
+	})
+
+	t.Run("Full last container", func(t *testing.T) {
+		bmp := New()
+		bmp.AddRange(0, 65536)
+		assert.Equal(t, int64(65536), bmp.NextAbsentValue(0))
+		assert.Equal(t, int64(65536), bmp.NextAbsentValue(65535))
+	})
+
+	t.Run("Full first container", func(t *testing.T) {
+		bmp := New()
+		bmp.AddRange(65536, 131072)
+		assert.Equal(t, int64(65535), bmp.PreviousAbsentValue(131071))
+		assert.Equal(t, int64(65535), bmp.PreviousAbsentValue(65536))
+	})
+
+	t.Run("Full containers next to each other", func(t *testing.T) {
+		bmp := New()
+		bmp.AddRange(0, 131072)
+		assert.Equal(t, int64(131072), bmp.NextAbsentValue(0))
+		assert.Equal(t, int64(131072), bmp.NextAbsentValue(65536))
+	})
+}
+
 func BenchmarkFromDense(b *testing.B) {
 	testDense(func(name string, rb *Bitmap) {
 		dense := make([]uint64, rb.DenseSize())