Improve roaring64 BSI value access and min/max

roaring64/bsi64.go

@@ -234,6 +234,49 @@ func (b *BSI) GetBigValues(columnIDs []uint64) []*big.Int {
 	return b.getBigValuesGeneric(request, values)
 }
 
+// GetValues gets int64 values for the column IDs. Returned values are aligned
+// with columnIDs, and a false exists entry means the corresponding column ID
+// has no value. This fast path is available only for BSIs whose values fit in
+// int64; it panics with the same representability semantics as GetValue when
+// a value cannot be represented as int64.
+func (b *BSI) GetValues(columnIDs []uint64) ([]int64, []bool) {
+	values := make([]int64, len(columnIDs))
+	exists := make([]bool, len(columnIDs))
+	if len(columnIDs) == 0 {
+		return values, exists
+	}
+	if len(columnIDs) == 1 {
+		if value, ok := b.GetValue(columnIDs[0]); ok {
+			values[0] = value
+			exists[0] = true
+		}
+		return values, exists
+	}
+	request := newBSIGetBigValuesRequest(columnIDs)
+	if b.isBig() {
+		bigValues := b.getBigValuesGeneric(request, make([]*big.Int, len(columnIDs)))
+		for i, value := range bigValues {
+			if value == nil {
+				continue
+			}
+			if !value.IsInt64() {
+				if value.Sign() == -1 {
+					msg := fmt.Errorf("can't represent a negative %d bit value as an int64", b.BitCount())
+					panic(msg)
+				}
+				if value.Sign() == 1 {
+					msg := fmt.Errorf("can't represent a positive %d bit value as an int64", b.BitCount())
+					panic(msg)
+				}
+			}
+			values[i] = value.Int64()
+			exists[i] = true
+		}
+		return values, exists
+	}
+	return b.getValuesInt64(request, values, exists)
+}
+
 type bsiGetBigValuesRequest struct {
 	foundSet           *Bitmap
 	positions          map[uint64]int
@@ -295,6 +338,40 @@ func (b *BSI) getBigValuesInt64(request bsiGetBigValuesRequest, values []*big.In
 	return values
 }
 
+func (b *BSI) getValuesInt64(request bsiGetBigValuesRequest, values []int64, exists []bool) ([]int64, []bool) {
+	existing := And(&b.eBM, request.foundSet)
+	if existing.IsEmpty() {
+		return values, exists
+	}
+
+	rawValues := make([]uint64, len(values))
+	signBit := b.BitCount()
+	for bit := 0; bit <= signBit; bit++ {
+		bitSet := And(&b.bA[bit], existing)
+		iter := bitSet.Iterator()
+		for iter.HasNext() {
+			columnID := iter.Next()
+			rawValues[request.positions[columnID]] |= uint64(1) << uint(bit)
+		}
+	}
+
+	width := uint(signBit + 1)
+	signMask := uint64(1) << uint(signBit)
+	iter := existing.Iterator()
+	for iter.HasNext() {
+		columnID := iter.Next()
+		position := request.positions[columnID]
+		rawValue := rawValues[position]
+		if rawValue&signMask != 0 && width < 64 {
+			rawValue |= ^uint64(0) << width
+		}
+		values[position] = int64(rawValue)
+		exists[position] = true
+	}
+	fillDuplicateValues(values, exists, request)
+	return values, exists
+}
+
 func (b *BSI) getBigValuesGeneric(request bsiGetBigValuesRequest, values []*big.Int) []*big.Int {
 	existing := And(&b.eBM, request.foundSet)
 	if existing.IsEmpty() {
@@ -339,6 +416,19 @@ func fillDuplicateBigValues(values []*big.Int, request bsiGetBigValuesRequest) {
 	}
 }
 
+func fillDuplicateValues(values []int64, exists []bool, request bsiGetBigValuesRequest) {
+	for columnID, extraPositions := range request.duplicatePositions {
+		position := request.positions[columnID]
+		if !exists[position] {
+			continue
+		}
+		for _, extraPosition := range extraPositions {
+			values[extraPosition] = values[position]
+			exists[extraPosition] = true
+		}
+	}
+}
+
 func negativeTwosComplementToInt(val *big.Int) *big.Int {
 	inverted := new(big.Int).Not(val)
 	mask := new(big.Int).Lsh(big.NewInt(1), uint(val.BitLen()))
@@ -875,53 +965,59 @@ func (b *BSI) MinMax(parallelism int, op Operation, foundSet *Bitmap) int64 {
 
 // MinMaxBig - Find minimum or maximum value.
 func (b *BSI) MinMaxBig(parallelism int, op Operation, foundSet *Bitmap) *big.Int {
-
-	var n int = parallelism
-	if n == 0 {
-		n = runtime.NumCPU()
-	}
-
-	resultsChan := make(chan *big.Int, n)
-
 	if foundSet == nil {
 		foundSet = &b.eBM
 	}
 
-	card := foundSet.GetCardinality()
-	x := card / uint64(n)
-
-	remainder := card - (x * uint64(n))
-	var batch []uint64
-	var wg sync.WaitGroup
-	iter := foundSet.ManyIterator()
-	for i := 0; i < n; i++ {
-		if i == n-1 {
-			batch = make([]uint64, x+remainder)
-		} else {
-			batch = make([]uint64, x)
-		}
-		iter.NextMany(batch)
-		wg.Add(1)
-		go b.minOrMax(op, batch, resultsChan, &wg)
-	}
-
-	wg.Wait()
-
-	close(resultsChan)
+	candidates := And(foundSet, &b.eBM)
 	var minMax *big.Int
 	minSigned, maxSigned := minMaxSignedInt(b.BitCount() + 1)
 	if op == MAX {
 		minMax = minSigned
 	} else {
 		minMax = maxSigned
 	}
+	if candidates.IsEmpty() {
+		return minMax
+	}
 
-	for val := range resultsChan {
-		if (op == MAX && val.Cmp(minMax) > 0) || (op == MIN && val.Cmp(minMax) <= 0) {
-			minMax = val
+	return b.minMaxBigByPlanes(op, candidates)
+}
+
+func (b *BSI) minMaxBigByPlanes(op Operation, candidates *Bitmap) *big.Int {
+	signPlane := &b.bA[b.BitCount()]
+	switch op {
+	case MIN:
+		negative := And(candidates, signPlane)
+		if !negative.IsEmpty() {
+			candidates = negative
+		}
+		for bit := b.BitCount() - 1; bit >= 0; bit-- {
+			unset := AndNot(candidates, &b.bA[bit])
+			if !unset.IsEmpty() {
+				candidates = unset
+				continue
+			}
+			candidates = And(candidates, &b.bA[bit])
 		}
+	case MAX:
+		nonNegative := AndNot(candidates, signPlane)
+		if !nonNegative.IsEmpty() {
+			candidates = nonNegative
+		}
+		for bit := b.BitCount() - 1; bit >= 0; bit-- {
+			set := And(candidates, &b.bA[bit])
+			if !set.IsEmpty() {
+				candidates = set
+				continue
+			}
+			candidates = AndNot(candidates, &b.bA[bit])
+		}
+	default:
+		panic(fmt.Sprintf("Operation [%v] not supported here", op))
 	}
-	return minMax
+	value, _ := b.GetBigValue(candidates.Minimum())
+	return value
 }
 
 func minMaxSignedInt(bits int) (*big.Int, *big.Int) {

roaring64/bsi64_get_big_values_test.go

@@ -70,6 +70,54 @@ func TestBSI64GetBigValuesHandlesBigWidthAndDuplicates(t *testing.T) {
 	assert.Equal(t, 0, stored.Cmp(negativeHuge), "mutating returned values must not alter the BSI")
 }
 
+func TestBSI64GetValuesConsistentWithGetValue(t *testing.T) {
+	rg := rand.New(rand.NewSource(1864))
+	for run := 0; run < 25; run++ {
+		bsi := NewDefaultBSI()
+		numCols := rg.Intn(1000) + 50
+		for col := 0; col < numCols; col++ {
+			if rg.Float64() < 0.85 {
+				bsi.SetValue(uint64(col), rg.Int63n(4000)-2000)
+			}
+		}
+
+		columnIDs := make([]uint64, 0, numCols+8)
+		for col := numCols - 1; col >= 0; col-- {
+			if col%3 != 0 {
+				columnIDs = append(columnIDs, uint64(col))
+			}
+		}
+		columnIDs = append(columnIDs, uint64(numCols+10), 7, 7, 11)
+
+		values, exists := bsi.GetValues(columnIDs)
+		if len(values) != len(columnIDs) {
+			t.Fatalf("run=%d values length = %d, want %d", run, len(values), len(columnIDs))
+		}
+		if len(exists) != len(columnIDs) {
+			t.Fatalf("run=%d exists length = %d, want %d", run, len(exists), len(columnIDs))
+		}
+		for i, columnID := range columnIDs {
+			expectedValue, expectedOK := bsi.GetValue(columnID)
+			assert.Equal(t, expectedOK, exists[i], "run=%d column=%d", run, columnID)
+			if !expectedOK {
+				continue
+			}
+			assert.Equal(t, expectedValue, values[i], "run=%d column=%d", run, columnID)
+		}
+	}
+}
+
+func TestBSI64GetValuesHandlesDuplicatesAndMissingValues(t *testing.T) {
+	bsi := NewDefaultBSI()
+	bsi.SetValue(1, -17)
+	bsi.SetValue(2, 44)
+	bsi.SetValue(4, 0)
+
+	values, exists := bsi.GetValues([]uint64{2, 3, 1, 2, 4})
+	assert.Equal(t, []int64{44, 0, -17, 44, 0}, values)
+	assert.Equal(t, []bool{true, false, true, true, true}, exists)
+}
+
 func BenchmarkBSI64GetBigValuesLargeFixture(b *testing.B) {
 	bsi, _ := setupBSI64CompareBSIFixture(b, 100000)
 	columnIDs := bsi64SequentialColumns(100000)
@@ -80,6 +128,16 @@ func BenchmarkBSI64GetBigValuesLargeFixture(b *testing.B) {
 	}
 }
 
+func BenchmarkBSI64GetValuesLargeFixture(b *testing.B) {
+	bsi, _ := setupBSI64CompareBSIFixture(b, 100000)
+	columnIDs := bsi64SequentialColumns(100000)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		values, exists := bsi.GetValues(columnIDs)
+		_, _ = values, exists
+	}
+}
+
 func BenchmarkBSI64GetBigValueLoopLargeFixture(b *testing.B) {
 	bsi, _ := setupBSI64CompareBSIFixture(b, 100000)
 	columnIDs := bsi64SequentialColumns(100000)

roaring64/bsi64_minmax_benchmark_test.go

@@ -0,0 +1,24 @@
+package roaring64
+
+import (
+	"math/big"
+	"testing"
+)
+
+var benchmarkMinMaxBigResult *big.Int
+
+func BenchmarkBSI64MinMaxBig(b *testing.B) {
+	const rows = 1_000_000
+	bsi := NewDefaultBSI()
+	foundSet := NewBitmap()
+	for row := uint64(0); row < rows; row++ {
+		bsi.SetValue(row, int64(row%50))
+		foundSet.Add(row)
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		benchmarkMinMaxBigResult = bsi.MinMaxBig(0, MIN, foundSet)
+		benchmarkMinMaxBigResult = bsi.MinMaxBig(0, MAX, foundSet)
+	}
+}

roaring64/bsi64_test.go

@@ -674,6 +674,33 @@ func TestMinMaxWithNilFoundSet(t *testing.T) {
 	assert.Equal(t, max, bsi.MinMax(0, MAX, nil))
 }
 
+func TestMinMaxWithSubset(t *testing.T) {
+	bsi := NewDefaultBSI()
+	bsi.SetValue(1, -40)
+	bsi.SetValue(2, -10)
+	bsi.SetValue(3, 5)
+	bsi.SetValue(4, 20)
+	bsi.SetValue(5, 100)
+
+	foundSet := BitmapOf(2, 3, 4)
+	assert.Equal(t, int64(-10), bsi.MinMax(0, MIN, foundSet))
+	assert.Equal(t, int64(20), bsi.MinMax(0, MAX, foundSet))
+}
+
+func TestMinMaxBigWithLargeValues(t *testing.T) {
+	bsi := NewDefaultBSI()
+	minValue := new(big.Int).Lsh(big.NewInt(1), 90)
+	midValue := new(big.Int).Lsh(big.NewInt(1), 95)
+	maxValue := new(big.Int).Lsh(big.NewInt(1), 100)
+	bsi.SetBigValue(10, minValue)
+	bsi.SetBigValue(11, midValue)
+	bsi.SetBigValue(12, maxValue)
+
+	foundSet := BitmapOf(10, 12)
+	assert.Equal(t, 0, bsi.MinMaxBig(0, MIN, foundSet).Cmp(minValue))
+	assert.Equal(t, 0, bsi.MinMaxBig(0, MAX, foundSet).Cmp(maxValue))
+}
+
 func TestBSIWriteToReadFrom(t *testing.T) {
 	file, err := os.CreateTemp(t.TempDir(), "bsi-test")
 	if err != nil {