diff --git a/roaring64/bsi64.go b/roaring64/bsi64.go index 4e918ddb..134eb58c 100644 --- a/roaring64/bsi64.go +++ b/roaring64/bsi64.go @@ -234,6 +234,49 @@ func (b *BSI) GetBigValues(columnIDs []uint64) []*big.Int { return b.getBigValuesGeneric(request, values) } +// GetValues gets int64 values for the column IDs. Returned values are aligned +// with columnIDs, and a false exists entry means the corresponding column ID +// has no value. This fast path is available only for BSIs whose values fit in +// int64; it panics with the same representability semantics as GetValue when +// a value cannot be represented as int64. +func (b *BSI) GetValues(columnIDs []uint64) ([]int64, []bool) { + values := make([]int64, len(columnIDs)) + exists := make([]bool, len(columnIDs)) + if len(columnIDs) == 0 { + return values, exists + } + if len(columnIDs) == 1 { + if value, ok := b.GetValue(columnIDs[0]); ok { + values[0] = value + exists[0] = true + } + return values, exists + } + request := newBSIGetBigValuesRequest(columnIDs) + if b.isBig() { + bigValues := b.getBigValuesGeneric(request, make([]*big.Int, len(columnIDs))) + for i, value := range bigValues { + if value == nil { + continue + } + if !value.IsInt64() { + if value.Sign() == -1 { + msg := fmt.Errorf("can't represent a negative %d bit value as an int64", b.BitCount()) + panic(msg) + } + if value.Sign() == 1 { + msg := fmt.Errorf("can't represent a positive %d bit value as an int64", b.BitCount()) + panic(msg) + } + } + values[i] = value.Int64() + exists[i] = true + } + return values, exists + } + return b.getValuesInt64(request, values, exists) +} + type bsiGetBigValuesRequest struct { foundSet *Bitmap positions map[uint64]int @@ -295,6 +338,40 @@ func (b *BSI) getBigValuesInt64(request bsiGetBigValuesRequest, values []*big.In return values } +func (b *BSI) getValuesInt64(request bsiGetBigValuesRequest, values []int64, exists []bool) ([]int64, []bool) { + existing := And(&b.eBM, request.foundSet) + if existing.IsEmpty() { + return values, exists + } + + rawValues := make([]uint64, len(values)) + signBit := b.BitCount() + for bit := 0; bit <= signBit; bit++ { + bitSet := And(&b.bA[bit], existing) + iter := bitSet.Iterator() + for iter.HasNext() { + columnID := iter.Next() + rawValues[request.positions[columnID]] |= uint64(1) << uint(bit) + } + } + + width := uint(signBit + 1) + signMask := uint64(1) << uint(signBit) + iter := existing.Iterator() + for iter.HasNext() { + columnID := iter.Next() + position := request.positions[columnID] + rawValue := rawValues[position] + if rawValue&signMask != 0 && width < 64 { + rawValue |= ^uint64(0) << width + } + values[position] = int64(rawValue) + exists[position] = true + } + fillDuplicateValues(values, exists, request) + return values, exists +} + func (b *BSI) getBigValuesGeneric(request bsiGetBigValuesRequest, values []*big.Int) []*big.Int { existing := And(&b.eBM, request.foundSet) if existing.IsEmpty() { @@ -339,6 +416,19 @@ func fillDuplicateBigValues(values []*big.Int, request bsiGetBigValuesRequest) { } } +func fillDuplicateValues(values []int64, exists []bool, request bsiGetBigValuesRequest) { + for columnID, extraPositions := range request.duplicatePositions { + position := request.positions[columnID] + if !exists[position] { + continue + } + for _, extraPosition := range extraPositions { + values[extraPosition] = values[position] + exists[extraPosition] = true + } + } +} + func negativeTwosComplementToInt(val *big.Int) *big.Int { inverted := new(big.Int).Not(val) mask := new(big.Int).Lsh(big.NewInt(1), uint(val.BitLen())) @@ -875,39 +965,11 @@ func (b *BSI) MinMax(parallelism int, op Operation, foundSet *Bitmap) int64 { // MinMaxBig - Find minimum or maximum value. func (b *BSI) MinMaxBig(parallelism int, op Operation, foundSet *Bitmap) *big.Int { - - var n int = parallelism - if n == 0 { - n = runtime.NumCPU() - } - - resultsChan := make(chan *big.Int, n) - if foundSet == nil { foundSet = &b.eBM } - card := foundSet.GetCardinality() - x := card / uint64(n) - - remainder := card - (x * uint64(n)) - var batch []uint64 - var wg sync.WaitGroup - iter := foundSet.ManyIterator() - for i := 0; i < n; i++ { - if i == n-1 { - batch = make([]uint64, x+remainder) - } else { - batch = make([]uint64, x) - } - iter.NextMany(batch) - wg.Add(1) - go b.minOrMax(op, batch, resultsChan, &wg) - } - - wg.Wait() - - close(resultsChan) + candidates := And(foundSet, &b.eBM) var minMax *big.Int minSigned, maxSigned := minMaxSignedInt(b.BitCount() + 1) if op == MAX { @@ -915,13 +977,47 @@ func (b *BSI) MinMaxBig(parallelism int, op Operation, foundSet *Bitmap) *big.In } else { minMax = maxSigned } + if candidates.IsEmpty() { + return minMax + } - for val := range resultsChan { - if (op == MAX && val.Cmp(minMax) > 0) || (op == MIN && val.Cmp(minMax) <= 0) { - minMax = val + return b.minMaxBigByPlanes(op, candidates) +} + +func (b *BSI) minMaxBigByPlanes(op Operation, candidates *Bitmap) *big.Int { + signPlane := &b.bA[b.BitCount()] + switch op { + case MIN: + negative := And(candidates, signPlane) + if !negative.IsEmpty() { + candidates = negative + } + for bit := b.BitCount() - 1; bit >= 0; bit-- { + unset := AndNot(candidates, &b.bA[bit]) + if !unset.IsEmpty() { + candidates = unset + continue + } + candidates = And(candidates, &b.bA[bit]) } + case MAX: + nonNegative := AndNot(candidates, signPlane) + if !nonNegative.IsEmpty() { + candidates = nonNegative + } + for bit := b.BitCount() - 1; bit >= 0; bit-- { + set := And(candidates, &b.bA[bit]) + if !set.IsEmpty() { + candidates = set + continue + } + candidates = AndNot(candidates, &b.bA[bit]) + } + default: + panic(fmt.Sprintf("Operation [%v] not supported here", op)) } - return minMax + value, _ := b.GetBigValue(candidates.Minimum()) + return value } func minMaxSignedInt(bits int) (*big.Int, *big.Int) { diff --git a/roaring64/bsi64_get_big_values_test.go b/roaring64/bsi64_get_big_values_test.go index 38049b2d..4f497874 100644 --- a/roaring64/bsi64_get_big_values_test.go +++ b/roaring64/bsi64_get_big_values_test.go @@ -70,6 +70,54 @@ func TestBSI64GetBigValuesHandlesBigWidthAndDuplicates(t *testing.T) { assert.Equal(t, 0, stored.Cmp(negativeHuge), "mutating returned values must not alter the BSI") } +func TestBSI64GetValuesConsistentWithGetValue(t *testing.T) { + rg := rand.New(rand.NewSource(1864)) + for run := 0; run < 25; run++ { + bsi := NewDefaultBSI() + numCols := rg.Intn(1000) + 50 + for col := 0; col < numCols; col++ { + if rg.Float64() < 0.85 { + bsi.SetValue(uint64(col), rg.Int63n(4000)-2000) + } + } + + columnIDs := make([]uint64, 0, numCols+8) + for col := numCols - 1; col >= 0; col-- { + if col%3 != 0 { + columnIDs = append(columnIDs, uint64(col)) + } + } + columnIDs = append(columnIDs, uint64(numCols+10), 7, 7, 11) + + values, exists := bsi.GetValues(columnIDs) + if len(values) != len(columnIDs) { + t.Fatalf("run=%d values length = %d, want %d", run, len(values), len(columnIDs)) + } + if len(exists) != len(columnIDs) { + t.Fatalf("run=%d exists length = %d, want %d", run, len(exists), len(columnIDs)) + } + for i, columnID := range columnIDs { + expectedValue, expectedOK := bsi.GetValue(columnID) + assert.Equal(t, expectedOK, exists[i], "run=%d column=%d", run, columnID) + if !expectedOK { + continue + } + assert.Equal(t, expectedValue, values[i], "run=%d column=%d", run, columnID) + } + } +} + +func TestBSI64GetValuesHandlesDuplicatesAndMissingValues(t *testing.T) { + bsi := NewDefaultBSI() + bsi.SetValue(1, -17) + bsi.SetValue(2, 44) + bsi.SetValue(4, 0) + + values, exists := bsi.GetValues([]uint64{2, 3, 1, 2, 4}) + assert.Equal(t, []int64{44, 0, -17, 44, 0}, values) + assert.Equal(t, []bool{true, false, true, true, true}, exists) +} + func BenchmarkBSI64GetBigValuesLargeFixture(b *testing.B) { bsi, _ := setupBSI64CompareBSIFixture(b, 100000) columnIDs := bsi64SequentialColumns(100000) @@ -80,6 +128,16 @@ func BenchmarkBSI64GetBigValuesLargeFixture(b *testing.B) { } } +func BenchmarkBSI64GetValuesLargeFixture(b *testing.B) { + bsi, _ := setupBSI64CompareBSIFixture(b, 100000) + columnIDs := bsi64SequentialColumns(100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + values, exists := bsi.GetValues(columnIDs) + _, _ = values, exists + } +} + func BenchmarkBSI64GetBigValueLoopLargeFixture(b *testing.B) { bsi, _ := setupBSI64CompareBSIFixture(b, 100000) columnIDs := bsi64SequentialColumns(100000) diff --git a/roaring64/bsi64_minmax_benchmark_test.go b/roaring64/bsi64_minmax_benchmark_test.go new file mode 100644 index 00000000..074069b9 --- /dev/null +++ b/roaring64/bsi64_minmax_benchmark_test.go @@ -0,0 +1,24 @@ +package roaring64 + +import ( + "math/big" + "testing" +) + +var benchmarkMinMaxBigResult *big.Int + +func BenchmarkBSI64MinMaxBig(b *testing.B) { + const rows = 1_000_000 + bsi := NewDefaultBSI() + foundSet := NewBitmap() + for row := uint64(0); row < rows; row++ { + bsi.SetValue(row, int64(row%50)) + foundSet.Add(row) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkMinMaxBigResult = bsi.MinMaxBig(0, MIN, foundSet) + benchmarkMinMaxBigResult = bsi.MinMaxBig(0, MAX, foundSet) + } +} diff --git a/roaring64/bsi64_test.go b/roaring64/bsi64_test.go index 9deb5dee..39dbdcde 100644 --- a/roaring64/bsi64_test.go +++ b/roaring64/bsi64_test.go @@ -674,6 +674,33 @@ func TestMinMaxWithNilFoundSet(t *testing.T) { assert.Equal(t, max, bsi.MinMax(0, MAX, nil)) } +func TestMinMaxWithSubset(t *testing.T) { + bsi := NewDefaultBSI() + bsi.SetValue(1, -40) + bsi.SetValue(2, -10) + bsi.SetValue(3, 5) + bsi.SetValue(4, 20) + bsi.SetValue(5, 100) + + foundSet := BitmapOf(2, 3, 4) + assert.Equal(t, int64(-10), bsi.MinMax(0, MIN, foundSet)) + assert.Equal(t, int64(20), bsi.MinMax(0, MAX, foundSet)) +} + +func TestMinMaxBigWithLargeValues(t *testing.T) { + bsi := NewDefaultBSI() + minValue := new(big.Int).Lsh(big.NewInt(1), 90) + midValue := new(big.Int).Lsh(big.NewInt(1), 95) + maxValue := new(big.Int).Lsh(big.NewInt(1), 100) + bsi.SetBigValue(10, minValue) + bsi.SetBigValue(11, midValue) + bsi.SetBigValue(12, maxValue) + + foundSet := BitmapOf(10, 12) + assert.Equal(t, 0, bsi.MinMaxBig(0, MIN, foundSet).Cmp(minValue)) + assert.Equal(t, 0, bsi.MinMaxBig(0, MAX, foundSet).Cmp(maxValue)) +} + func TestBSIWriteToReadFrom(t *testing.T) { file, err := os.CreateTemp(t.TempDir(), "bsi-test") if err != nil {