scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1106 from niaow/topk-time

Browse source 
TopK on time
This commit is contained in:
Nia 2020-11-20 13:24:05 -05:00 • committed by GitHub
commit 106952dc5e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 2040 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

240
bsi.go
View file

@ -14,29 +14,19 @@
package pilosa
import "math/bits"
import (
"math/bits"
"github.com/pilosa/pilosa/v2/roaring"
)
// bsiData contains BSI-structured data.
type bsiData []*Row
// insert a value for a column in the BSI data.
func (bsi *bsiData) insert(column uint64, value uint64) {
data := *bsi
for value != 0 {
bit := bits.TrailingZeros64(value)
value &^= 1 << bit
for len(data) <= bit {
data = append(data, NewRow())
}
data[bit].SetBit(column)
}
*bsi = data
}
// pivotDescending loops over nonzero BSI values in descending order.
// For each value, the provided function is called with the value and a slice of the associated columns.
// If limit or offset are not-nil, they will be applied.
// Applying a limit or offset may modify the pointed-to value.
func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
// This "pivot" algorithm works by treating the BSI data as a tree.
// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
@ -98,23 +88,209 @@ func (bsi bsiData) distribution(filter *Row) bsiData {
}
*/
// addBSI adds BSI values together.
var placeholderBitmap = roaring.NewBitmap()
// addBSI adds two BSI bitmaps together.
// It does not handle sign and has no concept of overflow.
func addBSI(x, y bsiData) bsiData {
if len(x) > len(y) {
x, y = y, x
// Accumulate row segments.
segments := make([][]rowSegment, len(x)+len(y))
xsegs, ysegs := segments[:len(x)], segments[len(x):]
for i, r := range x {
xsegs[i] = r.segments
}
carry := NewRow()
out := make(bsiData, 0, len(y))
for i, v := range x {
out = append(out, v.Xor(y[i]).Xor(carry))
carry = v.Intersect(y[i]).Union(v.Intersect(carry), y[i].Intersect(carry))
for i, r := range y {
ysegs[i] = r.segments
}
for _, v := range y[len(x):] {
out = append(out, v.Xor(carry))
carry = v.Intersect(carry)
var dst bsiData
var xbitmaps, ybitmaps []*roaring.Bitmap
for {
// Find the next shard.
next := ^uint64(0)
for _, s := range segments {
if len(s) == 0 {
continue
}
shard := s[0].shard
if shard < next {
next = shard
}
}
if next == ^uint64(0) {
// There are no remaining shards.
break
}
// Accumulate bitmaps for this shard.
xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
for i, segs := range xsegs {
if len(segs) == 0 || segs[0].shard != next {
continue
}
xsegs[i] = segs[1:]
bm := segs[0].data
if !bm.Any() {
continue
}
for len(xbitmaps) < i {
xbitmaps = append(xbitmaps, placeholderBitmap)
}
xbitmaps = append(xbitmaps, bm)
}
for i, segs := range ysegs {
if len(segs) == 0 || segs[0].shard != next {
continue
}
ysegs[i] = segs[1:]
bm := segs[0].data
if !bm.Any() {
continue
}
for len(ybitmaps) < i {
ybitmaps = append(ybitmaps, placeholderBitmap)
}
ybitmaps = append(ybitmaps, bm)
}
// Add the shard values together.
var out []*roaring.Bitmap
switch {
case len(xbitmaps) == 0:
// There are no values in x.
out = ybitmaps
case len(ybitmaps) == 0:
// There are no values in y.
out = xbitmaps
default:
out = roaring.Add(xbitmaps, ybitmaps)
}
// Convert the bitmaps to output segments.
for i, b := range out {
if !b.Any() {
continue
}
for len(dst) <= i {
dst = append(dst, NewRow())
}
dst[i].segments = append(dst[i].segments, rowSegment{
shard: next,
writable: true,
data: b,
n: b.Count(),
})
}
}
if carry.Any() {
out = append(out, carry)
}
return out
return dst
}
// rowBuilder builds a row quickly from individual values.
// It is optimized for the case in which values are generated sequentially.
type rowBuilder struct {
bm *roaring.Bitmap
mask *[1024]uint64
array []uint16
key uint64
n int32
}
// flushKey flushes the data at the current key to the bitmap.
func (b *rowBuilder) flushKey() {
var c *roaring.Container
switch {
case b.mask != nil:
c = roaring.NewContainerBitmapN(b.mask[:], b.n)
b.mask = nil
case len(b.array) > 0:
c = roaring.NewContainerArrayCopy(b.array)
b.array = b.array[:0]
default:
return
}
if b.bm == nil {
b.bm = roaring.NewBitmap()
}
if old := b.bm.Containers.Get(b.key); old != nil {
c = roaring.Union(c, old)
}
b.bm.Containers.Put(b.key, c)
}
// Add a value to the bitmap.
// Values must be added sequentially.
func (b *rowBuilder) Add(v uint64) {
vkey := v / (1 << 16)
if b.key != vkey {
// This is a new key, so flush the old one.
b.flushKey()
b.key = vkey
}
if b.mask != nil {
// Add to the mask.
b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
b.mask[uint16(v)/64] |= 1 << (v % 64)
return
}
// Add to an array.
b.array = append(b.array, uint16(v))
if len(b.array) >= roaring.ArrayMaxSize {
// The array is too big.
// Convert it to a bitmask.
m := [1024]uint64{}
for _, v := range b.array {
m[v/64] |= 1 << (v % 64)
}
b.n = int32(len(b.array))
b.array = b.array[:0]
b.mask = &m
}
}
// Build a Row from stored data.
// This resets the builder.
func (b *rowBuilder) Build() *Row {
// Flush the active key to the bitmap.
b.flushKey()
// Remove the bitmap and convert it to a Row.
bm := b.bm
b.bm = nil
if bm == nil {
return NewRow()
}
return NewRowFromBitmap(bm)
}
// bsiBuilder assembles BSI data.
// It is optimized for the case in which values are generated sequentially.
type bsiBuilder []rowBuilder
// Insert a value into the BSI data.
// Columns must be inserted sequentially, and duplicates are not allowed.
func (b *bsiBuilder) Insert(col, val uint64) {
for val != 0 {
i := bits.TrailingZeros64(val)
val &^= 1 << i
for len(*b) <= i {
*b = append(*b, rowBuilder{})
}
(*b)[i].Add(col)
}
}
// Build BSI data.
// This resets the builder.
func (b *bsiBuilder) Build() bsiData {
builders := *b
*b = builders[:0]
rows := make(bsiData, len(builders))
for i := range builders {
rows[i] = builders[i].Build()
}
return rows
}

175
bsi_test.go Normal file
View file

@ -0,0 +1,175 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"math/rand"
"sort"
"testing"
)
// TestBSIAdd does a number of iterations. For each iteration, it
// generates a random number of ids, and two random values for each id
// to add together.
func TestBSIAdd(t *testing.T) {
// TODO wouldn't it be cool if our test suite had a randomized
// burn-in mode where you could run any test which supported it
// with a random seed and way more iterations?
rnd := rand.New(rand.NewSource(99))
//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
var builderA, builderB bsiBuilder
// a and b are generated slices of numbers to add together
var a, b []uint64
// idToIndex maps record ids to indexes in a and b
idToIndex := make(map[int]int)
// indexToID has the record id for each value in a and b
indexToID := []uint64{}
min := 999999999
max := 0
for iteration := 0; iteration < 1; iteration++ {
t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
// reset generated data
a, b = a[:0], b[:0]
indexToID = indexToID[:0]
for k := range idToIndex {
delete(idToIndex, k)
}
// z generates the values, they can be fairly large, but are usually small
z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
id := -1
for i := 0; true; i++ {
// get the next id, skipping a random amount
id = id + int(idZipf.Uint64()+1)
if id >= ShardWidth {
if i < min {
min = i
}
if max < i {
max = i
}
t.Log("num: ", i)
break
}
idToIndex[id] = int(i)
indexToID = append(indexToID, uint64(id))
// append a random value to each data slice
a = append(a, z.Uint64())
b = append(b, z.Uint64())
}
// build the BSIs based on the data slices and generated IDs
for index, id := range indexToID {
va, vb := a[index], b[index]
builderA.Insert(uint64(id), va)
builderB.Insert(uint64(id), vb)
}
dataA, dataB := builderA.Build(), builderB.Build()
dataC := addBSI(dataA, dataB)
// build results from added bsiData; results[i] should hold a[i]+b[i]
results := make([]uint64, len(a))
dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
for _, id := range ids {
results[idToIndex[int(id)]] = count
}
})
for i, res := range results {
if res != a[i]+b[i] {
t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
}
}
})
}
t.Log("min", min)
t.Log("max", max)
}
type bsiAddCase struct {
positions []uint64
a []uint64
b []uint64
}
func (b bsiAddCase) Len() int {
return len(b.positions)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (b bsiAddCase) Less(i, j int) bool {
return b.positions[i] < b.positions[j]
}
// Swap swaps the elements with indexes i and j.
func (b bsiAddCase) Swap(i, j int) {
b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
b.a[i], b.a[j] = b.a[j], b.a[i]
b.b[i], b.b[j] = b.b[j], b.b[i]
}
// TestBSIAddCases tests specific cases of bsiAdd (would generally be
// pulled from randomly generated ones from TestBSIAdd upon failure).
func TestBSIAddCases(t *testing.T) {
tests := []bsiAddCase{
{
positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
a: []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
b: []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
},
{
positions: []uint64{17570, 54607},
a: []uint64{1010, 71},
b: []uint64{25, 150},
},
}
var builderA, builderB bsiBuilder
for i, tst := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
}
sort.Sort(tst)
for i := 0; i < len(tst.a); i++ {
builderA.Insert(tst.positions[i], tst.a[i])
builderB.Insert(tst.positions[i], tst.b[i])
}
dataA, dataB := builderA.Build(), builderB.Build()
dataC := addBSI(dataA, dataB)
// maps id to count
results := make(map[uint64]uint64)
dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
for _, id := range ids {
results[id] = count
}
})
for i, id := range tst.positions {
if results[id] != tst.a[i]+tst.b[i] {
t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
}
}
})
}
}

305
executor.go
View file

@ -1875,6 +1875,7 @@ func (e *executor) executeTopK(ctx context.Context, qcx *Qcx, index string, c *p
}, nil
}
// executeTopKShard builds a perpendicular BSI bitmap of a shard for TopK.
func (e *executor) executeTopKShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ []*Row, err0 error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopKShard")
defer span.Finish()
@ -1897,6 +1898,22 @@ func (e *executor) executeTopKShard(ctx context.Context, qcx *Qcx, index string,
return nil, ErrFieldNotFound
}
// Parse "from" time, if set.
var fromTime time.Time
if v, ok := c.Args["from"]; ok {
if fromTime, err = parseTime(v); err != nil {
return nil, errors.Wrap(err, "parsing from time")
}
}
// Parse "to" time, if set.
var toTime time.Time
if v, ok := c.Args["to"]; ok {
if toTime, err = parseTime(v); err != nil {
return nil, errors.Wrap(err, "parsing to time")
}
}
// Fetch the filter.
var filterBitmap *Row
if filter, hasFilter, err := c.CallArg("filter"); err != nil {
@ -1919,13 +1936,19 @@ func (e *executor) executeTopKShard(ctx context.Context, qcx *Qcx, index string,
ftype := f.Type()
switch ftype {
case FieldTypeSet, FieldTypeTime:
case FieldTypeTime:
if !(fromTime.IsZero() && toTime.IsZero()) {
return e.executeTopKShardTime(ctx, tx, filterBitmap, index, fieldName, shard, fromTime, toTime)
}
fallthrough
case FieldTypeSet:
return e.executeTopKShardSet(ctx, tx, filterBitmap, index, fieldName, shard)
default:
return nil, errors.Errorf("field type %q is not yet supported by TopK", ftype)
}
}
// executeTopKShardSet builds a perpendicular BSI bitmap of a set field within a shard.
func (e *executor) executeTopKShardSet(ctx context.Context, tx Tx, filter *Row, index, field string, shard uint64) ([]*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopKShardSet")
defer span.Finish()
@ -1935,7 +1958,285 @@ func (e *executor) executeTopKShardSet(ctx context.Context, tx Tx, filter *Row,
return nil, nil
}
return f.cardinalityBSISet(ctx, tx, filter)
return topKFragments(ctx, tx, filter, f)
}
// executeTopKShardTime builds a perpendicular BSI bitmap of a time field within a shard.
func (e *executor) executeTopKShardTime(ctx context.Context, tx Tx, filter *Row, index, field string, shard uint64, from, to time.Time) ([]*Row, error) {
// Fetch index.
idx := e.Holder.Index(index)
if idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
}
// Fetch field.
f := idx.Field(field)
if f == nil {
return nil, newNotFoundError(ErrFieldNotFound, field)
}
// Check the time quantum.
quantum := f.TimeQuantum()
if quantum == "" {
// ????????
return nil, nil
}
// Fetch fragments.
var fragments []*fragment
for _, view := range viewsByTimeRange(viewStandard, from, to, quantum) {
f := e.Holder.fragment(index, field, view, shard)
if f == nil {
continue
}
fragments = append(fragments, f)
}
return topKFragments(ctx, tx, filter, fragments...)
}
// topKFragments builds a perpendicular BSI bitmap from fragments.
// The fragments are expected to be from set fields.
func topKFragments(ctx context.Context, tx Tx, filter *Row, fragments ...*fragment) (bsiData, error) {
// Acquire fragment container iterators.
iters := make([]roaring.ContainerIterator, len(fragments))
for i, f := range fragments {
f.mu.RLock()
defer f.mu.RUnlock()
iter, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, 0)
if err != nil {
return nil, err
}
iters[i] = iter
}
// Merge to a single container iterator.
var it roaring.ContainerIterator
if len(iters) == 1 {
it = iters[0]
} else {
it = mergerate(iters...)
}
// Extract filter data if a filter was provided.
var filterData *topKFilter
if filter != nil {
var f topKFilter
f.fill(filter)
filterData = &f
}
return doTopK(ctx, it, filterData)
}
// mergerate returns a container iterator that unions many container iterators.
func mergerate(iters ...roaring.ContainerIterator) *mergerator {
iterStates := make([]mergeState, len(iters))
for i, s := range iters {
iterStates[i].iter = s
}
m := mergerator{
iters: iterStates,
heap: make(mergeratorHeap, 0, len(iters)),
}
for i := range iterStates {
m.pusherate(uint64(i))
}
return &m
}
// mergerator is a container iterator that merges container iterators (via unioning).
type mergerator struct {
iters []mergeState
heap mergeratorHeap
container *roaring.Container
key uint64
}
// pusherate pushes the iterator at the given index back onto the heap.
func (m *mergerator) pusherate(idx uint64) {
state := &m.iters[idx]
it := state.iter
if !it.Next() {
it.Close()
return
}
key, c := it.Value()
state.c = c
m.heap.push(mergeNode{
key: key,
idx: idx,
})
}
func (m *mergerator) Next() bool {
nodes := m.heap.pop()
if len(nodes) == 0 {
return false
}
key := nodes[0].key
var container *roaring.Container
for _, n := range nodes {
c := m.iters[n.idx].c
if container != nil {
container = roaring.Union(container, c)
} else {
container = c
}
m.pusherate(n.idx)
}
m.key, m.container = key, container
return true
}
func (m *mergerator) Value() (uint64, *roaring.Container) {
return m.key, m.container
}
func (m *mergerator) Close() {
for _, n := range m.heap {
m.iters[n.idx].iter.Close()
}
m.heap = nil
}
type mergeState struct {
c *roaring.Container
iter roaring.ContainerIterator
}
// mergeratorHeap is a binary min-heap over keys.
// This is used to find the next iterator to hit.
type mergeratorHeap []mergeNode
type mergeNode struct {
key, idx uint64
}
// push a node onto the heap.
func (h *mergeratorHeap) push(node mergeNode) {
s := *h
i := len(s)
s = append(s, node)
for i != 0 && s[(i-1)/2].key > s[i].key {
s[(i-1)/2], s[i] = s[i], s[(i-1)/2]
i = (i - 1) / 2
}
*h = s
}
// pop the minimum key off of the heap.
// If there are multiple iterators with this keys, this returns all of them.
func (h *mergeratorHeap) pop() []mergeNode {
s := *h
if len(s) == 0 {
return nil
}
n := 0
for key := s[0].key; len(s) > n && s[0].key == key; n++ {
s[0], s[len(s)-n-1] = s[len(s)-n-1], s[0]
s[:len(s)-n-1].minHeapify()
}
*h = s[:len(s)-n]
return s[len(s)-n:]
}
// minHeapify fixes the heap invariant after updating the heap's root.
func (h mergeratorHeap) minHeapify() {
i := 0
for {
l, r := 2*i+1, 2*i+2
min := i
if l < len(h) && h[l].key < h[min].key {
min = l
}
if r < len(h) && h[r].key < h[min].key {
min = r
}
if min == i {
return
}
h[min], h[i] = h[i], h[min]
i = min
}
}
// doTopK uses a raw Pilosa matrix to produce a perpendicular BSI bitmap.
// It will apply a row filter if one is provided.
func doTopK(ctx context.Context, it roaring.ContainerIterator, filter *topKFilter) (bsiData, error) {
row := ^uint64(0)
var count uint64
var builder bsiBuilder
var i uint16
for it.Next() {
if i == 0 {
if err := ctx.Err(); err != nil {
return nil, err
}
}
i++
// Fetch the next container.
key, container := it.Value()
keyrow, subkey := key/(ShardWidth>>16), key%(ShardWidth>>16)
if keyrow != row {
// The previous row has ended.
// Flush the count to the BSI data.
builder.Insert(row, count)
row, count = keyrow, 0
}
// Add the selected bits to the count.
if filter != nil {
fc := filter[subkey]
if fc == nil {
continue
}
count += uint64(roaring.IntersectionCount(container, fc))
} else {
count += uint64(container.N())
}
}
// Add the final count to the BSI data.
builder.Insert(row, count)
// Construct the result.
return builder.Build(), nil
}
// topKFilter is a row filter for a TopK query.
// It is represented as a contiguous array of containers.
type topKFilter [ShardWidth >> 16]*roaring.Container
// fill the filter with the contents of a Row.
func (f *topKFilter) fill(row *Row) {
for _, s := range row.segments {
it, _ := s.data.Containers.Iterator(0)
f.fillIt(it)
}
// I don't think multiple segments make sense here?
}
func (f *topKFilter) fillIt(it roaring.ContainerIterator) {
defer it.Close()
for it.Next() {
key, c := it.Value()
key %= uint64(len(f))
if f[key] != nil {
panic("duplicate container in topk filter")
}
f[key] = c
}
}
// executeTopN executes a TopN() call.

31
executor_test.go
View file

@ -1063,7 +1063,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
})
}
func TestExecutor_Execute_TopK(t *testing.T) {
func TestExecutor_Execute_TopK_Set(t *testing.T) {
c := test.MustRunCluster(t, 2)
defer c.Close()
@ -1094,6 +1094,35 @@ func TestExecutor_Execute_TopK(t *testing.T) {
}
}
func TestExecutor_Execute_TopK_Time(t *testing.T) {
c := test.MustRunCluster(t, 2)
defer c.Close()
// Load some test data into a time field.
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", true))
c.Query(t, "i", `
Set(0, f=0, 2016-01-02T00:00)
Set(0, f=1, 2016-01-02T00:00)
Set(0, f=0, 2016-01-03T00:00)
Set(1, f=0, 2016-01-10T00:00)
Set(100000000, f=2, 2016-02-02T00:00)
Set(200000000, f=3, 2015-01-02T00:00)
`)
// Execute query.
if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopK(f, k=3, from=2016-01-01T00:00, to=2016-01-11T00:00)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 0, Count: 2},
{ID: 1, Count: 1},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
// Ensure a TopN() query can be executed.
func TestExecutor_Execute_TopN(t *testing.T) {
t.Run("RowIDColumnID", func(t *testing.T) {

30
fragment.go
View file

@ -1775,36 +1775,6 @@ func (f *fragment) forEachBit(tx Tx, fn func(rowID, columnID uint64) error) erro
})
}
// cardinalityBSISet constructs a perpendicular BSI bitmap containing the cardinality of each specified row in a set field.
func (f *fragment) cardinalityBSISet(ctx context.Context, tx Tx, filter *Row) ([]*Row, error) {
f.mu.Lock()
defer f.mu.Unlock()
// Fetch row IDs.
rowIDs, err := f.unprotectedRows(ctx, tx, 0)
if err != nil {
return nil, err
}
// Count the bits in each row.
var out bsiData
for _, id := range rowIDs {
row, err := f.unprotectedRow(tx, id)
if err != nil {
return nil, err
}
var count uint64
if filter != nil {
count = row.intersectionCount(filter)
} else {
count = row.Count()
}
out.insert(id, count)
}
return out, nil
}
// top returns the top rows from the fragment.
// If opt.Src is specified then only rows which intersect src are returned.
// If opt.FilterValues exist then the row attribute specified by field is matched.

3
go.mod
View file

@ -45,7 +45,8 @@ require (
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/zeebo/blake3 v0.0.4
go.etcd.io/bbolt v1.3.5
golang.org/x/mod v0.3.0
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/text v0.3.3 // indirect

45
go.sum
View file

@ -1,6 +1,8 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
@ -56,6 +58,7 @@ github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06A
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
github.com/glycerine/lmdb-go v1.9.34 h1:0lymJjpdelYnIMcNzsKROfIaApt99zhaHtjDJTHjGkE=
github.com/glycerine/lmdb-go v1.9.34/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@ -70,19 +73,14 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@ -94,9 +92,7 @@ github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@ -127,7 +123,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@ -162,7 +157,6 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -173,9 +167,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -209,10 +201,8 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+D
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@ -224,7 +214,6 @@ github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@ -232,7 +221,6 @@ github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
@ -254,7 +242,6 @@ github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@ -265,21 +252,25 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 h1:2/QncOxxpPAdiH+E00abYw/SaQG353gltz79Nl1zrYE=
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7/go.mod h1:1phAWC201xIgDyaFpmDeZkgf70Q4Pd/CNqfRtVPtxNw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 h1:xUIPaMhvROX9dhPvRCenIJtU78+lbEenGbgqB5hfHCQ=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -287,16 +278,13 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -306,14 +294,14 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@ -324,17 +312,15 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
@ -345,7 +331,6 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ
google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=

2
pql/ast.go
View file

@ -422,6 +422,8 @@ var callInfoByFunc = map[string]callInfo{
"_field": "",
"k": int64(0),
"filter": nil,
"from": nil,
"to": nil,
},
},

861
roaring/add.go Normal file
View file

@ -0,0 +1,861 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
import (
"fmt"
"math/bits"
"unsafe"
)
// Add two BSI bitmaps producing a new BSI bitmap.
func Add(x, y []*Bitmap) []*Bitmap {
// Collect iterators.
type itNode struct {
it ContainerIterator
c *Container
key uint64
done bool
}
xits := make([]itNode, len(x))
for i, b := range x {
it, _ := b.Containers.Iterator(0)
defer it.Close()
node := &xits[i]
node.it = it
if !it.Next() {
node.done = true
continue
}
node.key, node.c = it.Value()
}
yits := make([]itNode, len(y))
for i, b := range y {
it, _ := b.Containers.Iterator(0)
defer it.Close()
node := &yits[i]
node.it = it
if !it.Next() {
node.done = true
continue
}
node.key, node.c = it.Value()
}
bits := len(x)
if len(y) > len(x) {
bits = len(y)
}
var carryRing [2]carryBuffer
var temp [1024]uint64
var dst []*Bitmap
for {
key := ^uint64(0)
for i := range xits {
if xits[i].done {
continue
}
k := xits[i].key
if k < key {
key = k
}
}
for i := range yits {
if yits[i].done {
continue
}
k := yits[i].key
if k < key {
key = k
}
}
if key == ^uint64(0) {
break
}
carryRing[0].clear()
for i := 0; i <= bits; i++ {
var x, y *Container
if i < len(xits) && xits[i].key == key && !xits[i].done {
x = xits[i].c
if xits[i].it.Next() {
xits[i].key, xits[i].c = xits[i].it.Value()
} else {
xits[i].done = true
}
}
if i < len(yits) && yits[i].key == key && !yits[i].done {
y = yits[i].c
if yits[i].it.Next() {
yits[i].key, yits[i].c = yits[i].it.Value()
} else {
yits[i].done = true
}
}
c := fullAddContainers(x, y, &carryRing[i%2], &carryRing[1-(i%2)], &temp)
if c == nil {
continue
}
for i >= len(dst) {
dst = append(dst, NewBitmap())
}
dst[i].Containers.Put(key, c)
}
}
return dst
}
// fullAddContainers implements the bitwise formula for a 3-input-2-output full adder.
// Any of the 3 inputs may be nil, in which case they are treated as zeroes.
// The carry is written to carryOut, which must not be nil.
// The temp buffer will be used to store intermediate values, and can be safely stack-allocated.
func fullAddContainers(x, y *Container, carryIn, carryOut *carryBuffer, temp *[1024]uint64) *Container {
if roaringParanoia {
x.CheckN()
y.CheckN()
carryIn.check()
defer carryOut.check()
}
// Accumulate inputs.
var xm, ym, zm *[1024]uint64
var xa, ya, za []uint16
switch x.typ() {
case ContainerNil:
case ContainerBitmap:
xm = x.bitmask()
case ContainerArray:
xa = x.array()
case ContainerRun:
// Create a temporary mask on the stack.
// This should not happen very often.
var mask [1024]uint64
for _, r := range x.runs() {
splatRun(&mask, r)
}
xm = &mask
default:
panic("invalid container type")
}
switch y.typ() {
case ContainerNil:
case ContainerBitmap:
ym = y.bitmask()
case ContainerArray:
ya = y.array()
case ContainerRun:
// Create a temporary mask on the stack.
// This should not happen very often.
var mask [1024]uint64
for _, r := range y.runs() {
splatRun(&mask, r)
}
ym = &mask
default:
panic("invalid container type")
}
if carryIn != nil && carryIn.count > 0 {
if carryIn.isBitmap {
zm = carryIn.bitmap()
} else {
za = carryIn.array()[:carryIn.count]
}
}
// Handle each of the 27 possible mask/array/nil combinations.
switch {
case xm != nil:
dst, carry := temp, carryOut.bitmap()
var do, co uint32
switch {
case ym != nil:
switch {
case zm != nil:
do, co = addMaskMaskMaskToMask(dst, carry, xm, ym, zm)
case len(za) > 0:
do, co = addArrayMaskMaskToMask(dst, carry, za, xm, ym)
default:
do, co = addMaskMaskToMask(dst, carry, xm, ym)
}
case len(ya) > 0:
switch {
case zm != nil:
do, co = addArrayMaskMaskToMask(dst, carry, ya, xm, zm)
case len(za) > 0:
do, co = addArrayArrayMaskToMask(dst, carry, ya, za, xm, uint32(x.N()))
default:
do, co = addArrayArrayMaskToMask(dst, carry, ya, nil, xm, uint32(x.N()))
}
default:
switch {
case zm != nil:
do, co = addMaskMaskToMask(dst, carry, xm, zm)
case len(za) > 0:
do, co = addArrayArrayMaskToMask(dst, carry, za, nil, xm, uint32(x.N()))
default:
carryOut.clear()
return x
}
}
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
return NewContainerBitmapN(append([]uint64(nil), dst[:]...), int32(do))
case len(xa) > 0:
switch {
case ym != nil:
dst, carry := temp, carryOut.bitmap()
var do, co uint32
switch {
case zm != nil:
do, co = addArrayMaskMaskToMask(dst, carry, xa, ym, zm)
case len(za) > 0:
do, co = addArrayArrayMaskToMask(dst, carry, xa, za, ym, uint32(y.N()))
default:
do, co = addArrayArrayMaskToMask(dst, carry, xa, nil, ym, uint32(y.N()))
}
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), dst[:]...), int32(do))
case len(ya) > 0:
switch {
case zm != nil:
do, co := addArrayArrayMaskToMask(temp, carryOut.bitmap(), xa, ya, zm, carryIn.count)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
case len(za) > 0:
if do, co := addArrayArrayArrayToArray((*[4096]uint16)(unsafe.Pointer(temp)), carryOut.array(), xa, ya, za); do|co != ^uint16(0) {
carryOut.isBitmap = false
carryOut.count = uint32(co)
if do == 0 {
return nil
}
return NewContainerArrayCopy((*[4096]uint16)(unsafe.Pointer(temp))[:do])
}
do, co := addArrayArrayArrayToMask(temp, carryOut.bitmap(), xa, ya, za)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
default:
if do, co := addArrayArrayToArray((*[4096]uint16)(unsafe.Pointer(temp)), carryOut.array(), xa, ya); do|co != ^uint16(0) {
carryOut.isBitmap = false
carryOut.count = uint32(co)
if do == 0 {
return nil
}
return NewContainerArrayCopy((*[4096]uint16)(unsafe.Pointer(temp))[:do])
}
do, co := addArrayArrayArrayToMask(temp, carryOut.bitmap(), xa, ya, nil)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
}
default:
switch {
case zm != nil:
do, co := addArrayArrayMaskToMask(temp, carryOut.bitmap(), xa, nil, zm, carryIn.count)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
case len(za) > 0:
if do, co := addArrayArrayToArray((*[4096]uint16)(unsafe.Pointer(temp)), carryOut.array(), xa, za); do|co != ^uint16(0) {
carryOut.isBitmap = false
carryOut.count = uint32(co)
if do == 0 {
return nil
}
return NewContainerArrayCopy((*[4096]uint16)(unsafe.Pointer(temp))[:do])
}
do, co := addArrayArrayArrayToMask(temp, carryOut.bitmap(), xa, za, nil)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
default:
carryOut.clear()
return x
}
}
default:
switch {
case ym != nil:
dst, carry := temp, carryOut.bitmap()
var do, co uint32
switch {
case zm != nil:
do, co = addMaskMaskToMask(dst, carry, ym, zm)
case len(za) > 0:
do, co = addArrayArrayMaskToMask(dst, carry, za, nil, ym, uint32(y.N()))
default:
carryOut.clear()
return y
}
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), dst[:]...), int32(do))
case len(ya) > 0:
switch {
case zm != nil:
do, co := addArrayArrayMaskToMask(temp, carryOut.bitmap(), ya, nil, zm, carryIn.count)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
case len(za) > 0:
if do, co := addArrayArrayToArray((*[4096]uint16)(unsafe.Pointer(temp)), carryOut.array(), ya, za); do|co != ^uint16(0) {
carryOut.isBitmap = false
carryOut.count = uint32(co)
if do == 0 {
return nil
}
return NewContainerArrayCopy((*[4096]uint16)(unsafe.Pointer(temp))[:do])
}
do, co := addArrayArrayArrayToMask(temp, carryOut.bitmap(), ya, za, nil)
if co > 0 {
carryOut.isBitmap = true
carryOut.count = co
} else {
carryOut.clear()
}
if do == 0 {
return nil
}
return NewContainerBitmapN(append([]uint64(nil), temp[:]...), int32(do))
default:
carryOut.clear()
return y
}
default:
carryOut.clear()
return carryIn.containerize()
}
}
}
// carryBuffer is a buffer used to store carry bits.
// It is designed to be stack-allocated.
// As such **IT SHOULD NOT BE POOLED**.
type carryBuffer struct {
isBitmap bool
count uint32
data [1024]uint64
}
func (b *carryBuffer) bitmap() *[1024]uint64 {
return &b.data
}
func (b *carryBuffer) array() *[4096]uint16 {
return (*[4096]uint16)(unsafe.Pointer(&b.data))
}
// compact converts the buffer to an array if it would be more efficient.
func (b *carryBuffer) compact() {
if b.isBitmap && b.count < 4096 {
b.compactSlow()
}
}
func (b *carryBuffer) compactSlow() {
var buf [4096]uint16
i := 0
for j, v := range b.data {
for v != 0 {
k := bits.TrailingZeros64(v)
v &^= 1 << k
buf[i] = 64*uint16(j) + uint16(k)
i++
}
}
copy(b.array()[:], buf[:i])
b.isBitmap = false
if roaringParanoia {
b.check()
}
}
// clear the buffer, making it effectively full of zeroes.
func (b *carryBuffer) clear() {
b.isBitmap = false
b.count = 0
}
// check that invariants hold.
// This exists mainly for debugging.
func (b *carryBuffer) check() {
if b.isBitmap {
var count int
for _, v := range b.bitmap() {
count += bits.OnesCount64(v)
}
if uint32(count) != b.count {
panic(fmt.Errorf("count mismatch: reported %d but got %d", b.count, count))
}
} else {
if b.count > uint32(len(b.array())) {
panic(fmt.Errorf("found too many bits: %d of a max of %d", b.count, len(b.array())))
}
if b.count > 0 {
arr := b.array()[:b.count]
for i, v := range arr {
if i > 0 && v <= arr[i-1] {
panic(fmt.Errorf("broken array: %d after %d", v, arr[i-1]))
}
}
}
}
}
// containerize the contents of the buffer.
func (b *carryBuffer) containerize() *Container {
if b.count == 0 {
return nil
}
b.compact()
if !b.isBitmap {
return NewContainerArray(append([]uint16(nil), b.array()[:b.count]...))
}
return NewContainerBitmapN(append([]uint64(nil), b.bitmap()[:]...), int32(b.count))
}
// addArrayArrayToArray implemnents a half-adder over two arrays, producing array outputs.
// If the results are too big, this returns ^uint16(0) to indicate that the operation failed.
func addArrayArrayToArray(dst, carry *[4096]uint16, x, y []uint16) (uint16, uint16) {
_, _ = &dst[0], &carry[0]
// Half-add x and y.
i, j, do, co := 0, 0, 0, 0
for i < len(x) && j < len(y) {
a, b := x[i], y[j]
switch {
case a < b:
// Copy all values under b to the lower output bit.
for ; i < len(x) && x[i] < b; i++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = x[i]
do++
}
case b < a:
// Copy all values under a to the lower output bit.
for ; j < len(y) && y[j] < a; j++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = y[j]
do++
}
default:
// Copy the value to the carry.
if co >= len(carry) {
return ^uint16(0), ^uint16(0)
}
carry[co] = a
co++
i++
j++
}
}
// Copy the remaining data to the lower output bit.
var remaining []uint16
switch {
case i < len(x):
remaining = x[i:]
case j < len(y):
remaining = y[j:]
}
if do+len(remaining) > len(dst) {
return ^uint16(0), ^uint16(0)
}
copy(dst[do:], remaining)
return uint16(do + len(remaining)), uint16(co)
}
// addArrayArrayArrayToArray implemnents a full-adder over three arrays, producing array outputs.
// If the results are too big, this returns ^uint16(0) to indicate that the operation failed.
func addArrayArrayArrayToArray(dst, carry *[4096]uint16, x, y, z []uint16) (uint16, uint16) {
_, _ = &dst[0], &carry[0]
// Run a full adder.
i, j, k, do, co := 0, 0, 0, 0, 0
for i < len(x) && j < len(y) && k < len(z) {
a, b, c := x[i], y[j], z[k]
switch {
case a < b && a < c:
// Find the lowest value in the other two inputs.
next := b
if c < b {
next = c
}
// Copy every value below that to the lower output bit.
for ; i < len(x) && x[i] < next; i++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = x[i]
do++
}
case b < a && b < c:
// Find the lowest value in the other two inputs.
next := a
if c < a {
next = c
}
// Copy every value below that to the lower output bit.
for ; j < len(y) && y[j] < next; j++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = y[j]
do++
}
case c < a && c < b:
// Find the lowest value in the other two inputs.
next := a
if b < a {
next = b
}
// Copy every value below that to the lower output bit.
for ; k < len(z) && z[k] < next; k++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = z[k]
do++
}
case co >= len(carry):
// At least two bits are set, so a carry bit will be produced.
return ^uint16(0), ^uint16(0)
case a == b && b != c:
// Carry the lower value (a/b).
carry[co] = a
co++
i++
j++
case b == c && a != b:
// Carry the lower value (b/c).
carry[co] = b
co++
j++
k++
case a == c && a != b:
// Carry the lower value (a/c).
carry[co] = a
co++
i++
k++
case do >= len(dst):
// All three inputs are set, so this will produce both a lower bit and a carry.
return ^uint16(0), ^uint16(0)
default:
// a == b == c; 1+1+1 = 0b11
dst[do] = a
do++
carry[co] = a
co++
i++
j++
k++
}
}
// Run a half adder.
if k < len(z) {
// Re-order the inputs such that the inputs (if any) which still have data are x and y.
if i < len(x) {
j, y = k, z
} else {
i, x = k, z
}
}
for i < len(x) && j < len(y) {
a, b := x[i], y[j]
switch {
case a < b:
// Copy all values under b to the lower output bit.
for ; i < len(x) && x[i] < b; i++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = x[i]
do++
}
case b < a:
// Copy all values under a to the lower output bit.
for ; j < len(y) && y[j] < a; j++ {
if do >= len(dst) {
return ^uint16(0), ^uint16(0)
}
dst[do] = y[j]
do++
}
default:
// Copy the value to the carry.
if co >= len(carry) {
return ^uint16(0), ^uint16(0)
}
carry[co] = a
co++
i++
j++
}
}
// Copy the remaining data to the lower output bit.
var remaining []uint16
switch {
case i < len(x):
remaining = x[i:]
case j < len(y):
remaining = y[j:]
}
if do+len(remaining) > len(dst) {
return ^uint16(0), ^uint16(0)
}
copy(dst[do:], remaining)
return uint16(do + len(remaining)), uint16(co)
}
// addArrayArrayArrayToMask implemnents a full-adder over three arrays, producing bitmask outputs.
// This is generally only needed when addArrayArrayArrayToArray fails.
func addArrayArrayArrayToMask(dst, carry *[1024]uint64, x, y, z []uint16) (uint32, uint32) {
// Start with blank outputs.
*dst = [1024]uint64{}
*carry = [1024]uint64{}
var co uint64
for _, v := range x {
// Add each value to the lower output bit.
dst[v/64] |= 1 << (v % 64)
// There is no carry because this is the first copy of the output.
}
for _, v := range y {
// Increment the carry output counter if the value is already included.
co += (dst[v/64] >> (v % 64)) & 1
// Insert the carry bit if the value is already in the lower output bit.
carry[v/64] |= dst[v/64] & (1 << (v % 64))
// Flip the lower output bit for the value.
dst[v/64] ^= 1 << (v % 64)
}
for _, v := range z {
// Increment the carry output counter if the value is already included.
co += (dst[v/64] >> (v % 64)) & 1
// Insert the carry bit if the value is already in the lower output bit.
carry[v/64] |= dst[v/64] & (1 << (v % 64))
// Flip the lower output bit for the value.
dst[v/64] ^= 1 << (v % 64)
}
// If there were no collisions, the number of values in the lower output bit would be equal to the sum of the counts of the inputs.
// For each carry, we subtract 2 as it was produced by combining two copies of a value.
return uint32(len(x)+len(y)+len(z)) - 2*uint32(co), uint32(co)
}
// addArrayArrayMaskToMask implemnents a full-adder over two arrays and one bitmask, producing bitmask outputs.
func addArrayArrayMaskToMask(dst, carry *[1024]uint64, x, y []uint16, z *[1024]uint64, zc uint32) (uint32, uint32) {
// Clear the carry output.
*carry = [1024]uint64{}
// Copy the mask input to the lower output bit.
*dst = *z
var co uint64
for _, v := range x {
// Increment the carry output counter if the value is already included.
co += (dst[v/64] >> (v % 64)) & 1
// Insert the carry bit if the value is already in the lower output bit.
carry[v/64] |= dst[v/64] & (1 << (v % 64))
// Flip the lower output bit for the value.
dst[v/64] ^= 1 << (v % 64)
}
for _, v := range y {
// Increment the carry output counter if the value is already included.
co += (dst[v/64] >> (v % 64)) & 1
// Insert the carry bit if the value is already in the lower output bit.
carry[v/64] |= dst[v/64] & (1 << (v % 64))
// Flip the lower output bit for the value.
dst[v/64] ^= 1 << (v % 64)
}
// If there were no collisions, the number of values in the lower output bit would be equal to the sum of the counts of the inputs.
// For each carry, we subtract 2 as it was produced by combining two copies of a value.
return uint32(len(x)+len(y)) + zc - 2*uint32(co), uint32(co)
}
// addArrayArrayMaskToMask implemnents a full-adder over one array and two bitmasks, producing bitmask outputs.
func addArrayMaskMaskToMask(dst, carry *[1024]uint64, x []uint16, y, z *[1024]uint64) (uint32, uint32) {
_, _, _, _ = &dst[0], &carry[0], &y[0], &z[0]
// Do a bitwise combine of y and z into dst and carry.
var doi, coi int
for i := range dst {
yv, zv := y[i], z[i]
dstv, carryv := yv^zv, yv&zv
dst[i], carry[i] = dstv, carryv
doi += bits.OnesCount64(dstv)
coi += bits.OnesCount64(carryv)
}
// Add the array values.
var newco uint64
for _, v := range x {
// Increment the carry output counter if the value is already included.
newco += (dst[v/64] >> (v % 64)) & 1
// Insert the carry bit if the value is already in the lower output bit.
carry[v/64] |= dst[v/64] & (1 << (v % 64))
// Flip the lower output bit for the value.
dst[v/64] ^= 1 << (v % 64)
}
// If there were no collisions, the number of values in the lower output bit would be equal to the sum of the counts of the inputs.
// For each carry, we subtract 2 as it was produced by combining two copies of a value.
return uint32(doi) + uint32(len(x)) - 2*uint32(newco), uint32(coi) + uint32(newco)
}
// addMaskMaskToMask implemnents a half-adder over two bitmasks, producing bitmask outputs.
func addMaskMaskToMask(dst, carry *[1024]uint64, x, y *[1024]uint64) (uint32, uint32) {
_, _, _, _ = &dst[0], &carry[0], &x[0], &y[0]
// Do a bitwise combine of both inputs into dst and carry.
var do, co int
for i := range dst {
xv, yv := x[i], y[i]
dstv, carryv := xv^yv, xv&yv
dst[i], carry[i] = dstv, carryv
do += bits.OnesCount64(dstv)
co += bits.OnesCount64(carryv)
}
return uint32(do), uint32(co)
}
// addMaskMaskMaskToMask implemnents a full-adder over three bitmasks, producing bitmask outputs.
func addMaskMaskMaskToMask(dst, carry *[1024]uint64, x, y, z *[1024]uint64) (uint32, uint32) {
_, _, _, _, _ = &dst[0], &carry[0], &x[0], &y[0], &z[0]
// Do a bitwise combine of all three inputs into dst and carry.
var do, co int
for i := range dst {
xv, yv, zv := x[i], y[i], z[i]
dstv, carryv := xv^yv^zv, (xv&yv)|(yv&zv)|(xv&zv)
dst[i], carry[i] = dstv, carryv
do += bits.OnesCount64(dstv)
co += bits.OnesCount64(carryv)
}
return uint32(do), uint32(co)
}

416
roaring/add_test.go Normal file
View file

@ -0,0 +1,416 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !race
package roaring
import (
"fmt"
"math/bits"
"testing"
"golang.org/x/exp/rand"
)
func randomMask(dst *[1024]uint64, seed uint64) uint32 {
_ = &dst[0]
var src rand.PCGSource
src.Seed(seed)
var n uint32
for i := range dst {
v := src.Uint64()
n += uint32(bits.OnesCount64(v))
dst[i] = src.Uint64()
}
return n
}
func randomArray(arr *[4096]uint16, mask *[1024]uint64, seed uint64) []uint16 {
_, _ = &arr[0], &mask[0]
var src rand.PCGSource
src.Seed(seed)
*mask = [1024]uint64{}
nv := src.Uint64()
n := (nv % 4096) >> ((nv / 4096) % 10)
for i := uint64(0); i < n; i++ {
v := uint16(src.Uint64())
mask[v/64] |= 1 << (v % 64)
}
x := arr[:0]
for i, v := range mask {
for v != 0 {
j := bits.TrailingZeros64(v)
v &^= 1 << j
x = append(x, 64*uint16(i)+uint16(j))
}
}
return x
}
func splatArray(arr ...uint16) (dst [1024]uint64) {
for _, v := range arr {
dst[v/64] |= 1 << (v % 64)
}
return
}
func diffMask(t *testing.T, in string, expect, got *[1024]uint64) bool {
t.Helper()
n := 0
for i := range expect {
expv, gotv := expect[i], got[i]
unexpected, missing := gotv&^expv, expv&^gotv
for unexpected != 0 {
j := bits.TrailingZeros64(unexpected)
unexpected &^= 1 << j
t.Errorf("unexpected value %d in %s", 64*i+j, in)
}
for missing != 0 {
j := bits.TrailingZeros64(missing)
missing &^= 1 << j
t.Errorf("missing value %d in %s", 64*i+j, in)
}
n += bits.OnesCount64(unexpected | missing)
if n > 20 {
t.Errorf("too many errors in %s", in)
break
}
}
return n > 0
}
// TestAddInternal randomly tests the full and half adder logic in Add.
func TestAddInternal(t *testing.T) {
t.Parallel()
testCount := 2000
if testing.Short() {
testCount = 20
}
t.Run("addArrayArrayToArray", func(t *testing.T) {
t.Parallel()
var src rand.PCGSource
src.Seed(1)
seedup := make(map[uint64]struct{})
for i := 0; i < testCount; i++ {
genSeed:
seed := src.Uint64()
if _, dup := seedup[seed]; dup {
goto genSeed
}
seedup[seed] = struct{}{}
t.Run(fmt.Sprint(seed), func(t *testing.T) {
t.Parallel()
tries := 0
var xmask, ymask [1024]uint64
var xarr, yarr [4096]uint16
var gotdst, gotcarry [4096]uint16
try:
x := randomArray(&xarr, &xmask, seed)
y := randomArray(&yarr, &ymask, 3*seed)
do, co := addArrayArrayToArray(&gotdst, &gotcarry, x, y)
if do|co == ^uint16(0) {
if tries > 100 {
t.Fatal("repeatedly failing")
}
seed++
tries++
goto try
}
var dstmask, dstcarry [1024]uint64
expdo, expco := addMaskMaskToMask(&dstmask, &dstcarry, &xmask, &ymask)
if uint32(expdo) != expdo || uint32(expco) != expco {
t.Errorf("expected %d/%d out but got %d/%d out", expdo, expco, do, co)
}
gotdstmask, gotcarrymask := splatArray(gotdst[:do]...), splatArray(gotcarry[:co]...)
dstfail := diffMask(t, "lower bit", &dstmask, &gotdstmask)
carryfail := diffMask(t, "carry bit", &dstcarry, &gotcarrymask)
if dstfail || carryfail {
t.Log(x, y)
}
})
}
})
t.Run("addArrayArrayArrayToArray", func(t *testing.T) {
t.Parallel()
var src rand.PCGSource
src.Seed(2)
seedup := make(map[uint64]struct{})
for i := 0; i < testCount; i++ {
genSeed:
seed := src.Uint64()
if _, dup := seedup[seed]; dup {
goto genSeed
}
seedup[seed] = struct{}{}
t.Run(fmt.Sprint(seed), func(t *testing.T) {
t.Parallel()
tries := 0
var xmask, ymask, zmask [1024]uint64
var xarr, yarr, zarr [4096]uint16
var gotdst, gotcarry [4096]uint16
try:
x := randomArray(&xarr, &xmask, seed)
y := randomArray(&yarr, &ymask, 3*seed)
z := randomArray(&zarr, &zmask, 5*seed)
do, co := addArrayArrayArrayToArray(&gotdst, &gotcarry, x, y, z)
if do|co == ^uint16(0) {
if tries > 100 {
t.Fatal("repeatedly failing")
}
seed++
tries++
goto try
}
var dstmask, dstcarry [1024]uint64
expdo, expco := addMaskMaskMaskToMask(&dstmask, &dstcarry, &xmask, &ymask, &zmask)
if uint32(expdo) != expdo || uint32(expco) != expco {
t.Errorf("expected %d/%d out but got %d/%d out", expdo, expco, do, co)
}
gotdstmask, gotcarrymask := splatArray(gotdst[:do]...), splatArray(gotcarry[:co]...)
dstfail := diffMask(t, "lower bit", &dstmask, &gotdstmask)
carryfail := diffMask(t, "carry bit", &dstcarry, &gotcarrymask)
if dstfail || carryfail {
t.Log(x, y, z)
}
})
}
})
t.Run("addArrayArrayArrayToMask", func(t *testing.T) {
t.Parallel()
var src rand.PCGSource
src.Seed(3)
seedup := make(map[uint64]struct{})
for i := 0; i < testCount; i++ {
genSeed:
seed := src.Uint64()
if _, dup := seedup[seed]; dup {
goto genSeed
}
seedup[seed] = struct{}{}
t.Run(fmt.Sprint(seed), func(t *testing.T) {
t.Parallel()
var xmask, ymask, zmask [1024]uint64
var xarr, yarr, zarr [4096]uint16
x := randomArray(&xarr, &xmask, seed)
y := randomArray(&yarr, &ymask, 3*seed)
z := randomArray(&zarr, &zmask, 5*seed)
var gotdst, gotcarry [1024]uint64
do, co := addArrayArrayArrayToMask(&gotdst, &gotcarry, x, y, z)
var dstmask, dstcarry [1024]uint64
expdo, expco := addMaskMaskMaskToMask(&dstmask, &dstcarry, &xmask, &ymask, &zmask)
if uint32(expdo) != expdo || uint32(expco) != expco {
t.Errorf("expected %d/%d out but got %d/%d out", expdo, expco, do, co)
}
dstfail := diffMask(t, "lower bit", &dstmask, &gotdst)
carryfail := diffMask(t, "carry bit", &dstcarry, &gotcarry)
if dstfail || carryfail {
t.Log(x, y, z)
}
})
}
})
t.Run("addArrayArrayMaskToMask", func(t *testing.T) {
t.Parallel()
var src rand.PCGSource
src.Seed(4)
seedup := make(map[uint64]struct{})
for i := 0; i < testCount; i++ {
genSeed:
seed := src.Uint64()
if _, dup := seedup[seed]; dup {
goto genSeed
}
seedup[seed] = struct{}{}
t.Run(fmt.Sprint(seed), func(t *testing.T) {
t.Parallel()
var xmask, ymask, z [1024]uint64
var xarr, yarr [4096]uint16
x := randomArray(&xarr, &xmask, seed)
y := randomArray(&yarr, &ymask, 3*seed)
zc := randomMask(&z, 5*seed)
var gotdst, gotcarry [1024]uint64
do, co := addArrayArrayMaskToMask(&gotdst, &gotcarry, x, y, &z, zc)
var dstmask, dstcarry [1024]uint64
expdo, expco := addMaskMaskMaskToMask(&dstmask, &dstcarry, &xmask, &ymask, &z)
if uint32(expdo) != expdo || uint32(expco) != expco {
t.Errorf("expected %d/%d out but got %d/%d out", expdo, expco, do, co)
}
dstfail := diffMask(t, "lower bit", &dstmask, &gotdst)
carryfail := diffMask(t, "carry bit", &dstcarry, &gotcarry)
if dstfail || carryfail {
t.Log(x, y, z)
}
})
}
})
t.Run("addArrayMaskMaskToMask", func(t *testing.T) {
t.Parallel()
var src rand.PCGSource
src.Seed(5)
seedup := make(map[uint64]struct{})
for i := 0; i < testCount; i++ {
genSeed:
seed := src.Uint64()
if _, dup := seedup[seed]; dup {
goto genSeed
}
seedup[seed] = struct{}{}
t.Run(fmt.Sprint(seed), func(t *testing.T) {
t.Parallel()
var xmask, y, z [1024]uint64
var xarr [4096]uint16
x := randomArray(&xarr, &xmask, seed)
randomMask(&y, 3*seed)
randomMask(&z, 5*seed)
var gotdst, gotcarry [1024]uint64
do, co := addArrayMaskMaskToMask(&gotdst, &gotcarry, x, &y, &z)
var dstmask, dstcarry [1024]uint64
expdo, expco := addMaskMaskMaskToMask(&dstmask, &dstcarry, &xmask, &y, &z)
if uint32(expdo) != expdo || uint32(expco) != expco {
t.Errorf("expected %d/%d out but got %d/%d out", expdo, expco, do, co)
}
dstfail := diffMask(t, "lower bit", &dstmask, &gotdst)
carryfail := diffMask(t, "carry bit", &dstcarry, &gotcarry)
if dstfail || carryfail {
t.Log(x, y, z)
}
})
}
})
}
// TestAdd randomly tests addition logic.
func TestAdd(t *testing.T) {
t.Parallel()
var src rand.PCGSource
src.Seed(8)
seedup := make(map[uint64]struct{})
for i := 0; i < 20; i++ {
genSeed:
seed := src.Uint64()
if _, dup := seedup[seed]; dup {
goto genSeed
}
seedup[seed] = struct{}{}
t.Run(fmt.Sprint(seed), func(t *testing.T) {
t.Parallel()
var pcg rand.PCGSource
pcg.Seed(seed)
rnd := rand.New(&pcg)
numzipf := rand.NewZipf(rnd, 1.5, 2, 1<<20)
countzipf := rand.NewZipf(rnd, 1.3, 7, 1<<44)
var x, y []*Bitmap
xvals, yvals := map[uint64]uint64{}, map[uint64]uint64{}
for n := numzipf.Uint64(); n > 0; n-- {
idx := pcg.Uint64() % (1 << 20)
xvals[idx] = countzipf.Uint64()
yvals[idx] = countzipf.Uint64()
}
for xn := numzipf.Uint64(); uint64(len(xvals)) < xn; {
xvals[pcg.Uint64()%(1<<20)] = countzipf.Uint64()
}
for yn := numzipf.Uint64(); uint64(len(yvals)) < yn; {
yvals[pcg.Uint64()%(1<<20)] = countzipf.Uint64()
}
expectSums := map[uint64]uint64{}
for k, v := range xvals {
if v == 0 {
delete(xvals, k)
continue
}
expectSums[k] = v
for v != 0 {
i := bits.TrailingZeros64(v)
v &^= 1 << i
for len(x) <= i {
x = append(x, NewBitmap())
}
x[i].DirectAdd(k)
}
}
for k, v := range yvals {
if v == 0 {
delete(xvals, k)
continue
}
expectSums[k] += v
for v != 0 {
i := bits.TrailingZeros64(v)
v &^= 1 << i
for len(y) <= i {
y = append(y, NewBitmap())
}
y[i].DirectAdd(k)
}
}
sumsBSI := Add(x, y)
gotSums := make(map[uint64]uint64, len(expectSums))
for i, b := range sumsBSI {
mask := uint64(1) << i
for _, k := range b.Slice() {
gotSums[k] |= mask
}
}
for k, v := range gotSums {
if _, ok := expectSums[k]; !ok {
t.Errorf("unexpected sum of %d for %d", v, k)
}
}
for k, v := range expectSums {
got, ok := gotSums[k]
if !ok {
t.Errorf("missing sum of %d for %d", v, k)
continue
}
if got != v {
t.Errorf("sum for %d differs: expected %d but got %d", k, v, got)
}
}
})
}
}

12
roaring/container_stash.go
View file

@ -472,6 +472,18 @@ func (c *Container) bitmap() []uint64 {
return (*[1024]uint64)(unsafe.Pointer(c.pointer))[:]
}
func (c *Container) bitmask() *[1024]uint64 {
if c == nil {
panic("attempt to read nil container's bitmap")
}
if roaringParanoia {
if c.typeID != ContainerBitmap {
panic("attempt to read non-bitmap's bitmap")
}
}
return (*[1024]uint64)(unsafe.Pointer(c.pointer))
}
// AsBitmap yields a 65k-bit bitmap, storing it in the target if a target
// is provided. The target should be zeroed, or this becomes an implicit
// union.

26
roaring/roaring.go
View file

@ -4473,17 +4473,18 @@ func intersectBitmapBitmap(a, b *Container) *Container {
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
var (
ab = a.bitmap()[:bitmapN]
bb = b.bitmap()[:bitmapN]
ob = make([]uint64, bitmapN)
ab = a.bitmask()
bb = b.bitmask()
ob = [1024]uint64{}
n int32
)
for i := 0; i < bitmapN; i++ {
_, _ = &ab[0], &bb[0]
for i := range ob {
ob[i] = ab[i] & bb[i]
n += int32(popcount(ob[i]))
}
output := NewContainerBitmapN(ob, n)
output := NewContainerBitmapN(ob[:], n)
return output
}
@ -5668,19 +5669,20 @@ func xorBitmapBitmap(a, b *Container) *Container {
// see https://go101.org/article/bounds-check-elimination.html
var (
ab = a.bitmap()[:bitmapN]
bb = b.bitmap()[:bitmapN]
ob = make([]uint64, bitmapN)[:bitmapN]
ab = a.bitmask()
bb = b.bitmask()
ob = [1024]uint64{}
n int32
)
for i := 0; i < bitmapN; i++ {
_, _ = &ab[0], &bb[0]
for i := range ob {
ob[i] = ab[i] ^ bb[i]
n += int32(popcount(ob[i]))
}
output := NewContainerBitmapN(ob, n)
output := NewContainerBitmapN(ob[:], n)
if n < ArrayMaxSize {
output = output.bitmapToArray()
}
@ -7115,6 +7117,10 @@ func Difference(a, b *Container) *Container {
return difference(a, b)
}
func IntersectionCount(x, y *Container) int32 {
return intersectionCount(x, y)
}
// Add yields a container identical to c, but with the given bit set; added
// is true if the bit wasn't previously set. It is unspecified whether
// the original container is modified.