diff --git a/executor.go b/executor.go index 7f7a1a476..b28d288f1 100644 --- a/executor.go +++ b/executor.go @@ -1400,6 +1400,10 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str var filter *Row var filterBitmap *roaring.Bitmap + // If a filter *is* specified, an empty filter means nothing, and any + // filter at all means there's filtering to do. If a filter is *not* + // specified, then we don't need to do any filtering. So a nil + // filterBitmap (which we get if there's no children) means no filter. if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) if err != nil { @@ -1408,17 +1412,81 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str filter = row if filter != nil && len(filter.segments) > 0 { filterBitmap = filter.segments[0].data - } else { - filterBitmap = roaring.NewFileBitmap() + } + // if we had a filter to consider, but it came back empty, we + // can go ahead and save time by returning the empty results, + // because the filter excluded everything. + if filterBitmap == nil || !filterBitmap.Any() { + return SignedRow{}, nil } } bsig := field.bsiGroup(fieldName) if bsig == nil { - return result, nil + return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } - view := viewBSIGroupPrefix + fieldName + return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) +} +func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result SignedRow, err error) { + index := idx.Name() + tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) + defer finisher(&err) + + fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) + if err != nil { + return SignedRow{}, errors.Wrap(err, "getting fragment data") + } + // We can't grab the containers "for each row" from the set-type field, + // because we don't know how many rows there are, and some of them + // might be empty, so really, we're going to iterate through the + // containers, and then intersect them with the filter if present. + var filter []*roaring.Container + if filterBitmap != nil { + filter = make([]*roaring.Container, 1<> shardVsContainerExponent + if row == prevRow && seenThisRow { + continue + } + prevRow = row + if filterBitmap != nil { + if roaring.IntersectionAny(c, filter[k%(1< MaxContainerVal+1 { + return true + } + if a.isArray() { + if b.isArray() { + return intersectionAnyArrayArray(a, b) + } else if b.isRun() { + return intersectionAnyArrayRun(a, b) + } else { + return intersectionAnyArrayBitmap(a, b) + } + } else if a.isRun() { + if b.isArray() { + return intersectionAnyArrayRun(b, a) + } else if b.isRun() { + return intersectionAnyRunRun(a, b) + } else { + return intersectionAnyRunBitmap(a, b) + } + } else { + if b.isArray() { + return intersectionAnyArrayBitmap(b, a) + } else if b.isRun() { + return intersectionAnyRunBitmap(b, a) + } else { + return intersectionAnyBitmapBitmap(a, b) + } + } +} + +func intersectionAnyArrayArray(a, b *Container) bool { + ca, cb := a.array(), b.array() + nb := len(cb) + j := 0 + for _, va := range ca { + for cb[j] < va { + j++ + if j >= nb { + return false + } + } + if cb[j] == va { + return true + } + } + return false +} + +func intersectionAnyArrayRun(a, b *Container) bool { + array, runs := a.array(), b.runs() + na, nb := len(array), len(runs) + for i, j := 0, 0; i < na && j < nb; { + va, vb := array[i], runs[j] + if va < vb.Start { + i++ + } else if va >= vb.Start && va <= vb.Last { + return true + } else if va > vb.Last { + j++ + } + } + return false +} + +func intersectionAnyArrayBitmap(a, b *Container) bool { + bitmap := b.bitmap()[:1024] + for _, val := range a.array() { + i := int(val >> 6) + off := val % 64 + if (bitmap[i]>>off)&1 != 0 { + return true + } + } + return false +} + +func intersectionAnyRunRun(a, b *Container) bool { + ra, rb := a.runs(), b.runs() + na, nb := len(ra), len(rb) + for i, j := 0, 0; i < na && j < nb; { + va, vb := ra[i], rb[j] + if va.Last < vb.Start { + // |--va--| |--vb--| + i++ + } else if va.Start > vb.Last { + // |--vb--| |--va--| + j++ + } else { + // va.Last >= vb.Start, and va.Start <= vb.Last, + // means there must be overlap + return true + } + } + return false +} + +func intersectionAnyRunBitmap(a, b *Container) bool { + bb := b.bitmap()[:1024] + runs := a.runs() + for _, r := range runs { + loWord, loBit := r.Start/64, r.Start%64 + hiWord, hiBit := r.Last/64, r.Last%64 + if loBit != 0 { + w := bb[loWord] + mask := (uint64(1) << loBit) - 1 + if w&^mask != 0 { + return true + } + } + for i := loWord; i < hiWord; i++ { + if bb[i] != 0 { + return true + } + } + if hiBit != 0 { + w := bb[hiWord] + mask := (uint64(1) << hiBit) - 1 + if w&mask != 0 { + return true + } + } + } + return false +} + +func intersectionAnyBitmapBitmap(a, b *Container) bool { + ba, bb := a.bitmap()[:1024], b.bitmap()[:1024] + for i, v := range ba { + if bb[i]&v != 0 { + return true + } + } + return false +} + func intersectionCount(a, b *Container) int32 { if a.N() == MaxContainerVal+1 { return b.N() diff --git a/roaring/roaring_container_test.go b/roaring/roaring_container_test.go index bf573e529..3b747b5c9 100644 --- a/roaring/roaring_container_test.go +++ b/roaring/roaring_container_test.go @@ -87,6 +87,11 @@ func TestIntersectVariants(t *testing.T) { t.Errorf("intersecting %s[%d] and %s[%d]: container has N %d, count was %d", n1, i1, n2, i2, full.N(), count) } + any := intersectionAny(c1, c2) + if any != (count != 0) { + t.Errorf("intersecting %s[%d] and %s[%d]: any %t, count was %d", + n1, i1, n2, i2, any, count) + } } } }