From a2151358ba1f612bdff01ec04e9614fb18d29326 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 22 Oct 2020 13:08:41 -0500 Subject: [PATCH] Partial implementation: Distinct() supporting set fields Add an exported IntersectionAny() from roaring to let us quickly check whether two containers have overlap, so we can avoid performing intersections we don't need to when evaluating containers within the same row as a previous match. (IntersectionCount on the whole bitmap would imply doing up to 16 intersections even if we find a bit right away.) We also allow ForeignIndex to be set on set, mutex, and time fields, since all of those could now be reasonable operands for Distinct ops. Not yet present: Handling time quantums, but that seems really desireable. --- executor.go | 78 +++++++++++++++- executor_test.go | 21 +++++ field.go | 4 +- http/handler.go | 6 -- roaring/roaring.go | 150 ++++++++++++++++++++++++++++++ roaring/roaring_container_test.go | 5 + 6 files changed, 251 insertions(+), 13 deletions(-) diff --git a/executor.go b/executor.go index 7f7a1a476..b28d288f1 100644 --- a/executor.go +++ b/executor.go @@ -1400,6 +1400,10 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str var filter *Row var filterBitmap *roaring.Bitmap + // If a filter *is* specified, an empty filter means nothing, and any + // filter at all means there's filtering to do. If a filter is *not* + // specified, then we don't need to do any filtering. So a nil + // filterBitmap (which we get if there's no children) means no filter. if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) if err != nil { @@ -1408,17 +1412,81 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str filter = row if filter != nil && len(filter.segments) > 0 { filterBitmap = filter.segments[0].data - } else { - filterBitmap = roaring.NewFileBitmap() + } + // if we had a filter to consider, but it came back empty, we + // can go ahead and save time by returning the empty results, + // because the filter excluded everything. + if filterBitmap == nil || !filterBitmap.Any() { + return SignedRow{}, nil } } bsig := field.bsiGroup(fieldName) if bsig == nil { - return result, nil + return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } - view := viewBSIGroupPrefix + fieldName + return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) +} +func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result SignedRow, err error) { + index := idx.Name() + tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) + defer finisher(&err) + + fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) + if err != nil { + return SignedRow{}, errors.Wrap(err, "getting fragment data") + } + // We can't grab the containers "for each row" from the set-type field, + // because we don't know how many rows there are, and some of them + // might be empty, so really, we're going to iterate through the + // containers, and then intersect them with the filter if present. + var filter []*roaring.Container + if filterBitmap != nil { + filter = make([]*roaring.Container, 1<> shardVsContainerExponent + if row == prevRow && seenThisRow { + continue + } + prevRow = row + if filterBitmap != nil { + if roaring.IntersectionAny(c, filter[k%(1< MaxContainerVal+1 { + return true + } + if a.isArray() { + if b.isArray() { + return intersectionAnyArrayArray(a, b) + } else if b.isRun() { + return intersectionAnyArrayRun(a, b) + } else { + return intersectionAnyArrayBitmap(a, b) + } + } else if a.isRun() { + if b.isArray() { + return intersectionAnyArrayRun(b, a) + } else if b.isRun() { + return intersectionAnyRunRun(a, b) + } else { + return intersectionAnyRunBitmap(a, b) + } + } else { + if b.isArray() { + return intersectionAnyArrayBitmap(b, a) + } else if b.isRun() { + return intersectionAnyRunBitmap(b, a) + } else { + return intersectionAnyBitmapBitmap(a, b) + } + } +} + +func intersectionAnyArrayArray(a, b *Container) bool { + ca, cb := a.array(), b.array() + nb := len(cb) + j := 0 + for _, va := range ca { + for cb[j] < va { + j++ + if j >= nb { + return false + } + } + if cb[j] == va { + return true + } + } + return false +} + +func intersectionAnyArrayRun(a, b *Container) bool { + array, runs := a.array(), b.runs() + na, nb := len(array), len(runs) + for i, j := 0, 0; i < na && j < nb; { + va, vb := array[i], runs[j] + if va < vb.Start { + i++ + } else if va >= vb.Start && va <= vb.Last { + return true + } else if va > vb.Last { + j++ + } + } + return false +} + +func intersectionAnyArrayBitmap(a, b *Container) bool { + bitmap := b.bitmap()[:1024] + for _, val := range a.array() { + i := int(val >> 6) + off := val % 64 + if (bitmap[i]>>off)&1 != 0 { + return true + } + } + return false +} + +func intersectionAnyRunRun(a, b *Container) bool { + ra, rb := a.runs(), b.runs() + na, nb := len(ra), len(rb) + for i, j := 0, 0; i < na && j < nb; { + va, vb := ra[i], rb[j] + if va.Last < vb.Start { + // |--va--| |--vb--| + i++ + } else if va.Start > vb.Last { + // |--vb--| |--va--| + j++ + } else { + // va.Last >= vb.Start, and va.Start <= vb.Last, + // means there must be overlap + return true + } + } + return false +} + +func intersectionAnyRunBitmap(a, b *Container) bool { + bb := b.bitmap()[:1024] + runs := a.runs() + for _, r := range runs { + loWord, loBit := r.Start/64, r.Start%64 + hiWord, hiBit := r.Last/64, r.Last%64 + if loBit != 0 { + w := bb[loWord] + mask := (uint64(1) << loBit) - 1 + if w&^mask != 0 { + return true + } + } + for i := loWord; i < hiWord; i++ { + if bb[i] != 0 { + return true + } + } + if hiBit != 0 { + w := bb[hiWord] + mask := (uint64(1) << hiBit) - 1 + if w&mask != 0 { + return true + } + } + } + return false +} + +func intersectionAnyBitmapBitmap(a, b *Container) bool { + ba, bb := a.bitmap()[:1024], b.bitmap()[:1024] + for i, v := range ba { + if bb[i]&v != 0 { + return true + } + } + return false +} + func intersectionCount(a, b *Container) int32 { if a.N() == MaxContainerVal+1 { return b.N() diff --git a/roaring/roaring_container_test.go b/roaring/roaring_container_test.go index bf573e529..3b747b5c9 100644 --- a/roaring/roaring_container_test.go +++ b/roaring/roaring_container_test.go @@ -87,6 +87,11 @@ func TestIntersectVariants(t *testing.T) { t.Errorf("intersecting %s[%d] and %s[%d]: container has N %d, count was %d", n1, i1, n2, i2, full.N(), count) } + any := intersectionAny(c1, c2) + if any != (count != 0) { + t.Errorf("intersecting %s[%d] and %s[%d]: any %t, count was %d", + n1, i1, n2, i2, any, count) + } } } }