diff --git a/executor.go b/executor.go index abf1aecc8..6e95346e1 100644 --- a/executor.go +++ b/executor.go @@ -17,7 +17,13 @@ import ( ) // DefaultFrame is the frame used if one is not specified. -const DefaultFrame = "general" +const ( + DefaultFrame = "general" + + // MinThreshold is the lowest count to use in a Top-N operation when + // looking for additional bitmap/count pairs. + MinThreshold = 1 +) // Executor recursively executes calls in a PQL query across all slices. type Executor struct { @@ -258,8 +264,9 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, n, _ := c.Args["n"].(uint64) field, _ := c.Args["field"].(string) bitmapIDs, _ := c.Args["ids"].([]uint64) + minThreshold, _ := c.Args["threshold"].(uint64) filters, _ := c.Args["filters"].([]interface{}) - + tanimotoThreshold, _ := c.Args["tanimotoThreshold"].(uint64) // Retrieve bitmap used to intersect. var src *Bitmap if len(c.Children) == 1 { @@ -282,12 +289,21 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, return nil, nil } + if minThreshold <= 0 { + minThreshold = MinThreshold + } + + if tanimotoThreshold > 100 { + return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") + } return f.Top(TopOptions{ - N: int(n), - Src: src, - BitmapIDs: bitmapIDs, - FilterField: field, - FilterValues: filters, + N: int(n), + Src: src, + BitmapIDs: bitmapIDs, + FilterField: field, + FilterValues: filters, + MinThreshold: minThreshold, + TanimotoThreshold: tanimotoThreshold, }) } diff --git a/fragment.go b/fragment.go index 9f7f155b9..8b58729aa 100644 --- a/fragment.go +++ b/fragment.go @@ -22,6 +22,8 @@ import ( "time" "unsafe" + "math" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" @@ -40,10 +42,6 @@ const ( // CacheExt is the file extension for persisted cache ids. CacheExt = ".cache" - // MinThreshold is the lowest count to use in a Top-N operation when - // looking for additional bitmap/count pairs. - MinThreshold = 10 - // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 ) @@ -491,7 +489,6 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } // Iterate over rankings and add to results until we have enough. - //results := make(PairHeap, 0, opt.N) results := &PairHeap{} for _, pair := range pairs { bitmapID, cnt := pair.ID, pair.Count @@ -501,6 +498,20 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { continue } + if opt.TanimotoThreshold > 0 && opt.Src != nil { + minTanimoto := float64(opt.Src.Count() * opt.TanimotoThreshold / 100) + maxTanimoto := float64(opt.Src.Count() * 100 / opt.TanimotoThreshold) + if float64(n) <= minTanimoto || float64(n) >= float64(maxTanimoto) { + continue + } + + } else { + // Ignore count that less than MinThreshold. + if cnt < opt.MinThreshold { + continue + } + } + // Apply filter, if set. if filters != nil { attr, err := f.BitmapAttrStore.Attrs(bitmapID) @@ -521,10 +532,23 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { count := cnt if opt.Src != nil { count = opt.Src.IntersectionCount(f.Bitmap(bitmapID)) + } if count == 0 { continue } + + if opt.TanimotoThreshold > 0 { + tanimoto := math.Ceil(float64(count * 100 / (cnt + opt.Src.Count() - count))) + if tanimoto <= float64(opt.TanimotoThreshold) { + continue + } + + } else { + if count < opt.MinThreshold { + continue + } + } heap.Push(results, Pair{Key: bitmapID, Count: count}) // If we reach the requested number of pairs and we are not computing @@ -545,7 +569,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // If the bitmap doesn't have enough bits set before the intersection // then we can assume that any remaining bitmaps also have a count too low. - if cnt < threshold { + if threshold < opt.MinThreshold || cnt < threshold { break } @@ -558,6 +582,8 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { heap.Push(results, Pair{Key: bitmapID, Count: count}) } + + //Pop first opt.N elements out of heap r := make(Pairs, results.Len(), results.Len()) x := results.Len() i := 1 @@ -611,11 +637,13 @@ type TopOptions struct { Src *Bitmap // Specific bitmaps to filter against. - BitmapIDs []uint64 + BitmapIDs []uint64 + MinThreshold uint64 // Filter field name & values. - FilterField string - FilterValues []interface{} + FilterField string + FilterValues []interface{} + TanimotoThreshold uint64 } // Checksum returns a checksum for the entire fragment. diff --git a/fragment_test.go b/fragment_test.go index da2edac6c..9780dfc17 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -659,3 +659,49 @@ func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint6 } return } + +func TestFragment_Tanimoto(t *testing.T) { + f := MustOpenFragment("d", "f", 0) + defer f.Close() + + src := pilosa.NewBitmap(1, 2, 3) + + // Set bits on the bitmaps 100, 101, & 102. + f.MustSetBits(100, 1, 3, 2, 200) + f.MustSetBits(101, 1, 3) + f.MustSetBits(102, 1, 2, 10, 12) + + if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { + t.Fatal(err) + } else if len(pairs) != 2 { + t.Fatalf("unexpected count: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } else if pairs[1] != (pilosa.Pair{Key: 101, Count: 2}) { + t.Fatalf("unexpected pair(1): %v", pairs[1]) + } +} + +func TestFragment_Zero_Tanimoto(t *testing.T) { + f := MustOpenFragment("d", "f", 0) + defer f.Close() + + src := pilosa.NewBitmap(1, 2, 3) + + // Set bits on the bitmaps 100, 101, & 102. + f.MustSetBits(100, 1, 3, 2, 200) + f.MustSetBits(101, 1, 3) + f.MustSetBits(102, 1, 2, 10, 12) + + if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { + t.Fatal(err) + } else if len(pairs) != 3 { + t.Fatalf("unexpected count: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } else if pairs[1] != (pilosa.Pair{Key: 101, Count: 2}) { + t.Fatalf("unexpected pair(1): %v", pairs[1]) + } else if pairs[2] != (pilosa.Pair{Key: 102, Count: 2}) { + t.Fatalf("unexpected pair(1): %v", pairs[2]) + } +}