Merge pull request #369 from linhvo/tanimoto

add tanimoto option to pilosa

executor.go

@@ -17,7 +17,13 @@ import (
 )
 
 // DefaultFrame is the frame used if one is not specified.
-const DefaultFrame = "general"
+const (
+	DefaultFrame = "general"
+
+	// MinThreshold is the lowest count to use in a Top-N operation when
+	// looking for additional bitmap/count pairs.
+	MinThreshold = 1
+)
 
 // Executor recursively executes calls in a PQL query across all slices.
 type Executor struct {
@@ -258,8 +264,9 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
 	n, _ := c.Args["n"].(uint64)
 	field, _ := c.Args["field"].(string)
 	bitmapIDs, _ := c.Args["ids"].([]uint64)
+	minThreshold, _ := c.Args["threshold"].(uint64)
 	filters, _ := c.Args["filters"].([]interface{})
-
+	tanimotoThreshold, _ := c.Args["tanimotoThreshold"].(uint64)
 	// Retrieve bitmap used to intersect.
 	var src *Bitmap
 	if len(c.Children) == 1 {
@@ -282,12 +289,21 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
 		return nil, nil
 	}
 
+	if minThreshold <= 0 {
+		minThreshold = MinThreshold
+	}
+
+	if tanimotoThreshold > 100 {
+		return nil, errors.New("Tanimoto Threshold is from 1 to 100 only")
+	}
 	return f.Top(TopOptions{
-		N:            int(n),
-		Src:          src,
-		BitmapIDs:    bitmapIDs,
-		FilterField:  field,
-		FilterValues: filters,
+		N:                 int(n),
+		Src:               src,
+		BitmapIDs:         bitmapIDs,
+		FilterField:       field,
+		FilterValues:      filters,
+		MinThreshold:      minThreshold,
+		TanimotoThreshold: tanimotoThreshold,
 	})
 }
 

fragment.go

@@ -22,6 +22,8 @@ import (
 	"time"
 	"unsafe"
 
+	"math"
+
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
 	"github.com/pilosa/pilosa/roaring"
@@ -40,10 +42,6 @@ const (
 	// CacheExt is the file extension for persisted cache ids.
 	CacheExt = ".cache"
 
-	// MinThreshold is the lowest count to use in a Top-N operation when
-	// looking for additional bitmap/count pairs.
-	MinThreshold = 10
-
 	// HashBlockSize is the number of bitmaps in a merkle hash block.
 	HashBlockSize = 100
 )
@@ -490,8 +488,18 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 		}
 	}
 
+	// Use `tanimotoThreshold > 0` to indicate whether or not we are considering Tanimoto.
+	var tanimotoThreshold uint64
+	var minTanimoto, maxTanimoto float64
+	var srcCount uint64
+	if opt.TanimotoThreshold > 0 && opt.Src != nil {
+		tanimotoThreshold = opt.TanimotoThreshold
+		srcCount = opt.Src.Count()
+		minTanimoto = float64(srcCount*tanimotoThreshold) / 100
+		maxTanimoto = float64(srcCount*100) / float64(tanimotoThreshold)
+	}
+
 	// Iterate over rankings and add to results until we have enough.
-	//results := make(PairHeap, 0, opt.N)
 	results := &PairHeap{}
 	for _, pair := range pairs {
 		bitmapID, cnt := pair.ID, pair.Count
@@ -501,6 +509,19 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 			continue
 		}
 
+		// Check against either Tanimoto threshold or minimum threshold.
+		if tanimotoThreshold > 0 {
+			// Ignore counts outside of the Tanimoto min/max values.
+			if float64(cnt) <= minTanimoto || float64(cnt) >= maxTanimoto {
+				continue
+			}
+		} else {
+			// Ignore counts less than MinThreshold.
+			if cnt < opt.MinThreshold {
+				continue
+			}
+		}
+
 		// Apply filter, if set.
 		if filters != nil {
 			attr, err := f.BitmapAttrStore.Attrs(bitmapID)
@@ -525,6 +546,19 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 			if count == 0 {
 				continue
 			}
+
+			// Check against either Tanimoto threshold or minimum threshold.
+			if tanimotoThreshold > 0 {
+				tanimoto := math.Ceil(float64(count*100) / float64(cnt+srcCount-count))
+				if tanimoto <= float64(tanimotoThreshold) {
+					continue
+				}
+			} else {
+				if count < opt.MinThreshold {
+					continue
+				}
+			}
+
 			heap.Push(results, Pair{Key: bitmapID, Count: count})
 
 			// If we reach the requested number of pairs and we are not computing
@@ -545,7 +579,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 
 		// If the bitmap doesn't have enough bits set before the intersection
 		// then we can assume that any remaining bitmaps also have a count too low.
-		if cnt < threshold {
+		if threshold < opt.MinThreshold || cnt < threshold {
 			break
 		}
 
@@ -558,6 +592,8 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 
 		heap.Push(results, Pair{Key: bitmapID, Count: count})
 	}
+
+	//Pop first opt.N elements out of heap
 	r := make(Pairs, results.Len(), results.Len())
 	x := results.Len()
 	i := 1
@@ -611,11 +647,13 @@ type TopOptions struct {
 	Src *Bitmap
 
 	// Specific bitmaps to filter against.
-	BitmapIDs []uint64
+	BitmapIDs    []uint64
+	MinThreshold uint64
 
 	// Filter field name & values.
-	FilterField  string
-	FilterValues []interface{}
+	FilterField       string
+	FilterValues      []interface{}
+	TanimotoThreshold uint64
 }
 
 // Checksum returns a checksum for the entire fragment.

fragment_test.go

@@ -659,3 +659,49 @@ func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint6
 	}
 	return
 }
+
+func TestFragment_Tanimoto(t *testing.T) {
+	f := MustOpenFragment("d", "f", 0)
+	defer f.Close()
+
+	src := pilosa.NewBitmap(1, 2, 3)
+
+	// Set bits on the bitmaps 100, 101, & 102.
+	f.MustSetBits(100, 1, 3, 2, 200)
+	f.MustSetBits(101, 1, 3)
+	f.MustSetBits(102, 1, 2, 10, 12)
+
+	if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil {
+		t.Fatal(err)
+	} else if len(pairs) != 2 {
+		t.Fatalf("unexpected count: %d", len(pairs))
+	} else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) {
+		t.Fatalf("unexpected pair(0): %v", pairs[0])
+	} else if pairs[1] != (pilosa.Pair{Key: 101, Count: 2}) {
+		t.Fatalf("unexpected pair(1): %v", pairs[1])
+	}
+}
+
+func TestFragment_Zero_Tanimoto(t *testing.T) {
+	f := MustOpenFragment("d", "f", 0)
+	defer f.Close()
+
+	src := pilosa.NewBitmap(1, 2, 3)
+
+	// Set bits on the bitmaps 100, 101, & 102.
+	f.MustSetBits(100, 1, 3, 2, 200)
+	f.MustSetBits(101, 1, 3)
+	f.MustSetBits(102, 1, 2, 10, 12)
+
+	if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil {
+		t.Fatal(err)
+	} else if len(pairs) != 3 {
+		t.Fatalf("unexpected count: %d", len(pairs))
+	} else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) {
+		t.Fatalf("unexpected pair(0): %v", pairs[0])
+	} else if pairs[1] != (pilosa.Pair{Key: 101, Count: 2}) {
+		t.Fatalf("unexpected pair(1): %v", pairs[1])
+	} else if pairs[2] != (pilosa.Pair{Key: 102, Count: 2}) {
+		t.Fatalf("unexpected pair(1): %v", pairs[2])
+	}
+}