WIP TopN accuracy

executor.go

@@ -176,7 +176,6 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.TopN, slic
 	if len(pairs) == 0 || len(c.BitmapIDs) > 0 || opt.Remote {
 		return pairs, nil
 	}
-
 	// Only the original caller should refetch the full counts.
 	other := *c
 	other.N = 0
@@ -780,6 +779,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
 				resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
 			} else if !opt.Remote {
 				results, err := e.exec(ctx, n, db, &pql.Query{Calls: pql.Calls{c}}, nodeSlices, opt)
+
 				if len(results) > 0 {
 					resp.result = results[0]
 				}

fragment.go

@@ -553,9 +553,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
 		//threshold := results[len(results)-1].Count
 
 		threshold := results.Pairs[0].Count
-		if threshold < MinThreshold {
-			break
-		}
+		//if threshold < MinThreshold {
+		//break
+		//}
 
 		// If the bitmap doesn't have enough bits set before the intersection
 		// then we can assume that any remaing bitmaps also have a count too low.
@@ -593,21 +593,24 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
 	}
 
 	// Otherwise retrieve specific bitmaps.
-	pairs := make([]BitmapPair, len(bitmapIDs))
-	for i, bitmapID := range bitmapIDs {
+	pairs := make([]BitmapPair, 0, len(bitmapIDs))
+	for _, bitmapID := range bitmapIDs {
 		// Look up cache first, if available.
 		if n := f.cache.Get(bitmapID); n > 0 {
-			pairs[i] = BitmapPair{
+			pairs = append(pairs, BitmapPair{
 				ID:    bitmapID,
 				Count: n,
-			}
+			})
 			continue
 		}
 
-		// Otherwise load from storage.
-		pairs[i] = BitmapPair{
-			ID:    bitmapID,
-			Count: f.Bitmap(bitmapID).Count(),
+		bm := f.Bitmap(bitmapID)
+		if bm.Count() > 0 {
+			// Otherwise load from storage.
+			pairs = append(pairs, BitmapPair{
+				ID:    bitmapID,
+				Count: bm.Count(),
+			})
 		}
 	}
 	sort.Sort(BitmapPairs(pairs))