TopN()

This commit adds TopN() functionality to the fragment, executor
and handler.

cache.go

@@ -1,7 +1,6 @@
 package pilosa
 
 import (
-	"encoding/json"
 	"io"
 	"sort"
 	"time"
@@ -13,40 +12,37 @@ import (
 
 // Cache represents a cache for bitmaps.
 type Cache interface {
-	io.WriterTo
-	io.ReaderFrom
-
-	Add(bitmapID, category uint64, bm *Bitmap)
+	Add(bitmapID uint64, bm *Bitmap)
 	Get(bitmapID uint64) *Bitmap
 	Len() int
 
 	// Updates the cache, if necessary.
 	Invalidate()
 
-	// Returns a list of all key/count pairs.
-	Pairs() []Pair
+	// Returns an ordered list of the top ranked bitmaps.
+	Top() []BitmapPair
 }
 
 // LRUCache represents a least recently used Cache implemenation.
 type LRUCache struct {
-	cache *lru.Cache
-	keys  map[uint64]struct{}
+	cache   *lru.Cache
+	bitmaps map[uint64]*Bitmap
 }
 
 // NewLRUCache returns a new instance of LRUCache.
 func NewLRUCache(maxEntries int) *LRUCache {
 	c := &LRUCache{
-		cache: lru.New(maxEntries),
-		keys:  make(map[uint64]struct{}),
+		cache:   lru.New(maxEntries),
+		bitmaps: make(map[uint64]*Bitmap),
 	}
 	c.cache.OnEvicted = c.onEvicted
 	return c
 }
 
 // Get returns a bitmap with a given id.
-func (c *LRUCache) Add(bitmapID, category uint64, bm *Bitmap) {
+func (c *LRUCache) Add(bitmapID uint64, bm *Bitmap) {
 	c.cache.Add(bitmapID, bm)
-	c.keys[bitmapID] = struct{}{}
+	c.bitmaps[bitmapID] = bm
 }
 
 // Get returns a bitmap with a given id.
@@ -64,53 +60,28 @@ func (c *LRUCache) Len() int { return c.cache.Len() }
 // Invalidate is a no-op.
 func (c *LRUCache) Invalidate() {}
 
-// Pairs returns all key/count pairs in the cache.
-func (c *LRUCache) Pairs() []Pair {
-	a := make([]Pair, 0, len(c.keys))
-	for k := range c.keys {
-		a = append(a, Pair{
-			Key:   k,
-			Count: c.Get(k).Count(),
+// Top returns all bitmaps in the cache.
+func (c *LRUCache) Top() []BitmapPair {
+	a := make([]BitmapPair, 0, len(c.bitmaps))
+	for id, bm := range c.bitmaps {
+		a = append(a, BitmapPair{
+			ID:     id,
+			Bitmap: bm,
 		})
 	}
+	sort.Sort(BitmapPairs(a))
 	return a
 }
 
-// WriteTo writes the cache to w.
-func (c *LRUCache) WriteTo(w io.Writer) (n int64, err error) {
-	// Write keys to slice.
-	a := make([]uint64, 0, len(c.keys))
-	for k := range c.keys {
-		a = append(a, k)
-	}
-
-	// Encode to file as array of keys.
-	if err := json.NewEncoder(w).Encode(a); err != nil {
-		return 0, err
-	}
-	return 0, nil
-}
-
-// ReadFrom read from r into the cache.
-func (c *LRUCache) ReadFrom(r io.Reader) (n int64, err error) {
-	var keys []uint64
-	if err := json.NewDecoder(r).Decode(&keys); err != nil {
-		return 0, err
-	}
-	panic("FIXME: TODO")
-}
-
-func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) {
-	delete(c.keys, key.(uint64))
-}
+func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.bitmaps, key.(uint64)) }
 
 // Ensure LRUCache implements Cache.
 var _ Cache = &LRUCache{}
 
 // RankCache represents a cache with sorted entries.
 type RankCache struct {
-	entries  map[uint64]*Pair
-	rankings []Pair // cached, ordered list
+	entries  map[uint64]*Bitmap
+	rankings []BitmapPair // cached, ordered list
 
 	updateN    int
 	updateTime time.Time
@@ -123,44 +94,33 @@ type RankCache struct {
 // NewRankCache returns a new instance of RankCache.
 func NewRankCache() *RankCache {
 	return &RankCache{
-		entries: make(map[uint64]*Pair),
+		entries: make(map[uint64]*Bitmap),
 	}
 }
 
 // Get returns a bitmap with a given id.
-func (c *RankCache) Add(bitmapID, category uint64, bm *Bitmap) {
+func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) {
 	// Ignore if the bit count on the bitmap is below the threshold.
 	if bm.Count() < c.ThresholdValue {
 		return
 	}
 
 	// Add to cache.
-	c.entries[bitmapID] = &Pair{
-		Key:      bitmapID,
-		Count:    bm.Count(),
-		bitmap:   bm,
-		category: category,
-	}
+	c.entries[bitmapID] = bm
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.ThresholdLength {
 		c.update()
-		for k, entry := range c.entries {
-			if entry.bitmap.Count() <= c.ThresholdValue {
-				delete(c.entries, k)
+		for id, bm := range c.entries {
+			if bm.Count() <= c.ThresholdValue {
+				delete(c.entries, id)
 			}
 		}
 	}
 }
 
 // Get returns a bitmap with a given id.
-func (c *RankCache) Get(bitmapID uint64) *Bitmap {
-	entry, ok := c.entries[bitmapID]
-	if !ok {
-		return nil
-	}
-	return entry.bitmap
-}
+func (c *RankCache) Get(bitmapID uint64) *Bitmap { return c.entries[bitmapID] }
 
 // Len returns the number of items in the cache.
 func (c *RankCache) Len() int { return len(c.entries) }
@@ -176,21 +136,19 @@ func (c *RankCache) Invalidate() {
 // update reorders the entries by rank.
 func (c *RankCache) update() {
 	// Convert cache to a sorted list.
-	list := make([]Pair, 0, len(c.entries))
-	for k, item := range c.entries {
-		list = append(list, Pair{
-			Key:      k,
-			Count:    item.bitmap.Count(),
-			bitmap:   item.bitmap,
-			category: item.category,
+	rankings := make([]BitmapPair, 0, len(c.entries))
+	for id, bm := range c.entries {
+		rankings = append(rankings, BitmapPair{
+			ID:     id,
+			Bitmap: bm,
 		})
 	}
-	sort.Sort(Pairs(list))
+	sort.Sort(BitmapPairs(rankings))
 
 	// Store the count of the item at the threshold index.
-	c.rankings = list
+	c.rankings = rankings
 	if len(c.rankings) > c.ThresholdIndex {
-		c.ThresholdValue = list[c.ThresholdIndex].bitmap.Count()
+		c.ThresholdValue = rankings[c.ThresholdIndex].Bitmap.Count()
 	} else {
 		c.ThresholdValue = 1
 	}
@@ -199,8 +157,8 @@ func (c *RankCache) update() {
 	c.updateTime, c.updateN = time.Now(), 0
 }
 
-// Pairs returns an ordered list of key/count pairs.
-func (c *RankCache) Pairs() []Pair { return c.rankings }
+// Top returns an ordered list of bitmaps.
+func (c *RankCache) Top() []BitmapPair { return c.rankings }
 
 // WriteTo writes the cache to w.
 func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
@@ -215,12 +173,22 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
 // Ensure RankCache implements Cache.
 var _ Cache = &RankCache{}
 
+// BitmapPair represents a bitmap with an associated identifier.
+type BitmapPair struct {
+	ID     uint64
+	Bitmap *Bitmap
+}
+
+// BitmapPairs is a sortable list of BitmapPair objects.
+type BitmapPairs []BitmapPair
+
+func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p BitmapPairs) Len() int           { return len(p) }
+func (p BitmapPairs) Less(i, j int) bool { return p[i].Bitmap.Count() > p[j].Bitmap.Count() }
+
 type Pair struct {
 	Key   uint64 `json:"key"`
 	Count uint64 `json:"count"`
-
-	bitmap   *Bitmap
-	category uint64
 }
 
 func encodePair(p Pair) *internal.Pair {
@@ -243,6 +211,27 @@ func (p Pairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p Pairs) Len() int           { return len(p) }
 func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
+// Add merges other into p and returns a new slice.
+func (p Pairs) Add(other []Pair) []Pair {
+	// Create lookup of key/counts.
+	m := make(map[uint64]uint64, len(p))
+	for _, pair := range p {
+		m[pair.Key] = pair.Count
+	}
+
+	// Add/merge from other.
+	for _, pair := range other {
+		m[pair.Key] += pair.Count
+	}
+
+	// Convert back to slice.
+	a := make([]Pair, 0, len(m))
+	for k, v := range m {
+		a = append(a, Pair{Key: k, Count: v})
+	}
+	return a
+}
+
 func encodePairs(a Pairs) []*internal.Pair {
 	other := make([]*internal.Pair, len(a))
 	for i := range a {

executor.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"sort"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/umbel/pilosa/internal"
@@ -147,7 +148,63 @@ func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uin
 
 // executeTopN executes a TopN() call.
 func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) {
-	panic("FIXME: calculate top n from each slice")
+	var results []Pair
+	for node, nodeSlices := range e.slicesByNode(slices) {
+		// Execute locally if the hostname matches.
+		if node.Host == e.Host {
+			for _, slice := range nodeSlices {
+				pairs, err := e.executeTopNSlice(db, c, slice)
+				if err != nil {
+					return nil, err
+				}
+				results = Pairs(results).Add(pairs)
+			}
+			continue
+		}
+
+		// Otherwise execute remotely.
+		res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
+		if err != nil {
+			return nil, err
+		}
+		results = Pairs(results).Add(res.([]Pair))
+	}
+
+	// Sort final merged results.
+	sort.Sort(Pairs(results))
+
+	// Only keep the top n after sorting.
+	if len(results) > c.N {
+		results = results[0:c.N]
+	}
+
+	return results, nil
+}
+
+// executeTopNSlice executes a TopN call for a single slice.
+func (e *Executor) executeTopNSlice(db string, c *pql.TopN, slice uint64) ([]Pair, error) {
+	// Retrieve bitmap used to intersect.
+	var src *Bitmap
+	if c.Src != nil {
+		bm, err := e.executeBitmapCallSlice(db, c.Src, slice)
+		if err != nil {
+			return nil, err
+		}
+		src = bm
+	}
+
+	// Set default frame.
+	frame := c.Frame
+	if frame == "" {
+		frame = DefaultFrame
+	}
+
+	f := e.Index().Fragment(db, frame, slice)
+	if f == nil {
+		return nil, nil
+	}
+
+	return f.TopN(c.N, src, c.Field, c.Filters)
 }
 
 // executeDifferenceSlice executes a difference() call for a local slice.

executor_test.go

@@ -14,8 +14,8 @@ import (
 func TestExecutor_Execute_Bitmap(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBit(10, 3)
-	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
 
 	if err := idx.Frame("d", "f").SetBitmapAttrs(10, map[string]interface{}{"foo": "bar", "baz": 123}); err != nil {
 		t.Fatal(err)
@@ -39,10 +39,10 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
 func TestExecutor_Execute_Difference(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 1)
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 2)
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 3)
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
 
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
@@ -58,13 +58,13 @@ func TestExecutor_Execute_Difference(t *testing.T) {
 func TestExecutor_Execute_Intersect(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 1)
-	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+1)
-	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
+	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2)
 
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 1)
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2)
-	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(11, SliceWidth+2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 1)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
 
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
@@ -82,12 +82,12 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
 func TestExecutor_Execute_Union(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 0)
-	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+1)
-	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0)
+	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2)
 
-	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2)
-	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(11, SliceWidth+2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
+	idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
 
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
@@ -105,9 +105,9 @@ func TestExecutor_Execute_Union(t *testing.T) {
 func TestExecutor_Execute_Count(t *testing.T) {
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBit(10, 3)
-	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+1)
-	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+2)
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2)
 
 	e := NewExecutor(idx.Index, NewCluster(1))
 	if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil {
@@ -162,6 +162,67 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
 	}
 }
 
+// Ensure a TopN() query can be executed.
+func TestExecutor_Execute_TopN(t *testing.T) {
+	idx := MustOpenIndex()
+	defer idx.Close()
+
+	// Set bits for bitmaps 0, 10, & 20 across two slices.
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2)
+	idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100)
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0)
+
+	// Execute query.
+	e := NewExecutor(idx.Index, NewCluster(1))
+	if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(result, []pilosa.Pair{
+		{Key: 0, Count: 5},
+		{Key: 10, Count: 2},
+	}) {
+		t.Fatalf("unexpected result: %s", spew.Sdump(result))
+	}
+}
+
+// Ensure a TopN() query with a source bitmap can be executed.
+func TestExecutor_Execute_TopN_Src(t *testing.T) {
+	idx := MustOpenIndex()
+	defer idx.Close()
+
+	// Set bits for bitmaps 0, 10, & 20 across two slices.
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
+	idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2)
+
+	// Create an intersecting bitmap.
+	idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth)
+	idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1)
+	idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2)
+
+	// Execute query.
+	e := NewExecutor(idx.Index, NewCluster(1))
+	if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(result, []pilosa.Pair{
+		{Key: 20, Count: 3},
+		{Key: 10, Count: 2},
+		{Key: 0, Count: 1},
+	}) {
+		t.Fatalf("unexpected result: %s", spew.Sdump(result))
+	}
+}
+
 // Ensure a remote query can return a bitmap.
 func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
 	c := NewCluster(2)
@@ -194,7 +255,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
 	// The local node owns slice 1.
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
 
 	e := NewExecutor(idx.Index, c)
 	if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil {
@@ -225,8 +286,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
 	// Create local executor data. The local node owns slice 1.
 	idx := MustOpenIndex()
 	defer idx.Close()
-	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+1)
-	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+2)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2)
 
 	e := NewExecutor(idx.Index, c)
 	if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil {
@@ -276,6 +337,51 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
 	}
 }
 
+// Ensure a remote query can return a top-n query.
+func TestExecutor_Execute_Remote_TopN(t *testing.T) {
+	c := NewCluster(2)
+
+	// Create secondary server and update second cluster node.
+	s := NewServer()
+	defer s.Close()
+	c.Nodes[1].Host = s.Host()
+
+	// Mock secondary server's executor to verify arguments and return a bitmap.
+	s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
+		if db != `d` {
+			t.Fatalf("unexpected db: %s", db)
+		} else if query.String() != `TopN(frame=f, n=3)` {
+			t.Fatalf("unexpected query: %s", query.String())
+		} else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) {
+			t.Fatalf("unexpected slices: %+v", slices)
+		}
+
+		// Return pair counts.
+		return []pilosa.Pair{
+			{Key: 0, Count: 5},
+			{Key: 10, Count: 2},
+			{Key: 30, Count: 2},
+		}, nil
+	}
+
+	// Create local executor data on slice 1 & 3.
+	idx := MustOpenIndex()
+	defer idx.Close()
+	idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(30, (1*SliceWidth)+1)
+	idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2)
+
+	e := NewExecutor(idx.Index, c)
+	if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(res, []pilosa.Pair{
+		{Key: 0, Count: 5},
+		{Key: 30, Count: 4},
+		{Key: 10, Count: 2},
+	}) {
+		t.Fatalf("unexpected results: %s", spew.Sdump(res))
+	}
+}
+
 // Executor represents a test wrapper for pilosa.Executor.
 type Executor struct {
 	*pilosa.Executor

fragment.go

@@ -20,6 +20,8 @@ const SliceWidth = 65536
 // SnapshotExt is the file extension used for an in-process snapshot.
 const SnapshotExt = ".snapshotting"
 
+const MinThreshold = 10
+
 // Fragment represents the intersection of a frame and slice in a database.
 type Fragment struct {
 	mu sync.Mutex
@@ -37,6 +39,12 @@ type Fragment struct {
 
 	// Bitmap cache.
 	cache Cache
+
+	// Bitmap attribute storage.
+	// Typically this is the parent frame unless overridden for testing.
+	BitmapAttrStore interface {
+		BitmapAttrs(id uint64) (map[string]interface{}, error)
+	}
 }
 
 // NewFragment returns a new instance of Fragment.
@@ -207,41 +215,11 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
 	})
 
 	// Add to the cache.
-	f.cache.Add(bitmapID, 0 /*filter*/, bm)
+	f.cache.Add(bitmapID, bm)
 
 	return bm
 }
 
-func (f *Fragment) TopNAll(n int, categories []uint64) []Pair {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.cache.Invalidate()
-
-	// Create a set of categories.
-	m := make(map[uint64]struct{})
-	for _, v := range categories {
-		m[v] = struct{}{}
-	}
-
-	// Iterate over rankings and add to results until we have enough.
-	var results []Pair
-	for _, pair := range f.cache.Pairs() {
-		// Skip if categories are specified but category is not found.
-		if _, ok := m[pair.category]; (len(categories) > 0 && !ok) || pair.Count <= 0 {
-			continue
-		}
-
-		// Append pair.
-		results = append(results, pair)
-
-		// Exit when we have enough pairs.
-		if len(results) >= n {
-			break
-		}
-	}
-	return results
-}
-
 // SetBit sets a bit for a given profile & bitmap within the fragment.
 // This updates both the on-disk storage and the in-cache bitmap.
 func (f *Fragment) SetBit(bitmapID, profileID uint64) error {
@@ -302,123 +280,105 @@ func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) {
 	return (bitmapID * SliceWidth) + (profileID % SliceWidth), nil
 }
 
-func (f *Fragment) TopN(src *Bitmap, n int, categories []uint64) []Pair {
+// TopN returns the top n bitmaps from the fragment.
+// If src is specified then only bitmaps which intersect src are returned.
+// If fieldValues exist then the bitmap attribute specified by field is matched.
+func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interface{}) ([]Pair, error) {
+	// Resort cache, if needed, and retrieve the top bitmaps.
 	f.mu.Lock()
-	defer f.mu.Unlock()
-
-	// Resort rank, if necessary.
 	f.cache.Invalidate()
+	pairs := f.cache.Top()
+	f.mu.Unlock()
 
-	// Create a set of categories.
-	set := make(map[uint64]struct{})
-	for _, v := range categories {
-		set[v] = struct{}{}
+	// Create a fast lookup of filter values.
+	var filters map[interface{}]struct{}
+	if len(fieldValues) > 0 {
+		filters = make(map[interface{}]struct{})
+		for _, v := range fieldValues {
+			filters[v] = struct{}{}
+		}
 	}
 
-	var results []Pair
-	var x int
-	breakout := 1000
+	// Iterate over rankings and add to results until we have enough.
+	results := make([]Pair, 0, n)
+	for _, pair := range pairs {
+		bitmapID, bm := pair.ID, pair.Bitmap
 
-	// Iterate over rankings.
-	rankings := f.cache.Pairs()
-	for i, pair := range rankings {
-		// Skip if category not found.
-		if len(set) > 0 {
-			if _, ok := set[pair.category]; !ok {
+		// Ignore empty bitmaps.
+		if bm.Count() <= 0 {
+			continue
+		}
+
+		// Apply filter, if set.
+		if filters != nil {
+			attr, err := f.BitmapAttrStore.BitmapAttrs(bitmapID)
+			if err != nil {
+				return nil, err
+			} else if attr == nil {
+				continue
+			} else if attrValue := attr[field]; attrValue == nil {
+				continue
+			} else if _, ok := filters[attrValue]; !ok {
 				continue
 			}
 		}
 
-		// Only append if there are intersecting bits with source bitmap.
-		bc := src.IntersectionCount(pair.bitmap)
-		if bc > 0 {
-			results = append(results, Pair{
-				Key:      pair.Key,
-				Count:    bc,
-				category: pair.category,
-			})
-		}
-		x = i
-
-		// Exit when we have enough.
-		if len(results) > n {
-			break
-		}
-	}
-
-	// Sort results by ranking.
-	sort.Sort(Pairs(results))
-
-	if len(results) < n {
-		return results
-	}
-
-	end := len(results) - 1
-	o := results[end]
-	threshold := o.Count
-
-	if threshold <= 10 {
-		return results
-	}
-
-	results = append(results, o)
-	for i := x + 1; i < len(rankings); i++ {
-		o = rankings[i]
-
-		if len(set) > 0 {
-			if _, ok := set[o.category]; !ok {
+		// The initial n pairs should simply be added to the results.
+		if len(results) < n {
+			// Calculate count and append.
+			count := bm.Count()
+			if src != nil {
+				count = src.IntersectionCount(bm)
+			}
+			if count == 0 {
 				continue
 			}
-		}
+			results = append(results, Pair{Key: bitmapID, Count: count})
 
-		// Need something to do with the size of initial bitmap
-		if len(results) > breakout || o.Count < threshold {
-			break
-		}
-
-		bc := src.IntersectionCount(o.bitmap)
-		if bc > threshold {
-			if results[end-1].Count > bc {
-				results[end] = Pair{Key: o.Key, Count: bc, category: o.category}
-				threshold = bc
-			} else {
-				results[end+1] = Pair{Key: o.Key, Count: bc, category: o.category}
+			// If we reach the requested number of pairs and we are not computing
+			// intersections then simply exit. If we are intersecting then sort
+			// and then only keep pairs that are higher than the lowest count.
+			if len(results) == n {
+				if src == nil {
+					break
+				}
 				sort.Sort(Pairs(results))
-				threshold = results[end].Count
-			}
-		}
-	}
-
-	return results[:end]
-}
-
-/*
-func (f *Fragment) TopFill(args FillArgs) ([]Pair, error) {
-	result := make([]Pair, 0)
-	for _, id := range args.Bitmaps {
-		if _, ok := f.cache.Get(id); !ok {
-			continue
-		}
-
-		if args.Handle == 0 {
-			if bm := f.Bitmap(id); bm != nil && bm.Count() > 0 {
-				result = append(result, Pair{Key: id, Count: bm.Count()})
 			}
 			continue
 		}
 
-		res := f.Intersect([]uint64{args.Handle, id})
-		if res == nil {
+		// Retrieve the lowest count we have.
+		// If it's too low then don't try finding anymore pairs.
+		threshold := results[len(results)-1].Count
+		if threshold < MinThreshold {
+			break
+		}
+
+		// If the bitmap doesn't have enough bits set before the intersection
+		// then we can assume that any remaing bitmaps also have a count too low.
+		if bm.Count() < threshold {
+			break
+		}
+
+		// Calculate the intersecting bit count and skip if it's below our
+		// last bitmap in our current result set.
+		count := src.IntersectionCount(bm)
+		if count < threshold {
 			continue
 		}
 
-		if bc := res.BitCount(); bc > 0 {
-			result = append(result, Pair{Key: id, Count: bc})
+		// Swap out the last pair for this new count.
+		results[len(results)-1] = Pair{Key: bitmapID, Count: count}
+
+		// If it's count is also higher than the second to last item then resort.
+		if len(results) >= 2 && count > results[len(results)-2].Count {
+			sort.Sort(Pairs(results))
 		}
 	}
-	return result, nil
+
+	sort.Sort(Pairs(results))
+	return results, nil
 }
-*/
 
 func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap {
 	f.mu.Lock()

fragment_test.go

@@ -3,8 +3,10 @@ package pilosa_test
 import (
 	"io/ioutil"
 	"os"
+	"reflect"
 	"testing"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/umbel/pilosa"
 )
 
@@ -98,9 +100,121 @@ func TestFragment_Snapshot(t *testing.T) {
 	}
 }
 
+// Ensure a fragment can return the top n results.
+func TestFragment_TopN(t *testing.T) {
+	f := MustOpenFragment("d", "f", 0)
+	defer f.Close()
+
+	// Set bits on the bitmaps 100, 101, & 102.
+	f.MustSetBits(100, 1, 3, 200)
+	f.MustSetBits(101, 1)
+	f.MustSetBits(102, 1, 2)
+
+	// Retrieve top bitmaps.
+	if pairs, err := f.TopN(2, nil, "", nil); err != nil {
+		t.Fatal(err)
+	} else if len(pairs) != 2 {
+		t.Fatalf("unexpected count: %d", len(pairs))
+	} else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) {
+		t.Fatalf("unexpected pair(0): %v", pairs[0])
+	} else if pairs[1] != (pilosa.Pair{Key: 102, Count: 2}) {
+		t.Fatalf("unexpected pair(1): %v", pairs[1])
+	}
+}
+
+// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps.
+func TestFragment_TopN_Filter(t *testing.T) {
+	f := MustOpenFragment("d", "f", 0)
+	defer f.Close()
+
+	// Set bits on the bitmaps 100, 101, & 102.
+	f.MustSetBits(100, 1, 3, 200)
+	f.MustSetBits(101, 1)
+	f.MustSetBits(102, 1, 2)
+
+	// Assign attributes.
+	f.BitmapAttrStore.SetBitmapAttrs(101, map[string]interface{}{"x": 10})
+	f.BitmapAttrStore.SetBitmapAttrs(102, map[string]interface{}{"x": 20})
+
+	// Retrieve top bitmaps.
+	if pairs, err := f.TopN(2, nil, "x", []interface{}{10, 15, 20}); err != nil {
+		t.Fatal(err)
+	} else if len(pairs) != 2 {
+		t.Fatalf("unexpected count: %d", len(pairs))
+	} else if pairs[0] != (pilosa.Pair{Key: 102, Count: 2}) {
+		t.Fatalf("unexpected pair(0): %v", pairs[0])
+	} else if pairs[1] != (pilosa.Pair{Key: 101, Count: 1}) {
+		t.Fatalf("unexpected pair(1): %v", pairs[1])
+	}
+}
+
+// Ensure a fragment can return top bitmaps that intersect with an input bitmap.
+func TestFragment_TopN_Intersect(t *testing.T) {
+	f := MustOpenFragment("d", "f", 0)
+	defer f.Close()
+
+	// Create an intersecting input bitmap.
+	src := pilosa.NewBitmap(1, 2, 3)
+
+	// Set bits on various bitmaps.
+	f.MustSetBits(100, 1, 10, 11, 12)    // one intersection
+	f.MustSetBits(101, 1, 2, 3, 4)       // three intersections
+	f.MustSetBits(102, 1, 2, 4, 5, 6)    // two intersections
+	f.MustSetBits(103, 1000, 1001, 1002) // no intersection
+
+	// Retrieve top bitmaps.
+	if pairs, err := f.TopN(3, src, "", nil); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+		{Key: 101, Count: 3},
+		{Key: 102, Count: 2},
+		{Key: 100, Count: 1},
+	}) {
+		t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
+	}
+}
+
+// Ensure a fragment can return top bitmaps that have many bits set.
+func TestFragment_TopN_Intersect_Large(t *testing.T) {
+	f := MustOpenFragment("d", "f", 0)
+	defer f.Close()
+
+	// Create an intersecting input bitmap.
+	src := pilosa.NewBitmap(
+		980, 981, 982, 983, 984, 985, 986, 987, 988, 989,
+		990, 991, 992, 993, 994, 995, 996, 997, 998, 999,
+	)
+
+	// Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts.
+	for i := uint64(0); i < 1000; i++ {
+		for j := uint64(0); j < i; j++ {
+			f.MustSetBits(i, j)
+		}
+	}
+
+	// Retrieve top bitmaps.
+	if pairs, err := f.TopN(10, src, "", nil); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+		{Key: 999, Count: 19},
+		{Key: 998, Count: 18},
+		{Key: 997, Count: 17},
+		{Key: 996, Count: 16},
+		{Key: 995, Count: 15},
+		{Key: 994, Count: 14},
+		{Key: 993, Count: 13},
+		{Key: 992, Count: 12},
+		{Key: 991, Count: 11},
+		{Key: 990, Count: 10},
+	}) {
+		t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
+	}
+}
+
 // Fragment is a test wrapper for pilosa.Fragment.
 type Fragment struct {
 	*pilosa.Fragment
+	BitmapAttrStore *BitmapAttrStore
 }
 
 // NewFragment returns a new instance of Fragment with a temporary path.
@@ -110,7 +224,13 @@ func NewFragment(db, frame string, slice uint64) *Fragment {
 		panic(err)
 	}
 	file.Close()
-	return &Fragment{Fragment: pilosa.NewFragment(file.Name(), db, frame, slice)}
+
+	f := &Fragment{
+		Fragment:        pilosa.NewFragment(file.Name(), db, frame, slice),
+		BitmapAttrStore: NewBitmapAttrStore(),
+	}
+	f.Fragment.BitmapAttrStore = f.BitmapAttrStore
+	return f
 }
 
 // MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
@@ -142,16 +262,42 @@ func (f *Fragment) Reopen() error {
 	return nil
 }
 
-// MustSetBit sets a bit on a bitmap. Panic on error.
-func (f *Fragment) MustSetBit(bitmapID, profileID uint64) {
-	if err := f.SetBit(bitmapID, profileID); err != nil {
-		panic(err)
+// MustSetBits sets bits on a bitmap. Panic on error.
+func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
+	for _, profileID := range profileIDs {
+		if err := f.SetBit(bitmapID, profileID); err != nil {
+			panic(err)
+		}
 	}
 }
 
-// MustClearBit clears a bit on a bitmap. Panic on error.
-func (f *Fragment) MustClearBit(bitmapID, profileID uint64) {
-	if err := f.ClearBit(bitmapID, profileID); err != nil {
-		panic(err)
+// MustClearBits clears bits on a bitmap. Panic on error.
+func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
+	for _, profileID := range profileIDs {
+		if err := f.ClearBit(bitmapID, profileID); err != nil {
+			panic(err)
+		}
 	}
 }
+
+// BitmapAttrStore provides simple storage for attributes.
+type BitmapAttrStore struct {
+	attrs map[uint64]map[string]interface{}
+}
+
+// NewBitmapAttrStore returns a new instance of BitmapAttrStore.
+func NewBitmapAttrStore() *BitmapAttrStore {
+	return &BitmapAttrStore{
+		attrs: make(map[uint64]map[string]interface{}),
+	}
+}
+
+// BitmapAttrs returns the attributes set to a bitmap id.
+func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) {
+	return s.attrs[id], nil
+}
+
+// SetBitmapAttrs assigns a set of attributes to a bitmap id.
+func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) {
+	s.attrs[id] = m
+}

frame.go

@@ -109,6 +109,7 @@ func (f *Frame) openFragments() error {
 		if err := frag.Open(); err != nil {
 			return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
 		}
+		frag.BitmapAttrStore = f
 		f.fragments[frag.Slice()] = frag
 	}
 
@@ -189,6 +190,7 @@ func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
 	if err := frag.Open(); err != nil {
 		return nil, err
 	}
+	frag.BitmapAttrStore = f
 	f.fragments[slice] = frag
 
 	return frag, nil

handler.go

@@ -400,7 +400,7 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
 		switch result := resp.Result.(type) {
 		case *Bitmap:
 			pb.Bitmap = encodeBitmap(result)
-		case Pairs:
+		case []Pair:
 			pb.Pairs = encodePairs(result)
 		case uint64:
 			pb.N = proto.Uint64(result)

handler_test.go

@@ -288,7 +288,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
 func TestHandler_Query_Pairs_JSON(t *testing.T) {
 	h := NewHandler()
 	h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
-		return pilosa.Pairs{
+		return []pilosa.Pair{
 			{Key: 1, Count: 2},
 			{Key: 3, Count: 4},
 		}, nil
@@ -307,7 +307,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
 func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
 	h := NewHandler()
 	h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
-		return pilosa.Pairs{
+		return []pilosa.Pair{
 			{Key: 1, Count: 2},
 			{Key: 3, Count: 4},
 		}, nil

pql/ast.go

@@ -111,7 +111,6 @@ func (c *Bitmap) String() string {
 type ClearBit struct {
 	ID        uint64
 	Frame     string
-	Filter    uint64
 	ProfileID uint64
 }
 
@@ -122,9 +121,6 @@ func (c *ClearBit) String() string {
 	if c.Frame != "" {
 		args = append(args, fmt.Sprintf("frame=%s", c.Frame))
 	}
-	if c.Filter != 0 {
-		args = append(args, fmt.Sprintf("filter=%d", c.Filter))
-	}
 	if c.ProfileID != 0 {
 		args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID))
 	}
@@ -203,7 +199,6 @@ func (c *Range) String() string {
 type SetBit struct {
 	ID        uint64
 	Frame     string
-	Filter    uint64
 	ProfileID uint64
 }
 
@@ -214,9 +209,6 @@ func (c *SetBit) String() string {
 	if c.Frame != "" {
 		args = append(args, fmt.Sprintf("frame=%s", c.Frame))
 	}
-	if c.Filter != 0 {
-		args = append(args, fmt.Sprintf("filter=%d", c.Filter))
-	}
 	if c.ProfileID != 0 {
 		args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID))
 	}
@@ -302,11 +294,48 @@ func (c *SetProfileAttrs) String() string {
 // TopN represents a TopN() function call.
 type TopN struct {
 	Frame string
-	N     int
+
+	// Bitmap to use for intersection while computing top results.
+	// Original bitmap counts are used if no Src is provided.
+	Src BitmapCall
+
+	// Maximum number of results to return.
+	N int
+
+	// Field name and values to filter on.
+	Field   string
+	Filters []interface{}
 }
 
 // String returns the string representation of the call.
-func (c *TopN) String() string { panic("FIXME") }
+func (c *TopN) String() string {
+	args := make([]string, 0, 2)
+	if c.Src != nil {
+		args = append(args, c.Src.String())
+	}
+	if c.Frame != "" {
+		args = append(args, fmt.Sprintf("frame=%s", c.Frame))
+	}
+	if c.N > 0 {
+		args = append(args, fmt.Sprintf("n=%d", c.N))
+	}
+	if c.Field != "" {
+		args = append(args, fmt.Sprintf("field=%q", c.Field))
+	}
+	if len(c.Filters) > 0 {
+		filters := make([]string, 0, len(c.Filters))
+		for i := range c.Filters {
+			switch filter := c.Filters[i].(type) {
+			case string:
+				filters = append(filters, fmt.Sprintf("%q", filter))
+			default:
+				filters = append(filters, fmt.Sprintf("%v", filter))
+			}
+		}
+		args = append(args, fmt.Sprintf("[%s]", strings.Join(filters, ",")))
+	}
+	return fmt.Sprintf("TopN(%s)", strings.Join(args, ", "))
+}
 
 // Union represents a union() function call.
 type Union struct {

pql/ast_test.go

@@ -17,8 +17,8 @@ func TestBitmap_String(t *testing.T) {
 
 // Ensure the ClearBit call can be converted into a string.
 func TestClearBit_String(t *testing.T) {
-	s := (&pql.ClearBit{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
-	if s != `ClearBit(id=1, frame=x.n, filter=2, profileID=3)` {
+	s := (&pql.ClearBit{ID: 1, Frame: "x.n", ProfileID: 3}).String()
+	if s != `ClearBit(id=1, frame=x.n, profileID=3)` {
 		t.Fatalf("unexpected string: %s", s)
 	}
 }
@@ -77,8 +77,8 @@ func TestRange_String(t *testing.T) {
 
 // Ensure the SetBit call can be converted into a string.
 func TestSetBit_String(t *testing.T) {
-	s := (&pql.SetBit{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
-	if s != `SetBit(id=1, frame=x.n, filter=2, profileID=3)` {
+	s := (&pql.SetBit{ID: 1, Frame: "x.n", ProfileID: 3}).String()
+	if s != `SetBit(id=1, frame=x.n, profileID=3)` {
 		t.Fatalf("unexpected string: %s", s)
 	}
 }

pql/parser.go

@@ -136,11 +136,7 @@ func (p *Parser) parseClearBitCall() (*ClearBit, error) {
 			if err := decodeString(arg.value, &c.Frame); err != nil {
 				return nil, parseErrorf(pos, "frame: %s", err)
 			}
-		case 2, "filter":
-			if err := decodeUint64(arg.value, &c.Filter); err != nil {
-				return nil, parseErrorf(pos, "filter: %s", err)
-			}
-		case 3, "profileID":
+		case 2, "profileID":
 			if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
 				return nil, parseErrorf(pos, "profileID: %s", err)
 			}
@@ -337,11 +333,7 @@ func (p *Parser) parseSetBitCall() (*SetBit, error) {
 			if err := decodeString(arg.value, &c.Frame); err != nil {
 				return nil, parseErrorf(pos, "frame: %s", err)
 			}
-		case 2, "filter":
-			if err := decodeUint64(arg.value, &c.Filter); err != nil {
-				return nil, parseErrorf(pos, "filter: %s", err)
-			}
-		case 3, "profileID":
+		case 2, "profileID":
 			if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
 				return nil, parseErrorf(pos, "profileID: %s", err)
 			}
@@ -477,6 +469,15 @@ func (p *Parser) parseTopNCall() (*TopN, error) {
 
 	// Copy arguments to AST.
 	for _, arg := range args {
+		if v, ok := arg.value.(BitmapCall); ok {
+			c.Src = v
+			continue
+		}
+		if v, ok := arg.value.([]interface{}); ok {
+			c.Filters = v
+			continue
+		}
+
 		switch arg.key {
 		case 0, "frame":
 			if err := decodeString(arg.value, &c.Frame); err != nil {
@@ -486,6 +487,10 @@ func (p *Parser) parseTopNCall() (*TopN, error) {
 			if err := decodeInt(arg.value, &c.N); err != nil {
 				return nil, parseErrorf(pos, "n: %s", err)
 			}
+		case 2, "field":
+			if err := decodeString(arg.value, &c.Field); err != nil {
+				return nil, parseErrorf(pos, "n: %s", err)
+			}
 		default:
 			return nil, parseErrorf(pos, "invalid TopN() arg: %v", arg.key)
 		}
@@ -608,7 +613,11 @@ func (p *Parser) parseArg() (arg, error) {
 		}
 		value = v
 	case LBRACK:
-		panic("FIXME: parse list of integers")
+		v, err := p.parseList()
+		if err != nil {
+			return arg{}, err
+		}
+		value = v
 	default:
 		return arg{}, parseErrorf(pos, "invalid value: %q", lit)
 	}
@@ -616,6 +625,43 @@ func (p *Parser) parseArg() (arg, error) {
 	return arg{key: key, value: value}, nil
 }
 
+// parseListArg parses a list of primitives. This is used by the TopN() filters.
+func (p *Parser) parseList() ([]interface{}, error) {
+	var values []interface{}
+	for {
+		// Read next value.
+		tok, pos, lit := p.scanIgnoreWhitespace()
+		switch tok {
+		case IDENT:
+			if lit == "true" {
+				values = append(values, true)
+			} else if lit == "false" {
+				values = append(values, false)
+			} else {
+				values = append(values, lit)
+			}
+		case STRING:
+			values = append(values, lit)
+		case NUMBER:
+			v, err := strconv.ParseUint(lit, 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			values = append(values, v)
+		default:
+			return nil, parseErrorf(pos, "invalid list value: %q", lit)
+		}
+
+		// Expect a comma or closing bracket next.
+		if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
+			break
+		} else if tok != COMMA {
+			return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
+		}
+	}
+	return values, nil
+}
+
 // scan returns the next token from the scanner.
 func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
 

pql/parser_test.go

@@ -41,14 +41,13 @@ func TestParser_Parse_Bitmap_Array(t *testing.T) {
 
 // Ensure the parser can parse a "ClearBit()" function with keyed args.
 func TestParser_Parse_ClearBit_Key(t *testing.T) {
-	q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", filter=2, profileID = 3)`)
+	q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", profileID = 3)`)
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(q, &pql.Query{
 		Root: &pql.ClearBit{
 			ID:        1,
 			Frame:     "b.n",
-			Filter:    2,
 			ProfileID: 3,
 		},
 	}) {
@@ -58,14 +57,13 @@ func TestParser_Parse_ClearBit_Key(t *testing.T) {
 
 // Ensure the parser can parse a "ClearBit()" function with array args.
 func TestParser_Parse_ClearBit_Array(t *testing.T) {
-	q, err := pql.ParseString(`ClearBit(1, "b.n", 2, 3)`)
+	q, err := pql.ParseString(`ClearBit(1, "b.n", 3)`)
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(q, &pql.Query{
 		Root: &pql.ClearBit{
 			ID:        1,
 			Frame:     "b.n",
-			Filter:    2,
 			ProfileID: 3,
 		},
 	}) {
@@ -183,14 +181,13 @@ func TestParser_Parse_Range_Array(t *testing.T) {
 
 // Ensure the parser can parse a "SetBit()" function with keyed args.
 func TestParser_Parse_SetBit_Key(t *testing.T) {
-	q, err := pql.ParseString(`SetBit(id=1, frame="b.n", filter=2, profileID = 3)`)
+	q, err := pql.ParseString(`SetBit(id=1, frame="b.n", profileID = 3)`)
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(q, &pql.Query{
 		Root: &pql.SetBit{
 			ID:        1,
 			Frame:     "b.n",
-			Filter:    2,
 			ProfileID: 3,
 		},
 	}) {
@@ -200,14 +197,13 @@ func TestParser_Parse_SetBit_Key(t *testing.T) {
 
 // Ensure the parser can parse a "SetBit()" function with array args.
 func TestParser_Parse_SetBit_Array(t *testing.T) {
-	q, err := pql.ParseString(`SetBit(1, "b.n", 2, 3)`)
+	q, err := pql.ParseString(`SetBit(1, "b.n", 3)`)
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(q, &pql.Query{
 		Root: &pql.SetBit{
 			ID:        1,
 			Frame:     "b.n",
-			Filter:    2,
 			ProfileID: 3,
 		},
 	}) {
@@ -258,32 +254,46 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) {
 
 // Ensure the parser can parse a "TopN()" function with keyed args.
 func TestParser_Parse_TopN_Key(t *testing.T) {
-	q, err := pql.ParseString(`TopN(frame="b.n", n=2)`)
+	q, err := pql.ParseString(`TopN(Bitmap(100), frame="b.n", n=2, field="XXX", [5,10,15])`)
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(q, &pql.Query{
 		Root: &pql.TopN{
-			Frame: "b.n",
-			N:     2,
+			Src:     &pql.Bitmap{ID: 100},
+			Frame:   "b.n",
+			N:       2,
+			Field:   "XXX",
+			Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
 		},
 	}) {
 		t.Fatalf("unexpected query: %s", spew.Sdump(q))
 	}
+
+	if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", [5,10,15])` {
+		t.Fatalf("unexpected string encoding: %s", s)
+	}
 }
 
 // Ensure the parser can parse a "TopN()" function with array args.
 func TestParser_Parse_TopN_Array(t *testing.T) {
-	q, err := pql.ParseString(`TopN("b.n", 2)`)
+	q, err := pql.ParseString(`TopN(Bitmap(100), "b.n", 2, "XXX", ["foo",true,false])`)
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(q, &pql.Query{
 		Root: &pql.TopN{
-			Frame: "b.n",
-			N:     2,
+			Src:     &pql.Bitmap{ID: 100},
+			Frame:   "b.n",
+			N:       2,
+			Field:   "XXX",
+			Filters: []interface{}{"foo", true, false},
 		},
 	}) {
 		t.Fatalf("unexpected query: %s", spew.Sdump(q))
 	}
+
+	if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", ["foo",true,false])` {
+		t.Fatalf("unexpected string encoding: %s", s)
+	}
 }
 
 // Ensure the parser can parse a "union()" function.