diff --git a/cache.go b/cache.go index 5a5316469..da4366122 100644 --- a/cache.go +++ b/cache.go @@ -1,7 +1,6 @@ package pilosa import ( - "encoding/json" "io" "sort" "time" @@ -13,40 +12,37 @@ import ( // Cache represents a cache for bitmaps. type Cache interface { - io.WriterTo - io.ReaderFrom - - Add(bitmapID, category uint64, bm *Bitmap) + Add(bitmapID uint64, bm *Bitmap) Get(bitmapID uint64) *Bitmap Len() int // Updates the cache, if necessary. Invalidate() - // Returns a list of all key/count pairs. - Pairs() []Pair + // Returns an ordered list of the top ranked bitmaps. + Top() []BitmapPair } // LRUCache represents a least recently used Cache implemenation. type LRUCache struct { - cache *lru.Cache - keys map[uint64]struct{} + cache *lru.Cache + bitmaps map[uint64]*Bitmap } // NewLRUCache returns a new instance of LRUCache. func NewLRUCache(maxEntries int) *LRUCache { c := &LRUCache{ - cache: lru.New(maxEntries), - keys: make(map[uint64]struct{}), + cache: lru.New(maxEntries), + bitmaps: make(map[uint64]*Bitmap), } c.cache.OnEvicted = c.onEvicted return c } // Get returns a bitmap with a given id. -func (c *LRUCache) Add(bitmapID, category uint64, bm *Bitmap) { +func (c *LRUCache) Add(bitmapID uint64, bm *Bitmap) { c.cache.Add(bitmapID, bm) - c.keys[bitmapID] = struct{}{} + c.bitmaps[bitmapID] = bm } // Get returns a bitmap with a given id. @@ -64,53 +60,28 @@ func (c *LRUCache) Len() int { return c.cache.Len() } // Invalidate is a no-op. func (c *LRUCache) Invalidate() {} -// Pairs returns all key/count pairs in the cache. -func (c *LRUCache) Pairs() []Pair { - a := make([]Pair, 0, len(c.keys)) - for k := range c.keys { - a = append(a, Pair{ - Key: k, - Count: c.Get(k).Count(), +// Top returns all bitmaps in the cache. +func (c *LRUCache) Top() []BitmapPair { + a := make([]BitmapPair, 0, len(c.bitmaps)) + for id, bm := range c.bitmaps { + a = append(a, BitmapPair{ + ID: id, + Bitmap: bm, }) } + sort.Sort(BitmapPairs(a)) return a } -// WriteTo writes the cache to w. -func (c *LRUCache) WriteTo(w io.Writer) (n int64, err error) { - // Write keys to slice. - a := make([]uint64, 0, len(c.keys)) - for k := range c.keys { - a = append(a, k) - } - - // Encode to file as array of keys. - if err := json.NewEncoder(w).Encode(a); err != nil { - return 0, err - } - return 0, nil -} - -// ReadFrom read from r into the cache. -func (c *LRUCache) ReadFrom(r io.Reader) (n int64, err error) { - var keys []uint64 - if err := json.NewDecoder(r).Decode(&keys); err != nil { - return 0, err - } - panic("FIXME: TODO") -} - -func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { - delete(c.keys, key.(uint64)) -} +func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.bitmaps, key.(uint64)) } // Ensure LRUCache implements Cache. var _ Cache = &LRUCache{} // RankCache represents a cache with sorted entries. type RankCache struct { - entries map[uint64]*Pair - rankings []Pair // cached, ordered list + entries map[uint64]*Bitmap + rankings []BitmapPair // cached, ordered list updateN int updateTime time.Time @@ -123,44 +94,33 @@ type RankCache struct { // NewRankCache returns a new instance of RankCache. func NewRankCache() *RankCache { return &RankCache{ - entries: make(map[uint64]*Pair), + entries: make(map[uint64]*Bitmap), } } // Get returns a bitmap with a given id. -func (c *RankCache) Add(bitmapID, category uint64, bm *Bitmap) { +func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) { // Ignore if the bit count on the bitmap is below the threshold. if bm.Count() < c.ThresholdValue { return } // Add to cache. - c.entries[bitmapID] = &Pair{ - Key: bitmapID, - Count: bm.Count(), - bitmap: bm, - category: category, - } + c.entries[bitmapID] = bm // If size is larger than the threshold then trim it. if len(c.entries) > c.ThresholdLength { c.update() - for k, entry := range c.entries { - if entry.bitmap.Count() <= c.ThresholdValue { - delete(c.entries, k) + for id, bm := range c.entries { + if bm.Count() <= c.ThresholdValue { + delete(c.entries, id) } } } } // Get returns a bitmap with a given id. -func (c *RankCache) Get(bitmapID uint64) *Bitmap { - entry, ok := c.entries[bitmapID] - if !ok { - return nil - } - return entry.bitmap -} +func (c *RankCache) Get(bitmapID uint64) *Bitmap { return c.entries[bitmapID] } // Len returns the number of items in the cache. func (c *RankCache) Len() int { return len(c.entries) } @@ -176,21 +136,19 @@ func (c *RankCache) Invalidate() { // update reorders the entries by rank. func (c *RankCache) update() { // Convert cache to a sorted list. - list := make([]Pair, 0, len(c.entries)) - for k, item := range c.entries { - list = append(list, Pair{ - Key: k, - Count: item.bitmap.Count(), - bitmap: item.bitmap, - category: item.category, + rankings := make([]BitmapPair, 0, len(c.entries)) + for id, bm := range c.entries { + rankings = append(rankings, BitmapPair{ + ID: id, + Bitmap: bm, }) } - sort.Sort(Pairs(list)) + sort.Sort(BitmapPairs(rankings)) // Store the count of the item at the threshold index. - c.rankings = list + c.rankings = rankings if len(c.rankings) > c.ThresholdIndex { - c.ThresholdValue = list[c.ThresholdIndex].bitmap.Count() + c.ThresholdValue = rankings[c.ThresholdIndex].Bitmap.Count() } else { c.ThresholdValue = 1 } @@ -199,8 +157,8 @@ func (c *RankCache) update() { c.updateTime, c.updateN = time.Now(), 0 } -// Pairs returns an ordered list of key/count pairs. -func (c *RankCache) Pairs() []Pair { return c.rankings } +// Top returns an ordered list of bitmaps. +func (c *RankCache) Top() []BitmapPair { return c.rankings } // WriteTo writes the cache to w. func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) { @@ -215,12 +173,22 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) { // Ensure RankCache implements Cache. var _ Cache = &RankCache{} +// BitmapPair represents a bitmap with an associated identifier. +type BitmapPair struct { + ID uint64 + Bitmap *Bitmap +} + +// BitmapPairs is a sortable list of BitmapPair objects. +type BitmapPairs []BitmapPair + +func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p BitmapPairs) Len() int { return len(p) } +func (p BitmapPairs) Less(i, j int) bool { return p[i].Bitmap.Count() > p[j].Bitmap.Count() } + type Pair struct { Key uint64 `json:"key"` Count uint64 `json:"count"` - - bitmap *Bitmap - category uint64 } func encodePair(p Pair) *internal.Pair { @@ -243,6 +211,27 @@ func (p Pairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Pairs) Len() int { return len(p) } func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count } +// Add merges other into p and returns a new slice. +func (p Pairs) Add(other []Pair) []Pair { + // Create lookup of key/counts. + m := make(map[uint64]uint64, len(p)) + for _, pair := range p { + m[pair.Key] = pair.Count + } + + // Add/merge from other. + for _, pair := range other { + m[pair.Key] += pair.Count + } + + // Convert back to slice. + a := make([]Pair, 0, len(m)) + for k, v := range m { + a = append(a, Pair{Key: k, Count: v}) + } + return a +} + func encodePairs(a Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { diff --git a/executor.go b/executor.go index 72c9b8e7b..406af17b3 100644 --- a/executor.go +++ b/executor.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" "net/url" + "sort" "github.com/gogo/protobuf/proto" "github.com/umbel/pilosa/internal" @@ -147,7 +148,63 @@ func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uin // executeTopN executes a TopN() call. func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) { - panic("FIXME: calculate top n from each slice") + var results []Pair + for node, nodeSlices := range e.slicesByNode(slices) { + // Execute locally if the hostname matches. + if node.Host == e.Host { + for _, slice := range nodeSlices { + pairs, err := e.executeTopNSlice(db, c, slice) + if err != nil { + return nil, err + } + results = Pairs(results).Add(pairs) + } + continue + } + + // Otherwise execute remotely. + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + if err != nil { + return nil, err + } + results = Pairs(results).Add(res.([]Pair)) + } + + // Sort final merged results. + sort.Sort(Pairs(results)) + + // Only keep the top n after sorting. + if len(results) > c.N { + results = results[0:c.N] + } + + return results, nil +} + +// executeTopNSlice executes a TopN call for a single slice. +func (e *Executor) executeTopNSlice(db string, c *pql.TopN, slice uint64) ([]Pair, error) { + // Retrieve bitmap used to intersect. + var src *Bitmap + if c.Src != nil { + bm, err := e.executeBitmapCallSlice(db, c.Src, slice) + if err != nil { + return nil, err + } + src = bm + } + + // Set default frame. + frame := c.Frame + if frame == "" { + frame = DefaultFrame + } + + f := e.Index().Fragment(db, frame, slice) + if f == nil { + return nil, nil + } + + return f.TopN(c.N, src, c.Field, c.Filters) } // executeDifferenceSlice executes a difference() call for a local slice. diff --git a/executor_test.go b/executor_test.go index 004957a31..209c894bd 100644 --- a/executor_test.go +++ b/executor_test.go @@ -14,8 +14,8 @@ import ( func TestExecutor_Execute_Bitmap(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBit(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) if err := idx.Frame("d", "f").SetBitmapAttrs(10, map[string]interface{}{"foo": "bar", "baz": 123}); err != nil { t.Fatal(err) @@ -39,10 +39,10 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 3) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { @@ -58,13 +58,13 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(11, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 1) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { @@ -82,12 +82,12 @@ func TestExecutor_Execute_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 0) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0) + idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(11, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { @@ -105,9 +105,9 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBit(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil { @@ -162,6 +162,67 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { } } +// Ensure a TopN() query can be executed. +func TestExecutor_Execute_TopN(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + // Set bits for bitmaps 0, 10, & 20 across two slices. + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) + + // Execute query. + e := NewExecutor(idx.Index, NewCluster(1)) + if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result, []pilosa.Pair{ + {Key: 0, Count: 5}, + {Key: 10, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } +} + +// Ensure a TopN() query with a source bitmap can be executed. +func TestExecutor_Execute_TopN_Src(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + // Set bits for bitmaps 0, 10, & 20 across two slices. + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) + + // Create an intersecting bitmap. + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) + + // Execute query. + e := NewExecutor(idx.Index, NewCluster(1)) + if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result, []pilosa.Pair{ + {Key: 20, Count: 3}, + {Key: 10, Count: 2}, + {Key: 0, Count: 1}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } +} + // Ensure a remote query can return a bitmap. func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c := NewCluster(2) @@ -194,7 +255,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(idx.Index, c) if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil { @@ -225,8 +286,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2) e := NewExecutor(idx.Index, c) if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil { @@ -276,6 +337,51 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } } +// Ensure a remote query can return a top-n query. +func TestExecutor_Execute_Remote_TopN(t *testing.T) { + c := NewCluster(2) + + // Create secondary server and update second cluster node. + s := NewServer() + defer s.Close() + c.Nodes[1].Host = s.Host() + + // Mock secondary server's executor to verify arguments and return a bitmap. + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + if db != `d` { + t.Fatalf("unexpected db: %s", db) + } else if query.String() != `TopN(frame=f, n=3)` { + t.Fatalf("unexpected query: %s", query.String()) + } else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) { + t.Fatalf("unexpected slices: %+v", slices) + } + + // Return pair counts. + return []pilosa.Pair{ + {Key: 0, Count: 5}, + {Key: 10, Count: 2}, + {Key: 30, Count: 2}, + }, nil + } + + // Create local executor data on slice 1 & 3. + idx := MustOpenIndex() + defer idx.Close() + idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(30, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2) + + e := NewExecutor(idx.Index, c) + if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(res, []pilosa.Pair{ + {Key: 0, Count: 5}, + {Key: 30, Count: 4}, + {Key: 10, Count: 2}, + }) { + t.Fatalf("unexpected results: %s", spew.Sdump(res)) + } +} + // Executor represents a test wrapper for pilosa.Executor. type Executor struct { *pilosa.Executor diff --git a/fragment.go b/fragment.go index 944eaad62..ff9fa29ac 100644 --- a/fragment.go +++ b/fragment.go @@ -20,6 +20,8 @@ const SliceWidth = 65536 // SnapshotExt is the file extension used for an in-process snapshot. const SnapshotExt = ".snapshotting" +const MinThreshold = 10 + // Fragment represents the intersection of a frame and slice in a database. type Fragment struct { mu sync.Mutex @@ -37,6 +39,12 @@ type Fragment struct { // Bitmap cache. cache Cache + + // Bitmap attribute storage. + // Typically this is the parent frame unless overridden for testing. + BitmapAttrStore interface { + BitmapAttrs(id uint64) (map[string]interface{}, error) + } } // NewFragment returns a new instance of Fragment. @@ -207,41 +215,11 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { }) // Add to the cache. - f.cache.Add(bitmapID, 0 /*filter*/, bm) + f.cache.Add(bitmapID, bm) return bm } -func (f *Fragment) TopNAll(n int, categories []uint64) []Pair { - f.mu.Lock() - defer f.mu.Unlock() - f.cache.Invalidate() - - // Create a set of categories. - m := make(map[uint64]struct{}) - for _, v := range categories { - m[v] = struct{}{} - } - - // Iterate over rankings and add to results until we have enough. - var results []Pair - for _, pair := range f.cache.Pairs() { - // Skip if categories are specified but category is not found. - if _, ok := m[pair.category]; (len(categories) > 0 && !ok) || pair.Count <= 0 { - continue - } - - // Append pair. - results = append(results, pair) - - // Exit when we have enough pairs. - if len(results) >= n { - break - } - } - return results -} - // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) SetBit(bitmapID, profileID uint64) error { @@ -302,123 +280,105 @@ func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) { return (bitmapID * SliceWidth) + (profileID % SliceWidth), nil } -func (f *Fragment) TopN(src *Bitmap, n int, categories []uint64) []Pair { +// TopN returns the top n bitmaps from the fragment. +// If src is specified then only bitmaps which intersect src are returned. +// If fieldValues exist then the bitmap attribute specified by field is matched. +func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interface{}) ([]Pair, error) { + // Resort cache, if needed, and retrieve the top bitmaps. f.mu.Lock() - defer f.mu.Unlock() - - // Resort rank, if necessary. f.cache.Invalidate() + pairs := f.cache.Top() + f.mu.Unlock() - // Create a set of categories. - set := make(map[uint64]struct{}) - for _, v := range categories { - set[v] = struct{}{} + // Create a fast lookup of filter values. + var filters map[interface{}]struct{} + if len(fieldValues) > 0 { + filters = make(map[interface{}]struct{}) + for _, v := range fieldValues { + filters[v] = struct{}{} + } } - var results []Pair - var x int - breakout := 1000 + // Iterate over rankings and add to results until we have enough. + results := make([]Pair, 0, n) + for _, pair := range pairs { + bitmapID, bm := pair.ID, pair.Bitmap - // Iterate over rankings. - rankings := f.cache.Pairs() - for i, pair := range rankings { - // Skip if category not found. - if len(set) > 0 { - if _, ok := set[pair.category]; !ok { + // Ignore empty bitmaps. + if bm.Count() <= 0 { + continue + } + + // Apply filter, if set. + if filters != nil { + attr, err := f.BitmapAttrStore.BitmapAttrs(bitmapID) + if err != nil { + return nil, err + } else if attr == nil { + continue + } else if attrValue := attr[field]; attrValue == nil { + continue + } else if _, ok := filters[attrValue]; !ok { continue } } - // Only append if there are intersecting bits with source bitmap. - bc := src.IntersectionCount(pair.bitmap) - if bc > 0 { - results = append(results, Pair{ - Key: pair.Key, - Count: bc, - category: pair.category, - }) - } - x = i - - // Exit when we have enough. - if len(results) > n { - break - } - } - - // Sort results by ranking. - sort.Sort(Pairs(results)) - - if len(results) < n { - return results - } - - end := len(results) - 1 - o := results[end] - threshold := o.Count - - if threshold <= 10 { - return results - } - - results = append(results, o) - for i := x + 1; i < len(rankings); i++ { - o = rankings[i] - - if len(set) > 0 { - if _, ok := set[o.category]; !ok { + // The initial n pairs should simply be added to the results. + if len(results) < n { + // Calculate count and append. + count := bm.Count() + if src != nil { + count = src.IntersectionCount(bm) + } + if count == 0 { continue } - } + results = append(results, Pair{Key: bitmapID, Count: count}) - // Need something to do with the size of initial bitmap - if len(results) > breakout || o.Count < threshold { - break - } - - bc := src.IntersectionCount(o.bitmap) - if bc > threshold { - if results[end-1].Count > bc { - results[end] = Pair{Key: o.Key, Count: bc, category: o.category} - threshold = bc - } else { - results[end+1] = Pair{Key: o.Key, Count: bc, category: o.category} + // If we reach the requested number of pairs and we are not computing + // intersections then simply exit. If we are intersecting then sort + // and then only keep pairs that are higher than the lowest count. + if len(results) == n { + if src == nil { + break + } sort.Sort(Pairs(results)) - threshold = results[end].Count - } - } - } - - return results[:end] -} - -/* -func (f *Fragment) TopFill(args FillArgs) ([]Pair, error) { - result := make([]Pair, 0) - for _, id := range args.Bitmaps { - if _, ok := f.cache.Get(id); !ok { - continue - } - - if args.Handle == 0 { - if bm := f.Bitmap(id); bm != nil && bm.Count() > 0 { - result = append(result, Pair{Key: id, Count: bm.Count()}) } continue } - res := f.Intersect([]uint64{args.Handle, id}) - if res == nil { + // Retrieve the lowest count we have. + // If it's too low then don't try finding anymore pairs. + threshold := results[len(results)-1].Count + if threshold < MinThreshold { + break + } + + // If the bitmap doesn't have enough bits set before the intersection + // then we can assume that any remaing bitmaps also have a count too low. + if bm.Count() < threshold { + break + } + + // Calculate the intersecting bit count and skip if it's below our + // last bitmap in our current result set. + count := src.IntersectionCount(bm) + if count < threshold { continue } - if bc := res.BitCount(); bc > 0 { - result = append(result, Pair{Key: id, Count: bc}) + // Swap out the last pair for this new count. + results[len(results)-1] = Pair{Key: bitmapID, Count: count} + + // If it's count is also higher than the second to last item then resort. + if len(results) >= 2 && count > results[len(results)-2].Count { + sort.Sort(Pairs(results)) } } - return result, nil + + sort.Sort(Pairs(results)) + return results, nil } -*/ func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap { f.mu.Lock() diff --git a/fragment_test.go b/fragment_test.go index ad8057be9..cbb514929 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -3,8 +3,10 @@ package pilosa_test import ( "io/ioutil" "os" + "reflect" "testing" + "github.com/davecgh/go-spew/spew" "github.com/umbel/pilosa" ) @@ -98,9 +100,121 @@ func TestFragment_Snapshot(t *testing.T) { } } +// Ensure a fragment can return the top n results. +func TestFragment_TopN(t *testing.T) { + f := MustOpenFragment("d", "f", 0) + defer f.Close() + + // Set bits on the bitmaps 100, 101, & 102. + f.MustSetBits(100, 1, 3, 200) + f.MustSetBits(101, 1) + f.MustSetBits(102, 1, 2) + + // Retrieve top bitmaps. + if pairs, err := f.TopN(2, nil, "", nil); err != nil { + t.Fatal(err) + } else if len(pairs) != 2 { + t.Fatalf("unexpected count: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } else if pairs[1] != (pilosa.Pair{Key: 102, Count: 2}) { + t.Fatalf("unexpected pair(1): %v", pairs[1]) + } +} + +// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps. +func TestFragment_TopN_Filter(t *testing.T) { + f := MustOpenFragment("d", "f", 0) + defer f.Close() + + // Set bits on the bitmaps 100, 101, & 102. + f.MustSetBits(100, 1, 3, 200) + f.MustSetBits(101, 1) + f.MustSetBits(102, 1, 2) + + // Assign attributes. + f.BitmapAttrStore.SetBitmapAttrs(101, map[string]interface{}{"x": 10}) + f.BitmapAttrStore.SetBitmapAttrs(102, map[string]interface{}{"x": 20}) + + // Retrieve top bitmaps. + if pairs, err := f.TopN(2, nil, "x", []interface{}{10, 15, 20}); err != nil { + t.Fatal(err) + } else if len(pairs) != 2 { + t.Fatalf("unexpected count: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 102, Count: 2}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } else if pairs[1] != (pilosa.Pair{Key: 101, Count: 1}) { + t.Fatalf("unexpected pair(1): %v", pairs[1]) + } +} + +// Ensure a fragment can return top bitmaps that intersect with an input bitmap. +func TestFragment_TopN_Intersect(t *testing.T) { + f := MustOpenFragment("d", "f", 0) + defer f.Close() + + // Create an intersecting input bitmap. + src := pilosa.NewBitmap(1, 2, 3) + + // Set bits on various bitmaps. + f.MustSetBits(100, 1, 10, 11, 12) // one intersection + f.MustSetBits(101, 1, 2, 3, 4) // three intersections + f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections + f.MustSetBits(103, 1000, 1001, 1002) // no intersection + + // Retrieve top bitmaps. + if pairs, err := f.TopN(3, src, "", nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + {Key: 101, Count: 3}, + {Key: 102, Count: 2}, + {Key: 100, Count: 1}, + }) { + t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) + } +} + +// Ensure a fragment can return top bitmaps that have many bits set. +func TestFragment_TopN_Intersect_Large(t *testing.T) { + f := MustOpenFragment("d", "f", 0) + defer f.Close() + + // Create an intersecting input bitmap. + src := pilosa.NewBitmap( + 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, + 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, + ) + + // Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts. + for i := uint64(0); i < 1000; i++ { + for j := uint64(0); j < i; j++ { + f.MustSetBits(i, j) + } + } + + // Retrieve top bitmaps. + if pairs, err := f.TopN(10, src, "", nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + {Key: 999, Count: 19}, + {Key: 998, Count: 18}, + {Key: 997, Count: 17}, + {Key: 996, Count: 16}, + {Key: 995, Count: 15}, + {Key: 994, Count: 14}, + {Key: 993, Count: 13}, + {Key: 992, Count: 12}, + {Key: 991, Count: 11}, + {Key: 990, Count: 10}, + }) { + t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) + } +} + // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { *pilosa.Fragment + BitmapAttrStore *BitmapAttrStore } // NewFragment returns a new instance of Fragment with a temporary path. @@ -110,7 +224,13 @@ func NewFragment(db, frame string, slice uint64) *Fragment { panic(err) } file.Close() - return &Fragment{Fragment: pilosa.NewFragment(file.Name(), db, frame, slice)} + + f := &Fragment{ + Fragment: pilosa.NewFragment(file.Name(), db, frame, slice), + BitmapAttrStore: NewBitmapAttrStore(), + } + f.Fragment.BitmapAttrStore = f.BitmapAttrStore + return f } // MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. @@ -142,16 +262,42 @@ func (f *Fragment) Reopen() error { return nil } -// MustSetBit sets a bit on a bitmap. Panic on error. -func (f *Fragment) MustSetBit(bitmapID, profileID uint64) { - if err := f.SetBit(bitmapID, profileID); err != nil { - panic(err) +// MustSetBits sets bits on a bitmap. Panic on error. +func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { + for _, profileID := range profileIDs { + if err := f.SetBit(bitmapID, profileID); err != nil { + panic(err) + } } } -// MustClearBit clears a bit on a bitmap. Panic on error. -func (f *Fragment) MustClearBit(bitmapID, profileID uint64) { - if err := f.ClearBit(bitmapID, profileID); err != nil { - panic(err) +// MustClearBits clears bits on a bitmap. Panic on error. +func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) { + for _, profileID := range profileIDs { + if err := f.ClearBit(bitmapID, profileID); err != nil { + panic(err) + } } } + +// BitmapAttrStore provides simple storage for attributes. +type BitmapAttrStore struct { + attrs map[uint64]map[string]interface{} +} + +// NewBitmapAttrStore returns a new instance of BitmapAttrStore. +func NewBitmapAttrStore() *BitmapAttrStore { + return &BitmapAttrStore{ + attrs: make(map[uint64]map[string]interface{}), + } +} + +// BitmapAttrs returns the attributes set to a bitmap id. +func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) { + return s.attrs[id], nil +} + +// SetBitmapAttrs assigns a set of attributes to a bitmap id. +func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) { + s.attrs[id] = m +} diff --git a/frame.go b/frame.go index 3ada6a707..f37ce0b3e 100644 --- a/frame.go +++ b/frame.go @@ -109,6 +109,7 @@ func (f *Frame) openFragments() error { if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) } + frag.BitmapAttrStore = f f.fragments[frag.Slice()] = frag } @@ -189,6 +190,7 @@ func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) { if err := frag.Open(); err != nil { return nil, err } + frag.BitmapAttrStore = f f.fragments[slice] = frag return frag, nil diff --git a/handler.go b/handler.go index a57b95f8e..5a05200b9 100644 --- a/handler.go +++ b/handler.go @@ -400,7 +400,7 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { switch result := resp.Result.(type) { case *Bitmap: pb.Bitmap = encodeBitmap(result) - case Pairs: + case []Pair: pb.Pairs = encodePairs(result) case uint64: pb.N = proto.Uint64(result) diff --git a/handler_test.go b/handler_test.go index e54dcd679..04fec304b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -288,7 +288,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { func TestHandler_Query_Pairs_JSON(t *testing.T) { h := NewHandler() h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { - return pilosa.Pairs{ + return []pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, }, nil @@ -307,7 +307,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { func TestHandler_Query_Pairs_Protobuf(t *testing.T) { h := NewHandler() h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { - return pilosa.Pairs{ + return []pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, }, nil diff --git a/pql/ast.go b/pql/ast.go index 0aeaa0abc..07f26e12d 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -111,7 +111,6 @@ func (c *Bitmap) String() string { type ClearBit struct { ID uint64 Frame string - Filter uint64 ProfileID uint64 } @@ -122,9 +121,6 @@ func (c *ClearBit) String() string { if c.Frame != "" { args = append(args, fmt.Sprintf("frame=%s", c.Frame)) } - if c.Filter != 0 { - args = append(args, fmt.Sprintf("filter=%d", c.Filter)) - } if c.ProfileID != 0 { args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID)) } @@ -203,7 +199,6 @@ func (c *Range) String() string { type SetBit struct { ID uint64 Frame string - Filter uint64 ProfileID uint64 } @@ -214,9 +209,6 @@ func (c *SetBit) String() string { if c.Frame != "" { args = append(args, fmt.Sprintf("frame=%s", c.Frame)) } - if c.Filter != 0 { - args = append(args, fmt.Sprintf("filter=%d", c.Filter)) - } if c.ProfileID != 0 { args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID)) } @@ -302,11 +294,48 @@ func (c *SetProfileAttrs) String() string { // TopN represents a TopN() function call. type TopN struct { Frame string - N int + + // Bitmap to use for intersection while computing top results. + // Original bitmap counts are used if no Src is provided. + Src BitmapCall + + // Maximum number of results to return. + N int + + // Field name and values to filter on. + Field string + Filters []interface{} } // String returns the string representation of the call. -func (c *TopN) String() string { panic("FIXME") } +func (c *TopN) String() string { + args := make([]string, 0, 2) + if c.Src != nil { + args = append(args, c.Src.String()) + } + if c.Frame != "" { + args = append(args, fmt.Sprintf("frame=%s", c.Frame)) + } + if c.N > 0 { + args = append(args, fmt.Sprintf("n=%d", c.N)) + } + if c.Field != "" { + args = append(args, fmt.Sprintf("field=%q", c.Field)) + } + if len(c.Filters) > 0 { + filters := make([]string, 0, len(c.Filters)) + for i := range c.Filters { + switch filter := c.Filters[i].(type) { + case string: + filters = append(filters, fmt.Sprintf("%q", filter)) + default: + filters = append(filters, fmt.Sprintf("%v", filter)) + } + } + args = append(args, fmt.Sprintf("[%s]", strings.Join(filters, ","))) + } + return fmt.Sprintf("TopN(%s)", strings.Join(args, ", ")) +} // Union represents a union() function call. type Union struct { diff --git a/pql/ast_test.go b/pql/ast_test.go index e3974013a..208f71dd3 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -17,8 +17,8 @@ func TestBitmap_String(t *testing.T) { // Ensure the ClearBit call can be converted into a string. func TestClearBit_String(t *testing.T) { - s := (&pql.ClearBit{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String() - if s != `ClearBit(id=1, frame=x.n, filter=2, profileID=3)` { + s := (&pql.ClearBit{ID: 1, Frame: "x.n", ProfileID: 3}).String() + if s != `ClearBit(id=1, frame=x.n, profileID=3)` { t.Fatalf("unexpected string: %s", s) } } @@ -77,8 +77,8 @@ func TestRange_String(t *testing.T) { // Ensure the SetBit call can be converted into a string. func TestSetBit_String(t *testing.T) { - s := (&pql.SetBit{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String() - if s != `SetBit(id=1, frame=x.n, filter=2, profileID=3)` { + s := (&pql.SetBit{ID: 1, Frame: "x.n", ProfileID: 3}).String() + if s != `SetBit(id=1, frame=x.n, profileID=3)` { t.Fatalf("unexpected string: %s", s) } } diff --git a/pql/parser.go b/pql/parser.go index 5f30d3280..824467103 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -136,11 +136,7 @@ func (p *Parser) parseClearBitCall() (*ClearBit, error) { if err := decodeString(arg.value, &c.Frame); err != nil { return nil, parseErrorf(pos, "frame: %s", err) } - case 2, "filter": - if err := decodeUint64(arg.value, &c.Filter); err != nil { - return nil, parseErrorf(pos, "filter: %s", err) - } - case 3, "profileID": + case 2, "profileID": if err := decodeUint64(arg.value, &c.ProfileID); err != nil { return nil, parseErrorf(pos, "profileID: %s", err) } @@ -337,11 +333,7 @@ func (p *Parser) parseSetBitCall() (*SetBit, error) { if err := decodeString(arg.value, &c.Frame); err != nil { return nil, parseErrorf(pos, "frame: %s", err) } - case 2, "filter": - if err := decodeUint64(arg.value, &c.Filter); err != nil { - return nil, parseErrorf(pos, "filter: %s", err) - } - case 3, "profileID": + case 2, "profileID": if err := decodeUint64(arg.value, &c.ProfileID); err != nil { return nil, parseErrorf(pos, "profileID: %s", err) } @@ -477,6 +469,15 @@ func (p *Parser) parseTopNCall() (*TopN, error) { // Copy arguments to AST. for _, arg := range args { + if v, ok := arg.value.(BitmapCall); ok { + c.Src = v + continue + } + if v, ok := arg.value.([]interface{}); ok { + c.Filters = v + continue + } + switch arg.key { case 0, "frame": if err := decodeString(arg.value, &c.Frame); err != nil { @@ -486,6 +487,10 @@ func (p *Parser) parseTopNCall() (*TopN, error) { if err := decodeInt(arg.value, &c.N); err != nil { return nil, parseErrorf(pos, "n: %s", err) } + case 2, "field": + if err := decodeString(arg.value, &c.Field); err != nil { + return nil, parseErrorf(pos, "n: %s", err) + } default: return nil, parseErrorf(pos, "invalid TopN() arg: %v", arg.key) } @@ -608,7 +613,11 @@ func (p *Parser) parseArg() (arg, error) { } value = v case LBRACK: - panic("FIXME: parse list of integers") + v, err := p.parseList() + if err != nil { + return arg{}, err + } + value = v default: return arg{}, parseErrorf(pos, "invalid value: %q", lit) } @@ -616,6 +625,43 @@ func (p *Parser) parseArg() (arg, error) { return arg{key: key, value: value}, nil } +// parseListArg parses a list of primitives. This is used by the TopN() filters. +func (p *Parser) parseList() ([]interface{}, error) { + var values []interface{} + for { + // Read next value. + tok, pos, lit := p.scanIgnoreWhitespace() + switch tok { + case IDENT: + if lit == "true" { + values = append(values, true) + } else if lit == "false" { + values = append(values, false) + } else { + values = append(values, lit) + } + case STRING: + values = append(values, lit) + case NUMBER: + v, err := strconv.ParseUint(lit, 10, 64) + if err != nil { + return nil, err + } + values = append(values, v) + default: + return nil, parseErrorf(pos, "invalid list value: %q", lit) + } + + // Expect a comma or closing bracket next. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { + break + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected COMMA, found %q", lit) + } + } + return values, nil +} + // scan returns the next token from the scanner. func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } diff --git a/pql/parser_test.go b/pql/parser_test.go index f0ff1e333..ad4e3a1aa 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -41,14 +41,13 @@ func TestParser_Parse_Bitmap_Array(t *testing.T) { // Ensure the parser can parse a "ClearBit()" function with keyed args. func TestParser_Parse_ClearBit_Key(t *testing.T) { - q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", filter=2, profileID = 3)`) + q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", profileID = 3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ Root: &pql.ClearBit{ ID: 1, Frame: "b.n", - Filter: 2, ProfileID: 3, }, }) { @@ -58,14 +57,13 @@ func TestParser_Parse_ClearBit_Key(t *testing.T) { // Ensure the parser can parse a "ClearBit()" function with array args. func TestParser_Parse_ClearBit_Array(t *testing.T) { - q, err := pql.ParseString(`ClearBit(1, "b.n", 2, 3)`) + q, err := pql.ParseString(`ClearBit(1, "b.n", 3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ Root: &pql.ClearBit{ ID: 1, Frame: "b.n", - Filter: 2, ProfileID: 3, }, }) { @@ -183,14 +181,13 @@ func TestParser_Parse_Range_Array(t *testing.T) { // Ensure the parser can parse a "SetBit()" function with keyed args. func TestParser_Parse_SetBit_Key(t *testing.T) { - q, err := pql.ParseString(`SetBit(id=1, frame="b.n", filter=2, profileID = 3)`) + q, err := pql.ParseString(`SetBit(id=1, frame="b.n", profileID = 3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ Root: &pql.SetBit{ ID: 1, Frame: "b.n", - Filter: 2, ProfileID: 3, }, }) { @@ -200,14 +197,13 @@ func TestParser_Parse_SetBit_Key(t *testing.T) { // Ensure the parser can parse a "SetBit()" function with array args. func TestParser_Parse_SetBit_Array(t *testing.T) { - q, err := pql.ParseString(`SetBit(1, "b.n", 2, 3)`) + q, err := pql.ParseString(`SetBit(1, "b.n", 3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ Root: &pql.SetBit{ ID: 1, Frame: "b.n", - Filter: 2, ProfileID: 3, }, }) { @@ -258,32 +254,46 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) { // Ensure the parser can parse a "TopN()" function with keyed args. func TestParser_Parse_TopN_Key(t *testing.T) { - q, err := pql.ParseString(`TopN(frame="b.n", n=2)`) + q, err := pql.ParseString(`TopN(Bitmap(100), frame="b.n", n=2, field="XXX", [5,10,15])`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ Root: &pql.TopN{ - Frame: "b.n", - N: 2, + Src: &pql.Bitmap{ID: 100}, + Frame: "b.n", + N: 2, + Field: "XXX", + Filters: []interface{}{uint64(5), uint64(10), uint64(15)}, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) } + + if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", [5,10,15])` { + t.Fatalf("unexpected string encoding: %s", s) + } } // Ensure the parser can parse a "TopN()" function with array args. func TestParser_Parse_TopN_Array(t *testing.T) { - q, err := pql.ParseString(`TopN("b.n", 2)`) + q, err := pql.ParseString(`TopN(Bitmap(100), "b.n", 2, "XXX", ["foo",true,false])`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q, &pql.Query{ Root: &pql.TopN{ - Frame: "b.n", - N: 2, + Src: &pql.Bitmap{ID: 100}, + Frame: "b.n", + N: 2, + Field: "XXX", + Filters: []interface{}{"foo", true, false}, }, }) { t.Fatalf("unexpected query: %s", spew.Sdump(q)) } + + if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", ["foo",true,false])` { + t.Fatalf("unexpected string encoding: %s", s) + } } // Ensure the parser can parse a "union()" function.