scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 00:25:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #45 from benbjohnson/topn

Browse source 
TopN()
This commit is contained in:
tgruben 2016-02-10 15:00:41 -06:00
commit 0e9e7f43a5
12 changed files with 626 additions and 281 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
cache.go
View file

@ -1,7 +1,6 @@
package pilosa
import (
"encoding/json"
"io"
"sort"
"time"
@ -13,40 +12,37 @@ import (
// Cache represents a cache for bitmaps.
type Cache interface {
io.WriterTo
io.ReaderFrom
Add(bitmapID, category uint64, bm *Bitmap)
Add(bitmapID uint64, bm *Bitmap)
Get(bitmapID uint64) *Bitmap
Len() int
// Updates the cache, if necessary.
Invalidate()
// Returns a list of all key/count pairs.
Pairs() []Pair
// Returns an ordered list of the top ranked bitmaps.
Top() []BitmapPair
}
// LRUCache represents a least recently used Cache implemenation.
type LRUCache struct {
cache *lru.Cache
keys map[uint64]struct{}
cache *lru.Cache
bitmaps map[uint64]*Bitmap
}
// NewLRUCache returns a new instance of LRUCache.
func NewLRUCache(maxEntries int) *LRUCache {
c := &LRUCache{
cache: lru.New(maxEntries),
keys: make(map[uint64]struct{}),
cache: lru.New(maxEntries),
bitmaps: make(map[uint64]*Bitmap),
}
c.cache.OnEvicted = c.onEvicted
return c
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Add(bitmapID, category uint64, bm *Bitmap) {
func (c *LRUCache) Add(bitmapID uint64, bm *Bitmap) {
c.cache.Add(bitmapID, bm)
c.keys[bitmapID] = struct{}{}
c.bitmaps[bitmapID] = bm
}
// Get returns a bitmap with a given id.
@ -64,53 +60,28 @@ func (c *LRUCache) Len() int { return c.cache.Len() }
// Invalidate is a no-op.
func (c *LRUCache) Invalidate() {}
// Pairs returns all key/count pairs in the cache.
func (c *LRUCache) Pairs() []Pair {
a := make([]Pair, 0, len(c.keys))
for k := range c.keys {
a = append(a, Pair{
Key: k,
Count: c.Get(k).Count(),
// Top returns all bitmaps in the cache.
func (c *LRUCache) Top() []BitmapPair {
a := make([]BitmapPair, 0, len(c.bitmaps))
for id, bm := range c.bitmaps {
a = append(a, BitmapPair{
ID: id,
Bitmap: bm,
})
}
sort.Sort(BitmapPairs(a))
return a
}
// WriteTo writes the cache to w.
func (c *LRUCache) WriteTo(w io.Writer) (n int64, err error) {
// Write keys to slice.
a := make([]uint64, 0, len(c.keys))
for k := range c.keys {
a = append(a, k)
}
// Encode to file as array of keys.
if err := json.NewEncoder(w).Encode(a); err != nil {
return 0, err
}
return 0, nil
}
// ReadFrom read from r into the cache.
func (c *LRUCache) ReadFrom(r io.Reader) (n int64, err error) {
var keys []uint64
if err := json.NewDecoder(r).Decode(&keys); err != nil {
return 0, err
}
panic("FIXME: TODO")
}
func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) {
delete(c.keys, key.(uint64))
}
func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.bitmaps, key.(uint64)) }
// Ensure LRUCache implements Cache.
var _ Cache = &LRUCache{}
// RankCache represents a cache with sorted entries.
type RankCache struct {
entries map[uint64]*Pair
rankings []Pair // cached, ordered list
entries map[uint64]*Bitmap
rankings []BitmapPair // cached, ordered list
updateN int
updateTime time.Time
@ -123,44 +94,33 @@ type RankCache struct {
// NewRankCache returns a new instance of RankCache.
func NewRankCache() *RankCache {
return &RankCache{
entries: make(map[uint64]*Pair),
entries: make(map[uint64]*Bitmap),
}
}
// Get returns a bitmap with a given id.
func (c *RankCache) Add(bitmapID, category uint64, bm *Bitmap) {
func (c *RankCache) Add(bitmapID uint64, bm *Bitmap) {
// Ignore if the bit count on the bitmap is below the threshold.
if bm.Count() < c.ThresholdValue {
return
}
// Add to cache.
c.entries[bitmapID] = &Pair{
Key: bitmapID,
Count: bm.Count(),
bitmap: bm,
category: category,
}
c.entries[bitmapID] = bm
// If size is larger than the threshold then trim it.
if len(c.entries) > c.ThresholdLength {
c.update()
for k, entry := range c.entries {
if entry.bitmap.Count() <= c.ThresholdValue {
delete(c.entries, k)
for id, bm := range c.entries {
if bm.Count() <= c.ThresholdValue {
delete(c.entries, id)
}
}
}
}
// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) *Bitmap {
entry, ok := c.entries[bitmapID]
if !ok {
return nil
}
return entry.bitmap
}
func (c *RankCache) Get(bitmapID uint64) *Bitmap { return c.entries[bitmapID] }
// Len returns the number of items in the cache.
func (c *RankCache) Len() int { return len(c.entries) }
@ -176,21 +136,19 @@ func (c *RankCache) Invalidate() {
// update reorders the entries by rank.
func (c *RankCache) update() {
// Convert cache to a sorted list.
list := make([]Pair, 0, len(c.entries))
for k, item := range c.entries {
list = append(list, Pair{
Key: k,
Count: item.bitmap.Count(),
bitmap: item.bitmap,
category: item.category,
rankings := make([]BitmapPair, 0, len(c.entries))
for id, bm := range c.entries {
rankings = append(rankings, BitmapPair{
ID: id,
Bitmap: bm,
})
}
sort.Sort(Pairs(list))
sort.Sort(BitmapPairs(rankings))
// Store the count of the item at the threshold index.
c.rankings = list
c.rankings = rankings
if len(c.rankings) > c.ThresholdIndex {
c.ThresholdValue = list[c.ThresholdIndex].bitmap.Count()
c.ThresholdValue = rankings[c.ThresholdIndex].Bitmap.Count()
} else {
c.ThresholdValue = 1
}
@ -199,8 +157,8 @@ func (c *RankCache) update() {
c.updateTime, c.updateN = time.Now(), 0
}
// Pairs returns an ordered list of key/count pairs.
func (c *RankCache) Pairs() []Pair { return c.rankings }
// Top returns an ordered list of bitmaps.
func (c *RankCache) Top() []BitmapPair { return c.rankings }
// WriteTo writes the cache to w.
func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
@ -215,12 +173,22 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
// Ensure RankCache implements Cache.
var _ Cache = &RankCache{}
// BitmapPair represents a bitmap with an associated identifier.
type BitmapPair struct {
ID uint64
Bitmap *Bitmap
}
// BitmapPairs is a sortable list of BitmapPair objects.
type BitmapPairs []BitmapPair
func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p BitmapPairs) Len() int { return len(p) }
func (p BitmapPairs) Less(i, j int) bool { return p[i].Bitmap.Count() > p[j].Bitmap.Count() }
type Pair struct {
Key uint64 `json:"key"`
Count uint64 `json:"count"`
bitmap *Bitmap
category uint64
}
func encodePair(p Pair) *internal.Pair {
@ -243,6 +211,27 @@ func (p Pairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Pairs) Len() int { return len(p) }
func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
// Add merges other into p and returns a new slice.
func (p Pairs) Add(other []Pair) []Pair {
// Create lookup of key/counts.
m := make(map[uint64]uint64, len(p))
for _, pair := range p {
m[pair.Key] = pair.Count
}
// Add/merge from other.
for _, pair := range other {
m[pair.Key] += pair.Count
}
// Convert back to slice.
a := make([]Pair, 0, len(m))
for k, v := range m {
a = append(a, Pair{Key: k, Count: v})
}
return a
}
func encodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {

59
executor.go
View file

@ -7,6 +7,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
"sort"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
@ -147,7 +148,63 @@ func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uin
// executeTopN executes a TopN() call.
func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) {
panic("FIXME: calculate top n from each slice")
var results []Pair
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
if node.Host == e.Host {
for _, slice := range nodeSlices {
pairs, err := e.executeTopNSlice(db, c, slice)
if err != nil {
return nil, err
}
results = Pairs(results).Add(pairs)
}
continue
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
if err != nil {
return nil, err
}
results = Pairs(results).Add(res.([]Pair))
}
// Sort final merged results.
sort.Sort(Pairs(results))
// Only keep the top n after sorting.
if len(results) > c.N {
results = results[0:c.N]
}
return results, nil
}
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(db string, c *pql.TopN, slice uint64) ([]Pair, error) {
// Retrieve bitmap used to intersect.
var src *Bitmap
if c.Src != nil {
bm, err := e.executeBitmapCallSlice(db, c.Src, slice)
if err != nil {
return nil, err
}
src = bm
}
// Set default frame.
frame := c.Frame
if frame == "" {
frame = DefaultFrame
}
f := e.Index().Fragment(db, frame, slice)
if f == nil {
return nil, nil
}
return f.TopN(c.N, src, c.Field, c.Filters)
}
// executeDifferenceSlice executes a difference() call for a local slice.

152
executor_test.go
View file

@ -14,8 +14,8 @@ import (
func TestExecutor_Execute_Bitmap(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBit(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
if err := idx.Frame("d", "f").SetBitmapAttrs(10, map[string]interface{}{"foo": "bar", "baz": 123}); err != nil {
t.Fatal(err)
@ -39,10 +39,10 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
func TestExecutor_Execute_Difference(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 3)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
@ -58,13 +58,13 @@ func TestExecutor_Execute_Difference(t *testing.T) {
func TestExecutor_Execute_Intersect(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(11, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
@ -82,12 +82,12 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
func TestExecutor_Execute_Union(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(10, 0)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBit(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBit(11, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
@ -105,9 +105,9 @@ func TestExecutor_Execute_Union(t *testing.T) {
func TestExecutor_Execute_Count(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBit(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil {
@ -162,6 +162,67 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
}
}
// Ensure a TopN() query can be executed.
func TestExecutor_Execute_TopN(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
{Key: 0, Count: 5},
{Key: 10, Count: 2},
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
// Ensure a TopN() query with a source bitmap can be executed.
func TestExecutor_Execute_TopN_Src(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2)
// Create an intersecting bitmap.
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
{Key: 20, Count: 3},
{Key: 10, Count: 2},
{Key: 0, Count: 1},
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
// Ensure a remote query can return a bitmap.
func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c := NewCluster(2)
@ -194,7 +255,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// The local node owns slice 1.
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil {
@ -225,8 +286,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create local executor data. The local node owns slice 1.
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBit(10, (1*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil {
@ -276,6 +337,51 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
}
// Ensure a remote query can return a top-n query.
func TestExecutor_Execute_Remote_TopN(t *testing.T) {
c := NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `TopN(frame=f, n=3)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) {
t.Fatalf("unexpected slices: %+v", slices)
}
// Return pair counts.
return []pilosa.Pair{
{Key: 0, Count: 5},
{Key: 10, Count: 2},
{Key: 30, Count: 2},
}, nil
}
// Create local executor data on slice 1 & 3.
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(30, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []pilosa.Pair{
{Key: 0, Count: 5},
{Key: 30, Count: 4},
{Key: 10, Count: 2},
}) {
t.Fatalf("unexpected results: %s", spew.Sdump(res))
}
}
// Executor represents a test wrapper for pilosa.Executor.
type Executor struct {
*pilosa.Executor

208
fragment.go
View file

@ -20,6 +20,8 @@ const SliceWidth = 65536
// SnapshotExt is the file extension used for an in-process snapshot.
const SnapshotExt = ".snapshotting"
const MinThreshold = 10
// Fragment represents the intersection of a frame and slice in a database.
type Fragment struct {
mu sync.Mutex
@ -37,6 +39,12 @@ type Fragment struct {
// Bitmap cache.
cache Cache
// Bitmap attribute storage.
// Typically this is the parent frame unless overridden for testing.
BitmapAttrStore interface {
BitmapAttrs(id uint64) (map[string]interface{}, error)
}
}
// NewFragment returns a new instance of Fragment.
@ -207,41 +215,11 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
})
// Add to the cache.
f.cache.Add(bitmapID, 0 /*filter*/, bm)
f.cache.Add(bitmapID, bm)
return bm
}
func (f *Fragment) TopNAll(n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
f.cache.Invalidate()
// Create a set of categories.
m := make(map[uint64]struct{})
for _, v := range categories {
m[v] = struct{}{}
}
// Iterate over rankings and add to results until we have enough.
var results []Pair
for _, pair := range f.cache.Pairs() {
// Skip if categories are specified but category is not found.
if _, ok := m[pair.category]; (len(categories) > 0 && !ok) || pair.Count <= 0 {
continue
}
// Append pair.
results = append(results, pair)
// Exit when we have enough pairs.
if len(results) >= n {
break
}
}
return results
}
// SetBit sets a bit for a given profile & bitmap within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) SetBit(bitmapID, profileID uint64) error {
@ -302,123 +280,105 @@ func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) {
return (bitmapID * SliceWidth) + (profileID % SliceWidth), nil
}
func (f *Fragment) TopN(src *Bitmap, n int, categories []uint64) []Pair {
// TopN returns the top n bitmaps from the fragment.
// If src is specified then only bitmaps which intersect src are returned.
// If fieldValues exist then the bitmap attribute specified by field is matched.
func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interface{}) ([]Pair, error) {
// Resort cache, if needed, and retrieve the top bitmaps.
f.mu.Lock()
defer f.mu.Unlock()
// Resort rank, if necessary.
f.cache.Invalidate()
pairs := f.cache.Top()
f.mu.Unlock()
// Create a set of categories.
set := make(map[uint64]struct{})
for _, v := range categories {
set[v] = struct{}{}
// Create a fast lookup of filter values.
var filters map[interface{}]struct{}
if len(fieldValues) > 0 {
filters = make(map[interface{}]struct{})
for _, v := range fieldValues {
filters[v] = struct{}{}
}
}
var results []Pair
var x int
breakout := 1000
// Iterate over rankings and add to results until we have enough.
results := make([]Pair, 0, n)
for _, pair := range pairs {
bitmapID, bm := pair.ID, pair.Bitmap
// Iterate over rankings.
rankings := f.cache.Pairs()
for i, pair := range rankings {
// Skip if category not found.
if len(set) > 0 {
if _, ok := set[pair.category]; !ok {
// Ignore empty bitmaps.
if bm.Count() <= 0 {
continue
}
// Apply filter, if set.
if filters != nil {
attr, err := f.BitmapAttrStore.BitmapAttrs(bitmapID)
if err != nil {
return nil, err
} else if attr == nil {
continue
} else if attrValue := attr[field]; attrValue == nil {
continue
} else if _, ok := filters[attrValue]; !ok {
continue
}
}
// Only append if there are intersecting bits with source bitmap.
bc := src.IntersectionCount(pair.bitmap)
if bc > 0 {
results = append(results, Pair{
Key: pair.Key,
Count: bc,
category: pair.category,
})
}
x = i
// Exit when we have enough.
if len(results) > n {
break
}
}
// Sort results by ranking.
sort.Sort(Pairs(results))
if len(results) < n {
return results
}
end := len(results) - 1
o := results[end]
threshold := o.Count
if threshold <= 10 {
return results
}
results = append(results, o)
for i := x + 1; i < len(rankings); i++ {
o = rankings[i]
if len(set) > 0 {
if _, ok := set[o.category]; !ok {
// The initial n pairs should simply be added to the results.
if len(results) < n {
// Calculate count and append.
count := bm.Count()
if src != nil {
count = src.IntersectionCount(bm)
}
if count == 0 {
continue
}
}
results = append(results, Pair{Key: bitmapID, Count: count})
// Need something to do with the size of initial bitmap
if len(results) > breakout || o.Count < threshold {
break
}
bc := src.IntersectionCount(o.bitmap)
if bc > threshold {
if results[end-1].Count > bc {
results[end] = Pair{Key: o.Key, Count: bc, category: o.category}
threshold = bc
} else {
results[end+1] = Pair{Key: o.Key, Count: bc, category: o.category}
// If we reach the requested number of pairs and we are not computing
// intersections then simply exit. If we are intersecting then sort
// and then only keep pairs that are higher than the lowest count.
if len(results) == n {
if src == nil {
break
}
sort.Sort(Pairs(results))
threshold = results[end].Count
}
}
}
return results[:end]
}
/*
func (f *Fragment) TopFill(args FillArgs) ([]Pair, error) {
result := make([]Pair, 0)
for _, id := range args.Bitmaps {
if _, ok := f.cache.Get(id); !ok {
continue
}
if args.Handle == 0 {
if bm := f.Bitmap(id); bm != nil && bm.Count() > 0 {
result = append(result, Pair{Key: id, Count: bm.Count()})
}
continue
}
res := f.Intersect([]uint64{args.Handle, id})
if res == nil {
// Retrieve the lowest count we have.
// If it's too low then don't try finding anymore pairs.
threshold := results[len(results)-1].Count
if threshold < MinThreshold {
break
}
// If the bitmap doesn't have enough bits set before the intersection
// then we can assume that any remaing bitmaps also have a count too low.
if bm.Count() < threshold {
break
}
// Calculate the intersecting bit count and skip if it's below our
// last bitmap in our current result set.
count := src.IntersectionCount(bm)
if count < threshold {
continue
}
if bc := res.BitCount(); bc > 0 {
result = append(result, Pair{Key: id, Count: bc})
// Swap out the last pair for this new count.
results[len(results)-1] = Pair{Key: bitmapID, Count: count}
// If it's count is also higher than the second to last item then resort.
if len(results) >= 2 && count > results[len(results)-2].Count {
sort.Sort(Pairs(results))
}
}
return result, nil
sort.Sort(Pairs(results))
return results, nil
}
*/
func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap {
f.mu.Lock()

164
fragment_test.go
View file

@ -3,8 +3,10 @@ package pilosa_test
import (
"io/ioutil"
"os"
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
)
@ -98,9 +100,121 @@ func TestFragment_Snapshot(t *testing.T) {
}
}
// Ensure a fragment can return the top n results.
func TestFragment_TopN(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
defer f.Close()
// Set bits on the bitmaps 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
f.MustSetBits(101, 1)
f.MustSetBits(102, 1, 2)
// Retrieve top bitmaps.
if pairs, err := f.TopN(2, nil, "", nil); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
t.Fatalf("unexpected count: %d", len(pairs))
} else if pairs[0] != (pilosa.Pair{Key: 100, Count: 3}) {
t.Fatalf("unexpected pair(0): %v", pairs[0])
} else if pairs[1] != (pilosa.Pair{Key: 102, Count: 2}) {
t.Fatalf("unexpected pair(1): %v", pairs[1])
}
}
// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps.
func TestFragment_TopN_Filter(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
defer f.Close()
// Set bits on the bitmaps 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
f.MustSetBits(101, 1)
f.MustSetBits(102, 1, 2)
// Assign attributes.
f.BitmapAttrStore.SetBitmapAttrs(101, map[string]interface{}{"x": 10})
f.BitmapAttrStore.SetBitmapAttrs(102, map[string]interface{}{"x": 20})
// Retrieve top bitmaps.
if pairs, err := f.TopN(2, nil, "x", []interface{}{10, 15, 20}); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
t.Fatalf("unexpected count: %d", len(pairs))
} else if pairs[0] != (pilosa.Pair{Key: 102, Count: 2}) {
t.Fatalf("unexpected pair(0): %v", pairs[0])
} else if pairs[1] != (pilosa.Pair{Key: 101, Count: 1}) {
t.Fatalf("unexpected pair(1): %v", pairs[1])
}
}
// Ensure a fragment can return top bitmaps that intersect with an input bitmap.
func TestFragment_TopN_Intersect(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
defer f.Close()
// Create an intersecting input bitmap.
src := pilosa.NewBitmap(1, 2, 3)
// Set bits on various bitmaps.
f.MustSetBits(100, 1, 10, 11, 12) // one intersection
f.MustSetBits(101, 1, 2, 3, 4) // three intersections
f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections
f.MustSetBits(103, 1000, 1001, 1002) // no intersection
// Retrieve top bitmaps.
if pairs, err := f.TopN(3, src, "", nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
{Key: 101, Count: 3},
{Key: 102, Count: 2},
{Key: 100, Count: 1},
}) {
t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
}
}
// Ensure a fragment can return top bitmaps that have many bits set.
func TestFragment_TopN_Intersect_Large(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
defer f.Close()
// Create an intersecting input bitmap.
src := pilosa.NewBitmap(
980, 981, 982, 983, 984, 985, 986, 987, 988, 989,
990, 991, 992, 993, 994, 995, 996, 997, 998, 999,
)
// Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts.
for i := uint64(0); i < 1000; i++ {
for j := uint64(0); j < i; j++ {
f.MustSetBits(i, j)
}
}
// Retrieve top bitmaps.
if pairs, err := f.TopN(10, src, "", nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
{Key: 999, Count: 19},
{Key: 998, Count: 18},
{Key: 997, Count: 17},
{Key: 996, Count: 16},
{Key: 995, Count: 15},
{Key: 994, Count: 14},
{Key: 993, Count: 13},
{Key: 992, Count: 12},
{Key: 991, Count: 11},
{Key: 990, Count: 10},
}) {
t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
}
}
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {
*pilosa.Fragment
BitmapAttrStore *BitmapAttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
@ -110,7 +224,13 @@ func NewFragment(db, frame string, slice uint64) *Fragment {
panic(err)
}
file.Close()
return &Fragment{Fragment: pilosa.NewFragment(file.Name(), db, frame, slice)}
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), db, frame, slice),
BitmapAttrStore: NewBitmapAttrStore(),
}
f.Fragment.BitmapAttrStore = f.BitmapAttrStore
return f
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
@ -142,16 +262,42 @@ func (f *Fragment) Reopen() error {
return nil
}
// MustSetBit sets a bit on a bitmap. Panic on error.
func (f *Fragment) MustSetBit(bitmapID, profileID uint64) {
if err := f.SetBit(bitmapID, profileID); err != nil {
panic(err)
// MustSetBits sets bits on a bitmap. Panic on error.
func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if err := f.SetBit(bitmapID, profileID); err != nil {
panic(err)
}
}
}
// MustClearBit clears a bit on a bitmap. Panic on error.
func (f *Fragment) MustClearBit(bitmapID, profileID uint64) {
if err := f.ClearBit(bitmapID, profileID); err != nil {
panic(err)
// MustClearBits clears bits on a bitmap. Panic on error.
func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if err := f.ClearBit(bitmapID, profileID); err != nil {
panic(err)
}
}
}
// BitmapAttrStore provides simple storage for attributes.
type BitmapAttrStore struct {
attrs map[uint64]map[string]interface{}
}
// NewBitmapAttrStore returns a new instance of BitmapAttrStore.
func NewBitmapAttrStore() *BitmapAttrStore {
return &BitmapAttrStore{
attrs: make(map[uint64]map[string]interface{}),
}
}
// BitmapAttrs returns the attributes set to a bitmap id.
func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) {
return s.attrs[id], nil
}
// SetBitmapAttrs assigns a set of attributes to a bitmap id.
func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}

2
frame.go
View file

@ -109,6 +109,7 @@ func (f *Frame) openFragments() error {
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
}
frag.BitmapAttrStore = f
f.fragments[frag.Slice()] = frag
}
@ -189,6 +190,7 @@ func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
if err := frag.Open(); err != nil {
return nil, err
}
frag.BitmapAttrStore = f
f.fragments[slice] = frag
return frag, nil

2
handler.go
View file

@ -400,7 +400,7 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
switch result := resp.Result.(type) {
case *Bitmap:
pb.Bitmap = encodeBitmap(result)
case Pairs:
case []Pair:
pb.Pairs = encodePairs(result)
case uint64:
pb.N = proto.Uint64(result)

4
handler_test.go
View file

@ -288,7 +288,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return pilosa.Pairs{
return []pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
@ -307,7 +307,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return pilosa.Pairs{
return []pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil

49
pql/ast.go
View file

@ -111,7 +111,6 @@ func (c *Bitmap) String() string {
type ClearBit struct {
ID uint64
Frame string
Filter uint64
ProfileID uint64
}
@ -122,9 +121,6 @@ func (c *ClearBit) String() string {
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.Filter != 0 {
args = append(args, fmt.Sprintf("filter=%d", c.Filter))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID))
}
@ -203,7 +199,6 @@ func (c *Range) String() string {
type SetBit struct {
ID uint64
Frame string
Filter uint64
ProfileID uint64
}
@ -214,9 +209,6 @@ func (c *SetBit) String() string {
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.Filter != 0 {
args = append(args, fmt.Sprintf("filter=%d", c.Filter))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profileID=%d", c.ProfileID))
}
@ -302,11 +294,48 @@ func (c *SetProfileAttrs) String() string {
// TopN represents a TopN() function call.
type TopN struct {
Frame string
N int
// Bitmap to use for intersection while computing top results.
// Original bitmap counts are used if no Src is provided.
Src BitmapCall
// Maximum number of results to return.
N int
// Field name and values to filter on.
Field string
Filters []interface{}
}
// String returns the string representation of the call.
func (c *TopN) String() string { panic("FIXME") }
func (c *TopN) String() string {
args := make([]string, 0, 2)
if c.Src != nil {
args = append(args, c.Src.String())
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.N > 0 {
args = append(args, fmt.Sprintf("n=%d", c.N))
}
if c.Field != "" {
args = append(args, fmt.Sprintf("field=%q", c.Field))
}
if len(c.Filters) > 0 {
filters := make([]string, 0, len(c.Filters))
for i := range c.Filters {
switch filter := c.Filters[i].(type) {
case string:
filters = append(filters, fmt.Sprintf("%q", filter))
default:
filters = append(filters, fmt.Sprintf("%v", filter))
}
}
args = append(args, fmt.Sprintf("[%s]", strings.Join(filters, ",")))
}
return fmt.Sprintf("TopN(%s)", strings.Join(args, ", "))
}
// Union represents a union() function call.
type Union struct {

8
pql/ast_test.go
View file

@ -17,8 +17,8 @@ func TestBitmap_String(t *testing.T) {
// Ensure the ClearBit call can be converted into a string.
func TestClearBit_String(t *testing.T) {
s := (&pql.ClearBit{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
if s != `ClearBit(id=1, frame=x.n, filter=2, profileID=3)` {
s := (&pql.ClearBit{ID: 1, Frame: "x.n", ProfileID: 3}).String()
if s != `ClearBit(id=1, frame=x.n, profileID=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
@ -77,8 +77,8 @@ func TestRange_String(t *testing.T) {
// Ensure the SetBit call can be converted into a string.
func TestSetBit_String(t *testing.T) {
s := (&pql.SetBit{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
if s != `SetBit(id=1, frame=x.n, filter=2, profileID=3)` {
s := (&pql.SetBit{ID: 1, Frame: "x.n", ProfileID: 3}).String()
if s != `SetBit(id=1, frame=x.n, profileID=3)` {
t.Fatalf("unexpected string: %s", s)
}
}

68
pql/parser.go
View file

@ -136,11 +136,7 @@ func (p *Parser) parseClearBitCall() (*ClearBit, error) {
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "filter":
if err := decodeUint64(arg.value, &c.Filter); err != nil {
return nil, parseErrorf(pos, "filter: %s", err)
}
case 3, "profileID":
case 2, "profileID":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profileID: %s", err)
}
@ -337,11 +333,7 @@ func (p *Parser) parseSetBitCall() (*SetBit, error) {
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "filter":
if err := decodeUint64(arg.value, &c.Filter); err != nil {
return nil, parseErrorf(pos, "filter: %s", err)
}
case 3, "profileID":
case 2, "profileID":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profileID: %s", err)
}
@ -477,6 +469,15 @@ func (p *Parser) parseTopNCall() (*TopN, error) {
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Src = v
continue
}
if v, ok := arg.value.([]interface{}); ok {
c.Filters = v
continue
}
switch arg.key {
case 0, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
@ -486,6 +487,10 @@ func (p *Parser) parseTopNCall() (*TopN, error) {
if err := decodeInt(arg.value, &c.N); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
case 2, "field":
if err := decodeString(arg.value, &c.Field); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid TopN() arg: %v", arg.key)
}
@ -608,7 +613,11 @@ func (p *Parser) parseArg() (arg, error) {
}
value = v
case LBRACK:
panic("FIXME: parse list of integers")
v, err := p.parseList()
if err != nil {
return arg{}, err
}
value = v
default:
return arg{}, parseErrorf(pos, "invalid value: %q", lit)
}
@ -616,6 +625,43 @@ func (p *Parser) parseArg() (arg, error) {
return arg{key: key, value: value}, nil
}
// parseListArg parses a list of primitives. This is used by the TopN() filters.
func (p *Parser) parseList() ([]interface{}, error) {
var values []interface{}
for {
// Read next value.
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
values = append(values, true)
} else if lit == "false" {
values = append(values, false)
} else {
values = append(values, lit)
}
case STRING:
values = append(values, lit)
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return nil, err
}
values = append(values, v)
default:
return nil, parseErrorf(pos, "invalid list value: %q", lit)
}
// Expect a comma or closing bracket next.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
}
}
return values, nil
}
// scan returns the next token from the scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }

38
pql/parser_test.go
View file

@ -41,14 +41,13 @@ func TestParser_Parse_Bitmap_Array(t *testing.T) {
// Ensure the parser can parse a "ClearBit()" function with keyed args.
func TestParser_Parse_ClearBit_Key(t *testing.T) {
q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", filter=2, profileID = 3)`)
q, err := pql.ParseString(`ClearBit(id=1, frame="b.n", profileID = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.ClearBit{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
@ -58,14 +57,13 @@ func TestParser_Parse_ClearBit_Key(t *testing.T) {
// Ensure the parser can parse a "ClearBit()" function with array args.
func TestParser_Parse_ClearBit_Array(t *testing.T) {
q, err := pql.ParseString(`ClearBit(1, "b.n", 2, 3)`)
q, err := pql.ParseString(`ClearBit(1, "b.n", 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.ClearBit{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
@ -183,14 +181,13 @@ func TestParser_Parse_Range_Array(t *testing.T) {
// Ensure the parser can parse a "SetBit()" function with keyed args.
func TestParser_Parse_SetBit_Key(t *testing.T) {
q, err := pql.ParseString(`SetBit(id=1, frame="b.n", filter=2, profileID = 3)`)
q, err := pql.ParseString(`SetBit(id=1, frame="b.n", profileID = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBit{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
@ -200,14 +197,13 @@ func TestParser_Parse_SetBit_Key(t *testing.T) {
// Ensure the parser can parse a "SetBit()" function with array args.
func TestParser_Parse_SetBit_Array(t *testing.T) {
q, err := pql.ParseString(`SetBit(1, "b.n", 2, 3)`)
q, err := pql.ParseString(`SetBit(1, "b.n", 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.SetBit{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
@ -258,32 +254,46 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) {
// Ensure the parser can parse a "TopN()" function with keyed args.
func TestParser_Parse_TopN_Key(t *testing.T) {
q, err := pql.ParseString(`TopN(frame="b.n", n=2)`)
q, err := pql.ParseString(`TopN(Bitmap(100), frame="b.n", n=2, field="XXX", [5,10,15])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Frame: "b.n",
N: 2,
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{uint64(5), uint64(10), uint64(15)},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", [5,10,15])` {
t.Fatalf("unexpected string encoding: %s", s)
}
}
// Ensure the parser can parse a "TopN()" function with array args.
func TestParser_Parse_TopN_Array(t *testing.T) {
q, err := pql.ParseString(`TopN("b.n", 2)`)
q, err := pql.ParseString(`TopN(Bitmap(100), "b.n", 2, "XXX", ["foo",true,false])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Frame: "b.n",
N: 2,
Src: &pql.Bitmap{ID: 100},
Frame: "b.n",
N: 2,
Field: "XXX",
Filters: []interface{}{"foo", true, false},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
if s := q.String(); s != `TopN(Bitmap(id=100), frame=b.n, n=2, field="XXX", ["foo",true,false])` {
t.Fatalf("unexpected string encoding: %s", s)
}
}
// Ensure the parser can parse a "union()" function.