diff --git a/core/batch.go b/core/batch.go index e5d1a1571..ec4b58a07 100644 --- a/core/batch.go +++ b/core/batch.go @@ -14,7 +14,7 @@ type BatchRequest struct { Fragment_id SUUID Bitmap_id uint64 Compressed_bitmap string - Filter int + Filter uint64 } type BatchResponse struct { @@ -33,7 +33,7 @@ func init() { gob.Register(BatchResponse{}) } -func (self *Service) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter int) error { +func (self *Service) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter uint64) error { //determine the fragment_id from database/frame/slice database := self.Cluster.GetOrCreateDatabase(database_name) oslice := database.GetOrCreateSlice(slice) diff --git a/core/http.go b/core/http.go index adce869bb..3ed1d1b6e 100644 --- a/core/http.go +++ b/core/http.go @@ -124,7 +124,7 @@ func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) { return } - results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice), int(filter)) + results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice), uint64(filter)) encoder := json.NewEncoder(w) err = encoder.Encode(results) diff --git a/core/query.go b/core/query.go index 3112df6ad..d2c09fb12 100644 --- a/core/query.go +++ b/core/query.go @@ -33,9 +33,10 @@ func (self *Service) CountQueryStepHandler(msg *db.Message) { func (self *Service) TopNQueryStepHandler(msg *db.Message) { //spew.Dump("TOP-N QUERYSTEP") qs := msg.Data.(query.TopNQueryStep) + spew.Dump(qs) //TRAVIS SEE HERE //need categories in qs I just added so it would compile - var categoryleaves []int + var categoryleaves []uint64 input := qs.Input value, _ := self.Hold.Get(input, 10) var bh index.BitmapHandle @@ -45,12 +46,22 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) { case []byte: bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val) } + + categoryleaves = qs.Filters + + /* + spew.Dump(bh, qs.N*2, categoryleaves) + result_message := db.Message{Data: "foobar"} + self.Transport.Send(&result_message, qs.Destination.ProcessId) + */ + topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, categoryleaves) if err != nil { spew.Dump(err) } result_message := db.Message{Data: query.TopNQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: topn}}} self.Transport.Send(&result_message, qs.Destination.ProcessId) + } func (self *Service) UnionQueryStepHandler(msg *db.Message) { diff --git a/db/topology.go b/db/topology.go index 5da89510e..0c6c74f29 100644 --- a/db/topology.go +++ b/db/topology.go @@ -413,5 +413,5 @@ func (d *Database) GetSliceForProfile(profile_id uint64) (*Slice, error) { type Bitmap struct { Id uint64 FrameType string - Filter int + Filter uint64 } diff --git a/index/bitmap.go b/index/bitmap.go index 4a10a8654..257caa7e8 100644 --- a/index/bitmap.go +++ b/index/bitmap.go @@ -31,25 +31,25 @@ func init() { // type IntSet struct { - set map[int]bool + set map[uint64]bool } func NewIntSet() *IntSet { - return &IntSet{make(map[int]bool)} + return &IntSet{make(map[uint64]bool)} } -func (set *IntSet) Add(i int) bool { +func (set *IntSet) Add(i uint64) bool { _, found := set.set[i] set.set[i] = true return !found //False if it existed already } -func (set *IntSet) Contains(i int) bool { +func (set *IntSet) Contains(i uint64) bool { _, found := set.set[i] return found //true if it existed already } -func (set *IntSet) Remove(i int) { +func (set *IntSet) Remove(i uint64) { delete(set.set, i) } diff --git a/index/brand.go b/index/brand.go index f9ace80be..9173a976b 100644 --- a/index/brand.go +++ b/index/brand.go @@ -17,7 +17,7 @@ type Pair struct { type Rank struct { *Pair bitmap IBitmap - category int + category uint64 } type RankList []*Rank @@ -72,7 +72,7 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap { return b } -func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category int) { +func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category uint64) { if bm.Count() >= self.threshold_value { self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category} if len(self.bitmap_cache) > self.threshold_length { @@ -89,7 +89,7 @@ func (self *Brand) trim() { } -func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter int) bool { +func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool { bm := self.Get(bitmap_id) change, chunk, address := SetBit(bm, bit_pos) if change { @@ -165,12 +165,12 @@ func (self *Brand) Stats() interface{} { "skip": self.skip} return stats } -func (self *Brand) Store(bitmap_id uint64, bm IBitmap, filter int) { +func (self *Brand) Store(bitmap_id uint64, bm IBitmap, filter uint64) { self.storage.Store(int64(bitmap_id), self.db, self.frame, self.slice, filter, bm.(*Bitmap)) self.cache_it(bm, bitmap_id, filter) } -func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []int) []Pair { +func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []uint64) []Pair { self.rank_counter = 0 self.Rank() // TODO: TERRIBLE REMOVE THIS ASAP is := new(IntSet) diff --git a/index/commands.go b/index/commands.go index cbea5d43c..0ddcb0d2b 100644 --- a/index/commands.go +++ b/index/commands.go @@ -99,10 +99,10 @@ type CmdSetBit struct { *Responder bitmap_id uint64 bit_pos uint64 - filter int + filter uint64 } -func NewSetBit(bitmap_id uint64, bit_pos uint64, filter int) *CmdSetBit { +func NewSetBit(bitmap_id uint64, bit_pos uint64, filter uint64) *CmdSetBit { result := &CmdSetBit{NewResponder("SetBit"), bitmap_id, bit_pos, filter} return result } @@ -184,10 +184,10 @@ type CmdTopN struct { *Responder bitmap BitmapHandle n int - categories []int + categories []uint64 } -func NewTopN(b BitmapHandle, n int, categories []int) *CmdTopN { +func NewTopN(b BitmapHandle, n int, categories []uint64) *CmdTopN { return &CmdTopN{NewResponder("TopN"), b, n, categories} } func (self *CmdTopN) Execute(f *Fragment) Calculation { @@ -209,10 +209,10 @@ type CmdLoader struct { *Responder bitmap_id uint64 compressed_bitmap string - filter int + filter uint64 } -func NewLoader(bitmap_id uint64, compressed_bitmap string, filter int) *CmdLoader { +func NewLoader(bitmap_id uint64, compressed_bitmap string, filter uint64) *CmdLoader { return &CmdLoader{NewResponder("Loader"), bitmap_id, compressed_bitmap, filter} } func (self *CmdLoader) Execute(f *Fragment) Calculation { diff --git a/index/fragment_container.go b/index/fragment_container.go index ef886bf7e..fe0a00ca8 100644 --- a/index/fragment_container.go +++ b/index/fragment_container.go @@ -59,7 +59,7 @@ func (self *FragmentContainer) Shutdown() { log.Println("Container Shutdown Complete") } -func (self *FragmentContainer) LoadBitmap(frag_id SUUID, bitmap_id uint64, compressed_bitmap string, filter int) { +func (self *FragmentContainer) LoadBitmap(frag_id SUUID, bitmap_id uint64, compressed_bitmap string, filter uint64) { if fragment, found := self.GetFragment(frag_id); found { request := NewLoader(bitmap_id, compressed_bitmap, filter) fragment.requestChan <- request @@ -120,7 +120,7 @@ func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandl return 0, errors.New("Invalid Bitmap Handle") } -func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []int) ([]Pair, error) { +func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewTopN(bh, n, categories) fragment.requestChan <- request @@ -165,7 +165,7 @@ func (self *FragmentContainer) FromBytes(frag_id SUUID, bytes []byte) (BitmapHan return 0, errors.New("Invalid Bitmap Handle") } -func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64, category int) (bool, error) { +func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) { if fragment, found := self.GetFragment(frag_id); found { request := NewSetBit(bitmap_id, pos, category) fragment.requestChan <- request @@ -194,10 +194,10 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i type Pilosa interface { Get(id uint64) IBitmap - SetBit(id uint64, bit_pos uint64, filter int) bool - TopN(b IBitmap, n int, categories []int) []Pair + SetBit(id uint64, bit_pos uint64, filter uint64) bool + TopN(b IBitmap, n int, categories []uint64) []Pair Clear() bool - Store(bitmap_id uint64, bm IBitmap, filter int) + Store(bitmap_id uint64, bm IBitmap, filter uint64) Stats() interface{} Persist() error Load(requestChan chan Command, fragment *Fragment) @@ -261,7 +261,7 @@ func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) { return bm.(IBitmap), ok } -func (self *Fragment) TopN(bitmap BitmapHandle, n int, categories []int) []Pair { +func (self *Fragment) TopN(bitmap BitmapHandle, n int, categories []uint64) []Pair { bm, ok := self.cache.Get(bitmap) if ok { diff --git a/index/general.go b/index/general.go index e1892a9cf..41c70e8dc 100644 --- a/index/general.go +++ b/index/general.go @@ -47,7 +47,7 @@ func (self *General) Get(bitmap_id uint64) IBitmap { self.keys[bitmap_id] = 0 return bm.(*Bitmap) } -func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter int) bool { +func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool { bm := self.Get(bitmap_id) change, chunk, address := SetBit(bm, bit_pos) if change { @@ -58,11 +58,11 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter int) bool { } return change } -func (self *General) TopN(b IBitmap, n int, categories []int) []Pair { +func (self *General) TopN(b IBitmap, n int, categories []uint64) []Pair { return nil } -func (self *General) Store(bitmap_id uint64, bm IBitmap, filter int) { +func (self *General) Store(bitmap_id uint64, bm IBitmap, filter uint64) { //oldbm:=self.Get(bitmap_id) //nbm = Union(oldbm, bm) self.storage.Store(int64(bitmap_id), self.db, self.frame, self.slice, filter, bm.(*Bitmap)) diff --git a/index/storage.go b/index/storage.go index 5ed4cefaf..4f1fcc75c 100644 --- a/index/storage.go +++ b/index/storage.go @@ -1,7 +1,7 @@ package index type Storage interface { - Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, int) - Store(id int64, db string, frame string, slice int, filter int, bitmap *Bitmap) error - StoreBlock(id int64, db string, frame string, slice int, filter int, chunk int64, block_index int32, block int64) error + Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) + Store(id int64, db string, frame string, slice int, filter uint64, bitmap *Bitmap) error + StoreBlock(id int64, db string, frame string, slice int, filter uint64, chunk int64, block_index int32, block int64) error } diff --git a/index/storage_cass.go b/index/storage_cass.go index 2d2ac7bbb..347afc494 100644 --- a/index/storage_cass.go +++ b/index/storage_cass.go @@ -15,10 +15,10 @@ type CassandraStorage struct { func BuildSchema() { /* - "CREATE KEYSPACE IF NOT EXISTS hotbox WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1" - create keyspace if not exists hotbox with replication = { 'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true; - CREATE TABLE IF NOT EXISTS bitmap ( bitmap_id bigint, db varchar, frame varchar, slice int, filter int, ChunkKey bigint, BlockIndex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame,slice),ChunkKey,BlockIndex) ) - " + "CREATE KEYSPACE IF NOT EXISTS hotbox WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1" + create keyspace if not exists hotbox with replication = { 'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true; + CREATE TABLE IF NOT EXISTS bitmap ( bitmap_id bigint, db varchar, frame varchar, slice int, filter int, ChunkKey bigint, BlockIndex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame,slice),ChunkKey,BlockIndex) ) + " */ } @@ -42,7 +42,7 @@ func NewCassStorage(host, keyspace string) Storage { return obj } -func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, int) { +func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { var dumb = COUNTERMASK last_key := int64(dumb) marker := int64(dumb) @@ -53,7 +53,7 @@ func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slic chunk_key, block int64 block_index uint32 s8 uint8 - filter int + filter uint64 ) log.Println("FETCHING ", bitmap_id, db, slice) @@ -80,7 +80,7 @@ func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slic return bitmap, filter } -func (c *CassandraStorage) Store(id int64, db string, frame string, slice int, filter int, bitmap *Bitmap) error { +func (c *CassandraStorage) Store(id int64, db string, frame string, slice int, filter uint64, bitmap *Bitmap) error { for i := bitmap.Min(); !i.Limit(); i = i.Next() { var chunk = i.Item() for idx, block := range chunk.Value.Block { @@ -100,7 +100,7 @@ func (c *CassandraStorage) Store(id int64, db string, frame string, slice int, f return nil } -func (c *CassandraStorage) StoreBlock(id int64, db string, frame string, slice int, filter int, chunk int64, block_index int32, block int64) error { +func (c *CassandraStorage) StoreBlock(id int64, db string, frame string, slice int, filter uint64, chunk int64, block_index int32, block int64) error { if err := c.db.Query(` INSERT INTO bitmap ( bitmap_id, db, frame, slice , filter, ChunkKey, BlockIndex, block) VALUES (?,?,?,?,?,?,?,?);`, id, db, frame, slice, filter, chunk, block_index, block).Exec(); err != nil { diff --git a/index/storage_mem.go b/index/storage_mem.go index bd8bf31a7..ed1073ebb 100644 --- a/index/storage_mem.go +++ b/index/storage_mem.go @@ -16,7 +16,7 @@ func NewMemoryStorage() Storage { return obj } -func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, int) { +func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { // log.Println("hello") key := fmt.Sprintf("%d:%s:%s:%d", bitmap_id, db, frame, slice) @@ -28,12 +28,12 @@ func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice i return bitmap, 0 } -func (c *MemoryStorage) Store(bitmap_id int64, db string, frame string, slice int, filter int, bitmap *Bitmap) error { +func (c *MemoryStorage) Store(bitmap_id int64, db string, frame string, slice int, filter uint64, bitmap *Bitmap) error { //only use the cache and throw away everything return nil } -func (c *MemoryStorage) StoreBlock(bitmap_id int64, db string, frame string, slice int, filter int, chunk_key int64, block_index int32, block int64) error { +func (c *MemoryStorage) StoreBlock(bitmap_id int64, db string, frame string, slice int, filter uint64, chunk_key int64, block_index int32, block int64) error { //only use the cache and throw away everything return nil diff --git a/index/storage_test.go b/index/storage_test.go index 3c3ec6a3f..93535fd84 100644 --- a/index/storage_test.go +++ b/index/storage_test.go @@ -46,7 +46,7 @@ func TestStorage(t *testing.T) { SetBit(bm, 1) SetBit(bm, 2) fmt.Println("STORE") - storage.Store(int64(bitmap_id), db, frame, slice, filter, bm.(*Bitmap)) + storage.Store(int64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap)) fmt.Println("FETCH") bm2, _ := storage.Fetch(bitmap_id, db, frame, slice) So(BitCount(bm), ShouldEqual, BitCount(bm2)) diff --git a/query/parser.go b/query/parser.go index 258d8b755..1d2a41d6b 100644 --- a/query/parser.go +++ b/query/parser.go @@ -122,7 +122,7 @@ ArgLoop: case 1: query.Args["frame"] = token.Text case 2: - i, err := strconv.Atoi(token.Text) + i, err := strconv.ParseUint(token.Text, 10, 64) if err != nil { return nil, fmt.Errorf("Expecting integer id! (%v)", err) } diff --git a/query/parser_test.go b/query/parser_test.go index 859ed6ffb..d51b848fa 100644 --- a/query/parser_test.go +++ b/query/parser_test.go @@ -24,7 +24,7 @@ func TestQueryParser(t *testing.T) { So(err, ShouldBeNil) So(query.Operation, ShouldEqual, "set") - So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "filter": int(0), "profile_id": uint64(20)}) + So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "filter": uint64(0), "profile_id": uint64(20)}) }) Convey("Basic nested query parse", t, func() { tokens, err := Lex("union(get(10,general), get(11,brand), get(12))") diff --git a/query/planner.go b/query/planner.go index 29aa18ca8..5e4303708 100644 --- a/query/planner.go +++ b/query/planner.go @@ -79,8 +79,9 @@ func (qt *CountQueryTree) getLocation(d *db.Database) *db.Location { /////////////////////////////////////////////////////////////////////////////////////////////////// type TopNQueryStep struct { *BaseQueryStep - Input *uuid.UUID - N int + Input *uuid.UUID + Filters []uint64 + N int } type TopNQueryResult struct { @@ -91,6 +92,7 @@ type TopNQueryResult struct { type TopNQueryTree struct { subquery QueryTree location *db.Location + Filters []uint64 N int } @@ -293,7 +295,7 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { // handle SET operation regardless of the slice if query.Operation == "set" { - tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string), query.Args["filter"].(int)}, query.Args["profile_id"].(uint64)} + tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string), query.Args["filter"].(uint64)}, query.Args["profile_id"].(uint64)} return tree } @@ -324,12 +326,18 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree { tree = &CountQueryTree{subquery: subquery} } else if query.Operation == "top-n" { var n int + var filters []uint64 n_, ok := query.Args["n"] if ok { n = n_.(int) } + filters_, ok := query.Args["ids"] + if ok { + filters = filters_.([]uint64) + } + subquery := qp.buildTree(&query.Subqueries[0], slice) - tree = &TopNQueryTree{subquery: subquery, N: n} + tree = &TopNQueryTree{subquery: subquery, Filters: filters, N: n} } else if query.Operation == "union" { subqueries := make([]QueryTree, len(query.Subqueries)) for i, query := range query.Subqueries { @@ -398,7 +406,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati plan = append(plan, step) } else if topn, ok := qt.(*TopNQueryTree); ok { sub_id := uuid.RandomUUID() - step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.N} + step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.Filters, topn.N} subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database)) plan = append(plan, *subq_steps...) plan = append(plan, step)