scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert filter arg from int to uint64 so it matches the IDS arg type from lexer

Browse source
This commit is contained in:
travisturner 2014-04-07 11:06:58 -05:00
parent 1b33a62736
commit a339e6ba11
16 changed files with 72 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
core/batch.go
View file

@ -14,7 +14,7 @@ type BatchRequest struct {
Fragment_id SUUID
Bitmap_id uint64
Compressed_bitmap string
Filter int
Filter uint64
}
type BatchResponse struct {
@ -33,7 +33,7 @@ func init() {
gob.Register(BatchResponse{})
}
func (self *Service) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter int) error {
func (self *Service) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter uint64) error {
//determine the fragment_id from database/frame/slice
database := self.Cluster.GetOrCreateDatabase(database_name)
oslice := database.GetOrCreateSlice(slice)

2
core/http.go
View file

@ -124,7 +124,7 @@ func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
return
}
results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice), int(filter))
results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice), uint64(filter))
encoder := json.NewEncoder(w)
err = encoder.Encode(results)

13
core/query.go
View file

@ -33,9 +33,10 @@ func (self *Service) CountQueryStepHandler(msg *db.Message) {
func (self *Service) TopNQueryStepHandler(msg *db.Message) {
//spew.Dump("TOP-N QUERYSTEP")
qs := msg.Data.(query.TopNQueryStep)
spew.Dump(qs)
//TRAVIS SEE HERE
//need categories in qs I just added so it would compile
var categoryleaves []int
var categoryleaves []uint64
input := qs.Input
value, _ := self.Hold.Get(input, 10)
var bh index.BitmapHandle
@ -45,12 +46,22 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) {
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
categoryleaves = qs.Filters
/*
spew.Dump(bh, qs.N*2, categoryleaves)
result_message := db.Message{Data: "foobar"}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
*/
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, categoryleaves)
if err != nil {
spew.Dump(err)
}
result_message := db.Message{Data: query.TopNQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: topn}}}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Service) UnionQueryStepHandler(msg *db.Message) {

2
db/topology.go
View file

@ -413,5 +413,5 @@ func (d *Database) GetSliceForProfile(profile_id uint64) (*Slice, error) {
type Bitmap struct {
Id uint64
FrameType string
Filter int
Filter uint64
}

10
index/bitmap.go
View file

@ -31,25 +31,25 @@ func init() {
//
type IntSet struct {
set map[int]bool
set map[uint64]bool
}
func NewIntSet() *IntSet {
return &IntSet{make(map[int]bool)}
return &IntSet{make(map[uint64]bool)}
}
func (set *IntSet) Add(i int) bool {
func (set *IntSet) Add(i uint64) bool {
_, found := set.set[i]
set.set[i] = true
return !found //False if it existed already
}
func (set *IntSet) Contains(i int) bool {
func (set *IntSet) Contains(i uint64) bool {
_, found := set.set[i]
return found //true if it existed already
}
func (set *IntSet) Remove(i int) {
func (set *IntSet) Remove(i uint64) {
delete(set.set, i)
}

10
index/brand.go
View file

@ -17,7 +17,7 @@ type Pair struct {
type Rank struct {
*Pair
bitmap IBitmap
category int
category uint64
}
type RankList []*Rank
@ -72,7 +72,7 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap {
return b
}
func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category int) {
func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category uint64) {
if bm.Count() >= self.threshold_value {
self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category}
if len(self.bitmap_cache) > self.threshold_length {
@ -89,7 +89,7 @@ func (self *Brand) trim() {
}
func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter int) bool {
func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := SetBit(bm, bit_pos)
if change {
@ -165,12 +165,12 @@ func (self *Brand) Stats() interface{} {
"skip": self.skip}
return stats
}
func (self *Brand) Store(bitmap_id uint64, bm IBitmap, filter int) {
func (self *Brand) Store(bitmap_id uint64, bm IBitmap, filter uint64) {
self.storage.Store(int64(bitmap_id), self.db, self.frame, self.slice, filter, bm.(*Bitmap))
self.cache_it(bm, bitmap_id, filter)
}
func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []int) []Pair {
func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []uint64) []Pair {
self.rank_counter = 0
self.Rank() // TODO: TERRIBLE REMOVE THIS ASAP
is := new(IntSet)

12
index/commands.go
View file

@ -99,10 +99,10 @@ type CmdSetBit struct {
*Responder
bitmap_id uint64
bit_pos uint64
filter int
filter uint64
}
func NewSetBit(bitmap_id uint64, bit_pos uint64, filter int) *CmdSetBit {
func NewSetBit(bitmap_id uint64, bit_pos uint64, filter uint64) *CmdSetBit {
result := &CmdSetBit{NewResponder("SetBit"), bitmap_id, bit_pos, filter}
return result
}
@ -184,10 +184,10 @@ type CmdTopN struct {
*Responder
bitmap BitmapHandle
n int
categories []int
categories []uint64
}
func NewTopN(b BitmapHandle, n int, categories []int) *CmdTopN {
func NewTopN(b BitmapHandle, n int, categories []uint64) *CmdTopN {
return &CmdTopN{NewResponder("TopN"), b, n, categories}
}
func (self *CmdTopN) Execute(f *Fragment) Calculation {
@ -209,10 +209,10 @@ type CmdLoader struct {
*Responder
bitmap_id uint64
compressed_bitmap string
filter int
filter uint64
}
func NewLoader(bitmap_id uint64, compressed_bitmap string, filter int) *CmdLoader {
func NewLoader(bitmap_id uint64, compressed_bitmap string, filter uint64) *CmdLoader {
return &CmdLoader{NewResponder("Loader"), bitmap_id, compressed_bitmap, filter}
}
func (self *CmdLoader) Execute(f *Fragment) Calculation {

14
index/fragment_container.go
View file

@ -59,7 +59,7 @@ func (self *FragmentContainer) Shutdown() {
log.Println("Container Shutdown Complete")
}
func (self *FragmentContainer) LoadBitmap(frag_id SUUID, bitmap_id uint64, compressed_bitmap string, filter int) {
func (self *FragmentContainer) LoadBitmap(frag_id SUUID, bitmap_id uint64, compressed_bitmap string, filter uint64) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewLoader(bitmap_id, compressed_bitmap, filter)
fragment.requestChan <- request
@ -120,7 +120,7 @@ func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandl
return 0, errors.New("Invalid Bitmap Handle")
}
func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []int) ([]Pair, error) {
func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewTopN(bh, n, categories)
fragment.requestChan <- request
@ -165,7 +165,7 @@ func (self *FragmentContainer) FromBytes(frag_id SUUID, bytes []byte) (BitmapHan
return 0, errors.New("Invalid Bitmap Handle")
}
func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64, category int) (bool, error) {
func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewSetBit(bitmap_id, pos, category)
fragment.requestChan <- request
@ -194,10 +194,10 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i
type Pilosa interface {
Get(id uint64) IBitmap
SetBit(id uint64, bit_pos uint64, filter int) bool
TopN(b IBitmap, n int, categories []int) []Pair
SetBit(id uint64, bit_pos uint64, filter uint64) bool
TopN(b IBitmap, n int, categories []uint64) []Pair
Clear() bool
Store(bitmap_id uint64, bm IBitmap, filter int)
Store(bitmap_id uint64, bm IBitmap, filter uint64)
Stats() interface{}
Persist() error
Load(requestChan chan Command, fragment *Fragment)
@ -261,7 +261,7 @@ func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) {
return bm.(IBitmap), ok
}
func (self *Fragment) TopN(bitmap BitmapHandle, n int, categories []int) []Pair {
func (self *Fragment) TopN(bitmap BitmapHandle, n int, categories []uint64) []Pair {
bm, ok := self.cache.Get(bitmap)
if ok {

6
index/general.go
View file

@ -47,7 +47,7 @@ func (self *General) Get(bitmap_id uint64) IBitmap {
self.keys[bitmap_id] = 0
return bm.(*Bitmap)
}
func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter int) bool {
func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := SetBit(bm, bit_pos)
if change {
@ -58,11 +58,11 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter int) bool {
}
return change
}
func (self *General) TopN(b IBitmap, n int, categories []int) []Pair {
func (self *General) TopN(b IBitmap, n int, categories []uint64) []Pair {
return nil
}
func (self *General) Store(bitmap_id uint64, bm IBitmap, filter int) {
func (self *General) Store(bitmap_id uint64, bm IBitmap, filter uint64) {
//oldbm:=self.Get(bitmap_id)
//nbm = Union(oldbm, bm)
self.storage.Store(int64(bitmap_id), self.db, self.frame, self.slice, filter, bm.(*Bitmap))

6
index/storage.go
View file

@ -1,7 +1,7 @@
package index
type Storage interface {
Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, int)
Store(id int64, db string, frame string, slice int, filter int, bitmap *Bitmap) error
StoreBlock(id int64, db string, frame string, slice int, filter int, chunk int64, block_index int32, block int64) error
Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64)
Store(id int64, db string, frame string, slice int, filter uint64, bitmap *Bitmap) error
StoreBlock(id int64, db string, frame string, slice int, filter uint64, chunk int64, block_index int32, block int64) error
}

16
index/storage_cass.go
View file

@ -15,10 +15,10 @@ type CassandraStorage struct {
func BuildSchema() {
/*
"CREATE KEYSPACE IF NOT EXISTS hotbox WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1"
create keyspace if not exists hotbox with replication = { 'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true;
CREATE TABLE IF NOT EXISTS bitmap ( bitmap_id bigint, db varchar, frame varchar, slice int, filter int, ChunkKey bigint, BlockIndex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame,slice),ChunkKey,BlockIndex) )
"
"CREATE KEYSPACE IF NOT EXISTS hotbox WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1"
create keyspace if not exists hotbox with replication = { 'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true;
CREATE TABLE IF NOT EXISTS bitmap ( bitmap_id bigint, db varchar, frame varchar, slice int, filter int, ChunkKey bigint, BlockIndex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame,slice),ChunkKey,BlockIndex) )
"
*/
}
@ -42,7 +42,7 @@ func NewCassStorage(host, keyspace string) Storage {
return obj
}
func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, int) {
func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) {
var dumb = COUNTERMASK
last_key := int64(dumb)
marker := int64(dumb)
@ -53,7 +53,7 @@ func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slic
chunk_key, block int64
block_index uint32
s8 uint8
filter int
filter uint64
)
log.Println("FETCHING ", bitmap_id, db, slice)
@ -80,7 +80,7 @@ func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slic
return bitmap, filter
}
func (c *CassandraStorage) Store(id int64, db string, frame string, slice int, filter int, bitmap *Bitmap) error {
func (c *CassandraStorage) Store(id int64, db string, frame string, slice int, filter uint64, bitmap *Bitmap) error {
for i := bitmap.Min(); !i.Limit(); i = i.Next() {
var chunk = i.Item()
for idx, block := range chunk.Value.Block {
@ -100,7 +100,7 @@ func (c *CassandraStorage) Store(id int64, db string, frame string, slice int, f
return nil
}
func (c *CassandraStorage) StoreBlock(id int64, db string, frame string, slice int, filter int, chunk int64, block_index int32, block int64) error {
func (c *CassandraStorage) StoreBlock(id int64, db string, frame string, slice int, filter uint64, chunk int64, block_index int32, block int64) error {
if err := c.db.Query(`
INSERT INTO bitmap ( bitmap_id, db, frame, slice , filter, ChunkKey, BlockIndex, block) VALUES (?,?,?,?,?,?,?,?);`, id, db, frame, slice, filter, chunk, block_index, block).Exec(); err != nil {

6
index/storage_mem.go
View file

@ -16,7 +16,7 @@ func NewMemoryStorage() Storage {
return obj
}
func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, int) {
func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) {
// log.Println("hello")
key := fmt.Sprintf("%d:%s:%s:%d", bitmap_id, db, frame, slice)
@ -28,12 +28,12 @@ func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice i
return bitmap, 0
}
func (c *MemoryStorage) Store(bitmap_id int64, db string, frame string, slice int, filter int, bitmap *Bitmap) error {
func (c *MemoryStorage) Store(bitmap_id int64, db string, frame string, slice int, filter uint64, bitmap *Bitmap) error {
//only use the cache and throw away everything
return nil
}
func (c *MemoryStorage) StoreBlock(bitmap_id int64, db string, frame string, slice int, filter int, chunk_key int64, block_index int32, block int64) error {
func (c *MemoryStorage) StoreBlock(bitmap_id int64, db string, frame string, slice int, filter uint64, chunk_key int64, block_index int32, block int64) error {
//only use the cache and throw away everything
return nil

2
index/storage_test.go
View file

@ -46,7 +46,7 @@ func TestStorage(t *testing.T) {
SetBit(bm, 1)
SetBit(bm, 2)
fmt.Println("STORE")
storage.Store(int64(bitmap_id), db, frame, slice, filter, bm.(*Bitmap))
storage.Store(int64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap))
fmt.Println("FETCH")
bm2, _ := storage.Fetch(bitmap_id, db, frame, slice)
So(BitCount(bm), ShouldEqual, BitCount(bm2))

2
query/parser.go
View file

@ -122,7 +122,7 @@ ArgLoop:
case 1:
query.Args["frame"] = token.Text
case 2:
i, err := strconv.Atoi(token.Text)
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}

2
query/parser_test.go
View file

@ -24,7 +24,7 @@ func TestQueryParser(t *testing.T) {
So(err, ShouldBeNil)
So(query.Operation, ShouldEqual, "set")
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "filter": int(0), "profile_id": uint64(20)})
So(query.Args, ShouldResemble, map[string]interface{}{"id": uint64(10), "frame": "general", "filter": uint64(0), "profile_id": uint64(20)})
})
Convey("Basic nested query parse", t, func() {
tokens, err := Lex("union(get(10,general), get(11,brand), get(12))")

18
query/planner.go
View file

@ -79,8 +79,9 @@ func (qt *CountQueryTree) getLocation(d *db.Database) *db.Location {
///////////////////////////////////////////////////////////////////////////////////////////////////
type TopNQueryStep struct {
*BaseQueryStep
Input *uuid.UUID
N int
Input *uuid.UUID
Filters []uint64
N int
}
type TopNQueryResult struct {
@ -91,6 +92,7 @@ type TopNQueryResult struct {
type TopNQueryTree struct {
subquery QueryTree
location *db.Location
Filters []uint64
N int
}
@ -293,7 +295,7 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
// handle SET operation regardless of the slice
if query.Operation == "set" {
tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string), query.Args["filter"].(int)}, query.Args["profile_id"].(uint64)}
tree = &SetQueryTree{&db.Bitmap{query.Args["id"].(uint64), query.Args["frame"].(string), query.Args["filter"].(uint64)}, query.Args["profile_id"].(uint64)}
return tree
}
@ -324,12 +326,18 @@ func (qp *QueryPlanner) buildTree(query *Query, slice int) QueryTree {
tree = &CountQueryTree{subquery: subquery}
} else if query.Operation == "top-n" {
var n int
var filters []uint64
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
filters_, ok := query.Args["ids"]
if ok {
filters = filters_.([]uint64)
}
subquery := qp.buildTree(&query.Subqueries[0], slice)
tree = &TopNQueryTree{subquery: subquery, N: n}
tree = &TopNQueryTree{subquery: subquery, Filters: filters, N: n}
} else if query.Operation == "union" {
subqueries := make([]QueryTree, len(query.Subqueries))
for i, query := range query.Subqueries {
@ -398,7 +406,7 @@ func (qp *QueryPlanner) flatten(qt QueryTree, id *uuid.UUID, location *db.Locati
plan = append(plan, step)
} else if topn, ok := qt.(*TopNQueryTree); ok {
sub_id := uuid.RandomUUID()
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.N}
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", topn.getLocation(qp.Database), location}, &sub_id, topn.Filters, topn.N}
subq_steps := qp.flatten(topn.subquery, &sub_id, topn.getLocation(qp.Database))
plan = append(plan, *subq_steps...)
plan = append(plan, step)