diff --git a/index/brand.go b/index/brand.go index 9002c93a8..40d63f8c1 100644 --- a/index/brand.go +++ b/index/brand.go @@ -172,6 +172,8 @@ func (self *Brand) Store(bitmap_id uint64, bm IBitmap) { } func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair { + self.rank_counter = 0 + self.Rank() //TERRIBLE REMOVE TIS ASAP is := new(IntSet) return self.TopNCat(src_bitmap, n, is) } @@ -193,7 +195,7 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { } } - if counter > n { + if counter >= n { break } bm := Intersection(src_bitmap, pair.bitmap) @@ -211,10 +213,12 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { } end := len(results) - 1 o = results[end] + current_threshold := o.Count if current_threshold <= 10 { return packagePairs(results) } + results = append(results, o) for i := x; i < len(self.rankings); i++ { @@ -249,7 +253,7 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { current_threshold = bc } } - return packagePairs(results) + return packagePairs(results[:end]) } func (self *Brand) getFileName() string { base := config.GetString("fragment_base") @@ -283,7 +287,7 @@ func (self *Brand) Persist() error { return encoder.Encode(results) } -func (self *Brand) Load() { +func (self *Brand) Load(requestChan chan Command, f *Fragment) { log.Println("Brand Load") r, err := util.Open(self.getFileName()) if err != nil { @@ -297,6 +301,9 @@ func (self *Brand) Load() { //log.Println("Bad mojo") } for _, k := range keys { - self.Get(k) + request := NewLoadRequest(k) + requestChan <- request + request.Response() + } } diff --git a/index/commands.go b/index/commands.go index 472ab8bee..1ffcc5c8b 100644 --- a/index/commands.go +++ b/index/commands.go @@ -229,3 +229,17 @@ func NewStats() *CmdStats { func (self *CmdStats) Execute(f *Fragment) Calculation { return f.impl.Stats() } + +type CmdLoadRequest struct { + *Responder + bitmap_id uint64 +} + +func NewLoadRequest(bitmap_id uint64) *CmdLoadRequest { + result := &CmdLoadRequest{NewResponder("LoadRequest"), bitmap_id} + return result +} +func (self *CmdLoadRequest) Execute(f *Fragment) Calculation { + f.impl.Get(self.bitmap_id) + return 0 +} diff --git a/index/fragment_container.go b/index/fragment_container.go index d6701b3be..ad3337f11 100644 --- a/index/fragment_container.go +++ b/index/fragment_container.go @@ -168,8 +168,8 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i log.Println("ADD FRAGMENT", frame) f := NewFragment(id, db, slice, frame) self.fragments[id] = f - f.Load() go f.ServeFragment() + go f.Load() } type Pilosa interface { @@ -180,7 +180,7 @@ type Pilosa interface { Store(bitmap_id uint64, bm IBitmap) Stats() interface{} Persist() error - Load() + Load(requestChan chan Command, fragment *Fragment) } type Fragment struct { @@ -309,7 +309,7 @@ func (self *Fragment) Persist() { } } func (self *Fragment) Load() { - self.impl.Load() + self.impl.Load(self.requestChan, self) } func (self *Fragment) ServeFragment() { diff --git a/index/general.go b/index/general.go index 5bde4352e..818c10510 100644 --- a/index/general.go +++ b/index/general.go @@ -99,7 +99,7 @@ func (self *General) Persist() error { return encoder.Encode(self.keys) } -func (self *General) Load() { +func (self *General) Load(requestChan chan Command, f *Fragment) { log.Println("General Load") r, err := util.Open(self.getFileName()) if err != nil { @@ -113,7 +113,9 @@ func (self *General) Load() { return //log.Println("Bad mojo") } - for k := range keys { - self.Get(k) + for k, _ := range keys { + request := NewLoadRequest(k) + requestChan <- request + request.Response() } } diff --git a/query/parser.go b/query/parser.go index 940b8fd2d..876364f5e 100644 --- a/query/parser.go +++ b/query/parser.go @@ -16,24 +16,25 @@ func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) { // BITMAP if tokens[0].Type == TYPE_ID { // TODO: look for frame type in the tokens list - b, err := strconv.Atoi(tokens[0].Text) + b, err := strconv.ParseUint(tokens[0].Text, 10, 64) bitmap_id := uint64(b) if err != nil { panic(err) } // if the next 2 tokens are comma-frame, then we have a frame, else set to a default frame_type := "general" - profile_id := 0 + profile_id := uint64(0) + if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE { frame_type = tokens[2].Text - profile_id, err = strconv.Atoi(tokens[4].Text) + profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64) if err != nil { panic(err) } } else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME { frame_type = tokens[2].Text } else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE { - profile_id, err = strconv.Atoi(tokens[2].Text) + profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64) if err != nil { panic(err) }