fixed int parser overflow in querylanner;changed order of loading startup; adjust top-n calc

index/brand.go

@@ -172,6 +172,8 @@ func (self *Brand) Store(bitmap_id uint64, bm IBitmap) {
 }
 
 func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair {
+	self.rank_counter = 0
+	self.Rank() //TERRIBLE REMOVE TIS ASAP
 	is := new(IntSet)
 	return self.TopNCat(src_bitmap, n, is)
 }
@@ -193,7 +195,7 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 			}
 		}
 
-		if counter > n {
+		if counter >= n {
 			break
 		}
 		bm := Intersection(src_bitmap, pair.bitmap)
@@ -211,10 +213,12 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 	}
 	end := len(results) - 1
 	o = results[end]
+
 	current_threshold := o.Count
 	if current_threshold <= 10 {
 		return packagePairs(results)
 	}
+
 	results = append(results, o)
 
 	for i := x; i < len(self.rankings); i++ {
@@ -249,7 +253,7 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
 			current_threshold = bc
 		}
 	}
-	return packagePairs(results)
+	return packagePairs(results[:end])
 }
 func (self *Brand) getFileName() string {
 	base := config.GetString("fragment_base")
@@ -283,7 +287,7 @@ func (self *Brand) Persist() error {
 	return encoder.Encode(results)
 }
 
-func (self *Brand) Load() {
+func (self *Brand) Load(requestChan chan Command, f *Fragment) {
 	log.Println("Brand Load")
 	r, err := util.Open(self.getFileName())
 	if err != nil {
@@ -297,6 +301,9 @@ func (self *Brand) Load() {
 		//log.Println("Bad mojo")
 	}
 	for _, k := range keys {
-		self.Get(k)
+		request := NewLoadRequest(k)
+		requestChan <- request
+		request.Response()
+
 	}
 }

index/commands.go

@@ -229,3 +229,17 @@ func NewStats() *CmdStats {
 func (self *CmdStats) Execute(f *Fragment) Calculation {
 	return f.impl.Stats()
 }
+
+type CmdLoadRequest struct {
+	*Responder
+	bitmap_id uint64
+}
+
+func NewLoadRequest(bitmap_id uint64) *CmdLoadRequest {
+	result := &CmdLoadRequest{NewResponder("LoadRequest"), bitmap_id}
+	return result
+}
+func (self *CmdLoadRequest) Execute(f *Fragment) Calculation {
+	f.impl.Get(self.bitmap_id)
+	return 0
+}

index/fragment_container.go

@@ -168,8 +168,8 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i
 	log.Println("ADD FRAGMENT", frame)
 	f := NewFragment(id, db, slice, frame)
 	self.fragments[id] = f
-	f.Load()
 	go f.ServeFragment()
+	go f.Load()
 }
 
 type Pilosa interface {
@@ -180,7 +180,7 @@ type Pilosa interface {
 	Store(bitmap_id uint64, bm IBitmap)
 	Stats() interface{}
 	Persist() error
-	Load()
+	Load(requestChan chan Command, fragment *Fragment)
 }
 
 type Fragment struct {
@@ -309,7 +309,7 @@ func (self *Fragment) Persist() {
 	}
 }
 func (self *Fragment) Load() {
-	self.impl.Load()
+	self.impl.Load(self.requestChan, self)
 }
 
 func (self *Fragment) ServeFragment() {

index/general.go

@@ -99,7 +99,7 @@ func (self *General) Persist() error {
 	return encoder.Encode(self.keys)
 }
 
-func (self *General) Load() {
+func (self *General) Load(requestChan chan Command, f *Fragment) {
 	log.Println("General Load")
 	r, err := util.Open(self.getFileName())
 	if err != nil {
@@ -113,7 +113,9 @@ func (self *General) Load() {
 		return
 		//log.Println("Bad mojo")
 	}
-	for k := range keys {
-		self.Get(k)
+	for k, _ := range keys {
+		request := NewLoadRequest(k)
+		requestChan <- request
+		request.Response()
 	}
 }

query/parser.go

@@ -16,24 +16,25 @@ func (qp *QueryParser) walkInputs(tokens []Token) ([]QueryInput, uint64) {
 	// BITMAP
 	if tokens[0].Type == TYPE_ID {
 		// TODO: look for frame type in the tokens list
-		b, err := strconv.Atoi(tokens[0].Text)
+		b, err := strconv.ParseUint(tokens[0].Text, 10, 64)
 		bitmap_id := uint64(b)
 		if err != nil {
 			panic(err)
 		}
 		// if the next 2 tokens are comma-frame, then we have a frame, else set to a default
 		frame_type := "general"
-		profile_id := 0
+		profile_id := uint64(0)
+
 		if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE {
 			frame_type = tokens[2].Text
-			profile_id, err = strconv.Atoi(tokens[4].Text)
+			profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64)
 			if err != nil {
 				panic(err)
 			}
 		} else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME {
 			frame_type = tokens[2].Text
 		} else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE {
-			profile_id, err = strconv.Atoi(tokens[2].Text)
+			profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64)
 			if err != nil {
 				panic(err)
 			}