diff --git a/core/etcd.go b/core/etcd.go index fb17685d0..c4a0a3ea7 100644 --- a/core/etcd.go +++ b/core/etcd.go @@ -271,15 +271,6 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { } func (self *TopologyMapper) remove_fragment(node *etcd.Node) error { log.Warn(" hot remove_fragment (Not Supported yet):", node) - /* - key := node.Key[len(self.namespace)+1:] - bits := strings.Split(key, "/") - process_uuid, err = util.ParseGUID(node.Value) - if self.service.Id.String() == process_uuid.String() { - fragment_id = util.Hex_to_SUUID(bits[7]) - self.service.Index.RemoveFragment(fragment_id) - } - */ return nil } diff --git a/core/http.go b/core/http.go index 0ed9be6f7..2924cb2d7 100644 --- a/core/http.go +++ b/core/http.go @@ -277,12 +277,6 @@ func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) { http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed) return } - /* values.Set("db", database) - values.Set("id", id) - values.Set("frame", fragment_type) - values.Set("slice", fmt.Sprintf("%d", slice)) - values.Set("bitmap", compressed_string) - */ err := r.ParseForm() if err != nil { @@ -562,13 +556,10 @@ func (self *WebService) HandleBit(w http.ResponseWriter, r *http.Request, ToSet } } - // remoteSetBit.Request() - // results = remoteSetBit.MergeResults(results) encoder := json.NewEncoder(w) err = encoder.Encode(results) if err != nil { log.Warn("JSON SetBit ERROR:", err, ToSet) - //http.Error(w, "Error econding set_bit", http.StatusInternalServerError) return } @@ -583,27 +574,6 @@ func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) { m := &runtime.MemStats{} runtime.ReadMemStats(m) - //self.Report(fmt.Sprintf("%s.goroutines", prefix), - // float64(runtime.NumGoroutine()), now, context, dimensions) - //self.Report(fmt.Sprintf("%s.memory.allocated", prefix), - // float64(memStats.Alloc), now, context, dimensions) - //self.Report(fmt.Sprintf("%s.memory.mallocs", prefix), - // float64(memStats.Mallocs), now, context, dimensions) - //self.Report(fmt.Sprintf("%s.memory.frees", prefix), - // float64(memStats.Frees), now, context, dimensions) - //self.Report(fmt.Sprintf("%s.memory.gc.total_pause", prefix), - // float64(memStats.PauseTotalNs)/nsInMs, now, context, dimensions) - //self.Report(fmt.Sprintf("%s.memory.heap", prefix), - // float64(memStats.HeapAlloc), now, context, dimensions) - //self.Report(fmt.Sprintf("%s.memory.stack", prefix), - // float64(memStats.StackInuse), now, context, dimensions) - - //stats := map[string]interface{}{ - // "num_goroutines": runtime.NumGoroutine(), - // "memory_allocated": m.Sys, - // "memory_": m.Alloc - //} - err := encoder.Encode(m) if err != nil { log.Warn("Error encoding stats") diff --git a/core/load.go b/core/load.go index 7a99e4f3e..c0bd4258c 100644 --- a/core/load.go +++ b/core/load.go @@ -65,7 +65,7 @@ func FromApiString(service *Service, db string, frame string, api_string string, if first { first = false } else { - //make async latter + //make async later sent_count += 1 go sendBitmap(service, bitmap, db, frame, bitmap_id, filter, int(last_slice), finish) bitmap = index.NewBitmap() diff --git a/core/query.go b/core/query.go index a54869673..867b9ae22 100644 --- a/core/query.go +++ b/core/query.go @@ -11,17 +11,8 @@ import ( "github.com/umbel/pilosa/util" ) -/* -func (self *Service) RecallQueryStepHandler(msg *db.Message) { - qs := msg.Data.(query.RecallQueryStep) - - result_message := db.Message{Data: query.RecallQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: count}}} - self.Transport.Send(&result_message, qs.Destination.ProcessId) -} -*/ func (self *Service) CountQueryStepHandler(msg *db.Message) { log.Trace("CountQueryStepHandler") - //spew.Dump("COUNT QUERYSTEP") qs := msg.Data.(query.CountQueryStep) input := qs.Input value, _ := self.Hold.Get(input, util.TimeOut) @@ -36,14 +27,12 @@ func (self *Service) CountQueryStepHandler(msg *db.Message) { if err != nil { spew.Dump(err) } - //spew.Dump("SLICE COUNT", count) result_message := db.Message{Data: query.CountQueryResult{&query.BaseQueryResult{Id: qs.Id, Data: count}}} self.Transport.Send(&result_message, qs.Destination.ProcessId) } func (self *Service) UnionQueryStepHandler(msg *db.Message) { log.Trace("UnionQueryStepHandler") - //spew.Dump("UNION QUERYSTEP") qs := msg.Data.(query.UnionQueryStep) var handles []index.BitmapHandle // create a list of bitmap handles @@ -79,7 +68,6 @@ func (self *Service) UnionQueryStepHandler(msg *db.Message) { func (self *Service) IntersectQueryStepHandler(msg *db.Message) { log.Trace("IntersectQueryStepHandler") - //spew.Dump("INTERSECT QUERYSTEP") qs := msg.Data.(query.IntersectQueryStep) var handles []index.BitmapHandle // create a list of bitmap handles @@ -115,7 +103,6 @@ func (self *Service) IntersectQueryStepHandler(msg *db.Message) { func (self *Service) DifferenceQueryStepHandler(msg *db.Message) { log.Trace("DifferenceQueryStepHandler") - //spew.Dump("DIFFERENCE QUERYSTEP") qs := msg.Data.(query.DifferenceQueryStep) var handles []index.BitmapHandle // create a list of bitmap handles @@ -303,7 +290,6 @@ func min(a, b int) int { } func (self *Service) GetQueryStepHandler(msg *db.Message) { qs := msg.Data.(query.GetQueryStep) - //spew.Dump("GET QUERYSTEP") bh, err := self.Index.Get(qs.Location.FragmentId, qs.Bitmap.Id) if err != nil { @@ -329,7 +315,6 @@ func (self *Service) GetQueryStepHandler(msg *db.Message) { } func (self *Service) SetQueryStepHandler(msg *db.Message) { - //spew.Dump("SET QUERYSTEP") qs := msg.Data.(query.SetQueryStep) result, _ := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId, qs.Bitmap.Filter) @@ -338,7 +323,6 @@ func (self *Service) SetQueryStepHandler(msg *db.Message) { } func (self *Service) ClearQueryStepHandler(msg *db.Message) { - //spew.Dump("SET QUERYSTEP") qs := msg.Data.(query.ClearQueryStep) result, _ := self.Index.ClearBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId) @@ -348,7 +332,6 @@ func (self *Service) ClearQueryStepHandler(msg *db.Message) { func (self *Service) RangeQueryStepHandler(msg *db.Message) { qs := msg.Data.(query.RangeQueryStep) - //spew.Dump("RANDE QUERYSTEP") bh, err := self.Index.Range(qs.Location.FragmentId, qs.Bitmap.Id, qs.Start, qs.End) if err != nil { diff --git a/core/service.go b/core/service.go index 142d93774..2efcc3010 100644 --- a/core/service.go +++ b/core/service.go @@ -86,7 +86,6 @@ func (self *Service) getDev() string { } func (self *Service) PrepareLogging() { logger, _ := log.LoggerFromConfigAsBytes([]byte(self.getProduction())) - // logger, _ := log.LoggerFromConfigAsBytes([]byte(self.getDev())) log.ReplaceLogger(logger) } diff --git a/core/topn.go b/core/topn.go index c895b7132..c3843437b 100644 --- a/core/topn.go +++ b/core/topn.go @@ -81,9 +81,7 @@ func BuildTask(merge_map map[uint64]uint64, tasks := make(map[util.GUID]*Task) for bitmap_id, _ := range merge_map { //for all brands - //for fragment_id, reported_fragments := range slice_map[bitmap_id] { //find missing fragments reporting_fragments := slice_map[bitmap_id] - //id slice ==> SUUID,BitmapHandle for _, p := range missing(reporting_fragments, total_fragments) { task, ok := tasks[p.process] if !ok { @@ -93,7 +91,6 @@ func BuildTask(merge_map map[uint64]uint64, task.Add(p.fragment, bitmap_id, p.handle) } - //} } return tasks @@ -179,7 +176,6 @@ func (self *Service) TopNQueryStepHandler(msg *db.Message) { } else { input := qs.Input value, _ := self.Hold.Get(input, 10) - //var bh index.BitmapHandle switch val := value.(type) { case index.BitmapHandle: bh = val diff --git a/cruncher/cruncher.go b/cruncher/cruncher.go index 50349f9d7..cda812ed6 100644 --- a/cruncher/cruncher.go +++ b/cruncher/cruncher.go @@ -10,31 +10,14 @@ import ( type Cruncher struct { *core.Service close_chan chan bool - // api *index.FragmentContainer } func (cruncher *Cruncher) Run() { - // go cruncher.Cleanup() cruncher.Service.Run() } -/* -func (cruncher *Cruncher) Cleanup() { - exit, done := cruncher.Service.GetExitChannels() - for { - select { - case <-exit: - spew.Dump(cruncher) - cruncher.api.Shutdown() - done <- 1 - } - } -} -*/ - func NewCruncher() *Cruncher { service := core.NewService() - //fragment_container := index.NewFragmentContainer() cruncher := Cruncher{service, make(chan bool)} cruncher.Transport = transport.NewTcpTransport(service) cruncher.Dispatch = dispatch.NewDispatch(service) diff --git a/cruncher/cruncher_test.go b/cruncher/cruncher_test.go index 3e6df2c72..b9f4544f9 100644 --- a/cruncher/cruncher_test.go +++ b/cruncher/cruncher_test.go @@ -10,21 +10,5 @@ import ( func TestCruncher(t *testing.T) { Convey("Basic Cruncher Tests", t, func() { spew.Dump("cruncher test") - - /* - cluster := NewCluster() - database := cluster.GetOrCreateDatabase("main") - - frame := database.GetOrCreateFrame("general") - slice := database.GetOrCreateSlice(0) - - fragment_id, _ := uuid.ParseHex("6a9aea17-2915-4eb4-858f-a8d7d4dc0a1e") - spew.Dump(fragment_id) - database.GetOrCreateFragment(frame, slice, fragment_id) - - spew.Dump(database) - spew.Dump("DONE") - */ - }) } diff --git a/db/constants.go b/db/constants.go index 656be0f29..d2fbdd0eb 100644 --- a/db/constants.go +++ b/db/constants.go @@ -1,66 +1,3 @@ package db const SLICE_WIDTH = 65536 - -/* -DEMOGRAPHIC_TILES = { - 'gender': { - 'male': 1342, - 'female': 1343, - }, - 'marital_status': { - 'married': 1352, - 'single': 1353, - }, - 'age_range': { - '18-21': 2936, - '21-24': 1344, - '25-34': 1345, - '35-44': 1346, - '45-54': 1347, - '55-64': 1348, - '65+': 2937, - }, - 'children': { - 'Yes': 1349, - }, - 'education': { - 'Completed High School': 15723, - 'Attended College': 15726, - 'Completed College': 15724, - 'Completed Graduate School': 15725, - }, - 'home_owner_status': { - 'Own': 1350, - 'Rent': 1351, - }, - 'home_market_value': { - '1k-25k': 5700, - '25k-50k': 2954, - '50k-75k': 2955, - '75k-100k': 2956, - '100k-150k': 1360, - '150k-200k': 1364, - '200k-250k': 2957, - '250k-300k': 2958, - '300k-350k': 2959, - '350k-500k': 16048, - '500k-1mm': 1361, - '1mm+': 1359, - }, - 'household_income_range': { - '25k-35k': 2960, - '125k-150k': 16049, - '100k-125k': 16050, - '150k-175k': 1362, - '35k-50k': 16051, - '75k-100k': 1363, - '50k-75k': 16052, - '15k-25k': 2961, - '250k+': 2962, - '200k-250k': 16053, - '0-15k': 16054, - '175k-200k': 16055, - }, -} -*/ diff --git a/db/topology.go b/db/topology.go index b343b2444..2075c1229 100644 --- a/db/topology.go +++ b/db/topology.go @@ -76,30 +76,6 @@ func (self *Process) SetPortHttp(port int) { self.port_http = port } -/* -// Create a Location struct given a string in form "0.0.0.0:0" -func NewLocation(location_string string) (*Location, error) { - splitstring := strings.Split(location_string, ":") - if len(splitstring) != 2 { - return nil, errors.New("Location string must be in form -0.0.0.0:0") - } - ip := splitstring[0] - port, err := strconv.Atoi(splitstring[1]) - if err != nil { - return nil, errors.New("Port is not a number!") - } - return &Location{ip, port}, nil -} - -func (location *Location) ToString() string { - return fmt.Sprintf("%s:%d", location.Ip, location.Port) -} - -// Map of node location to their router -type NodeMap map[Location]Location -*/ - /////////// CLUSTERS ////////////////////////////////////////////////////////////////////// @@ -379,15 +355,12 @@ func (self *Fragment) GetLocation() *Location { // rename this one func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment, error) { - //d.mutex.Lock() - //defer d.mutex.Unlock() frame, err := d.getFrame(bitmap.FrameType) if err != nil { log.Warn("Missing FrameType", bitmap.FrameType, d.Name, slice) log.Warn(err) return nil, err } - //log.Println(frame, slice) fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { log.Warn("Missing frame,slice", frame, slice) @@ -422,14 +395,6 @@ func (d *Database) GetFragmentForFrameSlice(frame *Frame, slice *Slice) (*Fragme return fsi.GetFragment(frag_id) } -/* -// NOT IMPLEMENTED -// this would loop through all frame_slice_intersect[], then all fragmments to -// find a match -func (d *Database) GetFragmentById(fragment_id *GUID) *Fragment { -} -*/ - func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) (*Fragment, error) { fsi, err := d.GetFrameSliceIntersect(frame, slice) if err != nil { @@ -450,39 +415,6 @@ func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id util.SUUI return &fragment } -/* -func (d *Database) AllocateFragment(frame *Frame, slice *Slice) *Fragment { - // from ETCD, randomly get a process that has available_fragments > 0 - // atomically decrement available_fragments (as long as it's not 0) - // if it IS 0, try until we find a process with available capacity - - * - process, err := GetAvailableProcess() - if err != nil { - log.Fatal(err) - } - * - process_id, _ := uuid.NewV4() - process := NewProcess(process_id) - return nil - //return d.AddFragment(&frame, &slice, process) -} -*/ - -/* -func (d *Database) AddFragmentByProcess(frame *Frame, slice *Slice, process -*Process) *Fragment { - d.mutex.Lock() - defer d.mutex.Unlock() - frameslice, _ := d.GetFrameSliceIntersect(frame, slice) - fragment_id, _ := uuid.NewV4() - fragment := Fragment{id: fragment_id, process: process} - frameslice.fragments = append(frameslice.fragments, &fragment) - frameslice.hashring.Add(fragment.id.String()) - return &fragment -} -*/ - func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id util.SUUID) *Fragment { d.mutex.Lock() defer d.mutex.Unlock() @@ -501,7 +433,6 @@ func GetSlice(profile_id uint64) int { } /////////////////////////////////////////////////////////////////////////////////////////////// - // Get a slice from a database func (d *Database) GetSliceForProfile(profile_id uint64) (*Slice, error) { diff --git a/db/topology_test.go b/db/topology_test.go index fe4bd70ee..1b6ff3cbf 100644 --- a/db/topology_test.go +++ b/db/topology_test.go @@ -22,9 +22,5 @@ func TestTopology(t *testing.T) { fragment_id := util.Id() spew.Dump(fragment_id) database.GetOrCreateFragment(frame, slice, fragment_id) - - // spew.Dump(database) - // spew.Dump("DONE") - }) } diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 3c865971e..84e1043d4 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -44,7 +44,6 @@ func (self *Dispatch) Run() { } else { result, _ = self.service.Index.ClearBit(v.Fragment_id, v.Bitmap_id, v.Profile_id) } - //jbundle := core.SBResult{v.Bitmap_id, ''v.Frame, v.Filter, v.Profile_id, result} bundle := core.SBResult{v.Bitmap_id, v.Frame, v.Filter, v.Profile_id, result} results = append(results, bundle) } diff --git a/executor/executor.go b/executor/executor.go index 26b0b4e57..8fbeae279 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -49,10 +49,6 @@ func (self *Executor) NewJob(job *db.Message) { self.service.RangeQueryStepHandler(job) case query.StashQueryStep: self.service.StashQueryStepHandler(job) - //case query.RecallQueryStep: - // self.service.RecallQueryStepHandler(job) - // case query.MaskQueryStep: - // self.service.MaskQueryStepHandler(job) default: log.Warn("unknown") log.Warn(spew.Sdump(job.Data)) @@ -87,7 +83,6 @@ func (self *Executor) runQuery(database *db.Database, qry *query.Query) error { query_plan, err := query.QueryPlanForQuery(database, qry, &destination) if err != nil { switch obj := err.(type) { - //case *query.InvalidFrame: case *query.FragmentNotFound: self.service.TopologyMapper.MakeFragments(obj.Db, obj.Slice) } diff --git a/hold/hold.go b/hold/hold.go index 65154eacb..d212c6ac8 100644 --- a/hold/hold.go +++ b/hold/hold.go @@ -23,8 +23,6 @@ type Holder struct { delchan chan delhold } -//var Hold Holder - func (self *Holder) DelChan(id *GUID) { log.Trace("Holder.DelChan", id) req := delhold{id} @@ -85,10 +83,3 @@ func NewHolder() *Holder { h := Holder{make(map[GUID]holdchan), make(chan gethold), make(chan delhold)} return &h } - -/* -func init() { - Hold = Holder{make(map[GUID]holdchan), make(chan gethold), make(chan delhold)} - go Hold.Run() -} -*/ diff --git a/index/bitmap.go b/index/bitmap.go index a3b002fb2..e8a08438b 100644 --- a/index/bitmap.go +++ b/index/bitmap.go @@ -61,37 +61,11 @@ func (self *IntSet) Size() int { return len(self.set) } -// - -/* ** native version turned out to be slower -func popcount(i uint64)uint64{ - val:= C.__builtin_popcountll(C.ulonglong(i)) - //x:= uint64(val) - return uint64(val) -} -func popcount(x uint64) (n uint64) { - // bit population count, see - // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return uint64(x >> 56) -} -*/ - type BlockArray struct { Block []uint64 } func (s *BlockArray) bitcount() uint64 { - /*var sum uint64 - for _, b := range (*s).Block { - sum += popcount(b) - } - return sum - */ return popcntSlice(s.Block) } func BlockArray_union(a *BlockArray, b *BlockArray) BlockArray { @@ -343,8 +317,6 @@ func Difference(a_bm IBitmap, b_bm IBitmap) IBitmap { b = b.Next() } else if a.Item().Key == b.Item().Key { var a_node = a.Item() - //var b_node = BlockArray_invert(&b.Item().Value) //probably need to copy this out - //var o = BlockArray_intersection(&a_node.Value, &b_node) var b_node = b.Item().Value var o = BlockArray_difference(&a_node.Value, &b_node) diff --git a/index/brand.go b/index/brand.go index afd16186a..4a42ea926 100644 --- a/index/brand.go +++ b/index/brand.go @@ -133,11 +133,6 @@ func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool change, chunk, address := SetBit(bm, bit_pos) if change { val := chunk.Value.Block[address.BlockIndex] - /* self.storage.BeginBatch() - self.storage.StoreBlock(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val) - self.storage.StoreBlock(bitmap_id, self.db, self.frame, self.slice, filter, COUNTERMASK, 0, bm.Count()) - self.storage.EndBatch() - */ self.storage.StoreBit(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count()) self.rank_count++ } @@ -148,9 +143,7 @@ func (self *Brand) Rank() { start := time.Now() var list RankList for k, item := range self.bitmap_cache { - // if item.bitmap.Count() > 50 { list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category}) - // } } sort.Sort(list) self.rankings = list @@ -161,7 +154,6 @@ func (self *Brand) Rank() { self.threshold_value = 1 } - //dump(self.rankings, 10) self.rank_count = 0 delta := time.Since(start) util.SendTimer("brand_Rank", delta.Nanoseconds()) @@ -171,7 +163,6 @@ func (self *Brand) Rank() { func packagePairs(r RankList) []Pair { res := make([]Pair, r.Len()) - //for i := 0; i < r.Len(); i++ { for i, v := range r { res[i] = Pair{v.Key, v.Count} } @@ -294,7 +285,6 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { break } bc := IntersectionCount(src_bitmap, pair.bitmap) - //bc := BitCount(bm) if bc > 0 { results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category}) counter = counter + 1 @@ -340,7 +330,6 @@ func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair { } bc := IntersectionCount(src_bitmap, o.bitmap) - //bc := BitCount(abitmap) if bc > current_threshold { if results[end-1].Count > bc { @@ -413,7 +402,6 @@ func (self *Brand) Load(requestChan chan Command, f *Fragment) { var keys []uint64 if err := dec.Decode(&keys); err != nil { return - //log.Println("Bad mojo") } globalLock.Lock() defer globalLock.Unlock() @@ -445,11 +433,6 @@ func (self *Brand) ClearBit(bitmap_id uint64, bit_pos uint64) bool { change, chunk, address := ClearBit(bm, bit_pos) if change { val := chunk.Value.Block[address.BlockIndex] - /* self.storage.BeginBatch() - self.storage.StoreBlock(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val) - self.storage.StoreBlock(bitmap_id, self.db, self.frame, self.slice, filter, COUNTERMASK, 0, bm.Count()) - self.storage.EndBatch() - */ if val == 0 { self.storage.RemoveBit(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), bm.Count()) } else { diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index f19d0dda9..082ac7f9d 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -9,7 +9,6 @@ import ( func TestFragment(t *testing.T) { - //id := util.Id() general := util.Hex_to_SUUID("1") brand := util.Hex_to_SUUID("2") dummy := NewFragmentContainer() @@ -22,7 +21,6 @@ func TestFragment(t *testing.T) { }) Convey("SetBit/Count 1 1", t, func() { - // bh, _ := dummy.Get(id, 1234) bi1 := uint64(1234) changed, _ := dummy.SetBit(general, bi1, 1, 0) So(changed, ShouldEqual, true) @@ -153,8 +151,6 @@ func TestFragment(t *testing.T) { }) Convey("store ", t, func() { b := uint64(1029) - // compressed := "H4sIAAAJbogA/2JmYRBQ+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGB4R+j+v8mJkaFH/8h4B+M8X+UgcwAhZTm/yZgMCLCarC4bbAxYGHFNBpWBBmwsGIeDSuCDFhYsYyGFUEGLKxYR8OKIAMWVmyjYUWQwcDwfyYwqNgHLKjk4Q7BBAAAAAD//wEAAP//QNipzzcJAAA=" - // compressed := "H4sIAAAJbogA/2JmYWBS+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdT/VqAIV7SxUWxpZl6JmQlImJGN0YGBweN/E+M/RgdGRoUf/6EAk/GbGUQy4AUAAAAA//8BAAD//2vjG9ezAAAA" compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" dummy.LoadBitmap(brand, b, compressed, 0) bh1, _ := dummy.Get(brand, b) diff --git a/index/general.go b/index/general.go index 73d767838..82cf0ca29 100644 --- a/index/general.go +++ b/index/general.go @@ -27,7 +27,6 @@ func NewGeneral(db string, frame string, slice int, s Storage) *General { f.db = db f.Clear() f.keys = make(map[uint64]interface{}) - //f.bitmap_cache = lru.New(10000) return f } @@ -57,12 +56,7 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) boo change, chunk, address := SetBit(bm, bit_pos) if change { val := chunk.Value.Block[address.BlockIndex] - //self.storage.BeginBatch() - //self.storage.StoreBlock(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val) - //self.storage.StoreBlock(bitmap_id, self.db, self.frame, self.slice, filter, COUNTERMASK, 0, bm.Count()) - //self.storage.EndBatch() self.storage.StoreBit(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count()) - } return change } @@ -72,8 +66,6 @@ func (self *General) TopN(b IBitmap, n int, categories []uint64) []Pair { return empty } func (self *General) Store(bitmap_id uint64, bm IBitmap, filter uint64) { - //oldbm:=self.Get(bitmap_id) - //nbm = Union(oldbm, bm) self.storage.Store(bitmap_id, self.db, self.frame, self.slice, filter, bm.(*Bitmap)) self.bitmap_cache.Add(bitmap_id, bm) self.keys[bitmap_id] = 0 diff --git a/index/storage_cass.go b/index/storage_cass.go index edf7b27ae..b1de73677 100644 --- a/index/storage_cass.go +++ b/index/storage_cass.go @@ -140,14 +140,6 @@ func (self *CassandraStorage) EndBatch() { start := time.Now() if self.batch != nil { self.FlushBatch() - /* - last := time.Since(self.batch_time) - if last.Seconds() > self.cass_time_window_secs { - self.FlushBatch() - } else if self.batch_counter > self.cass_flush_size { - self.FlushBatch() - } - */ } else { log.Warn("NIL BATCH") } diff --git a/index/storage_leveldb.go b/index/storage_leveldb.go index 579a5578d..c2b3e246d 100644 --- a/index/storage_leveldb.go +++ b/index/storage_leveldb.go @@ -96,7 +96,6 @@ func (self *LevelDBStorage) Fetch(bitmap_id uint64, db string, frame string, sli last_key = chunk_key } iter.Release() - //err = iter.Error() delta := time.Since(start) util.SendTimer("leveldb_storage_Fetch", delta.Nanoseconds()) diff --git a/index/storage_mem.go b/index/storage_mem.go index c3d22f19c..bd5a0801a 100644 --- a/index/storage_mem.go +++ b/index/storage_mem.go @@ -9,7 +9,6 @@ type MemoryStorage struct { } func NewMemoryStorage() Storage { - // log.Println("Hello") obj := new(MemoryStorage) obj.db = make(map[string]*Bitmap) @@ -25,7 +24,6 @@ func (c *MemoryStorage) EndBatch() { func (c *MemoryStorage) FlushBatch() { } func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { - // log.Println("hello") key := fmt.Sprintf("%d:%s:%s:%d", bitmap_id, db, frame, slice) bitmap, found := c.db[key] diff --git a/index/timeframe_test.go b/index/timeframe_test.go index 52b42a7a8..74297d8cb 100644 --- a/index/timeframe_test.go +++ b/index/timeframe_test.go @@ -41,7 +41,6 @@ func TestTimeFrame(t *testing.T) { return } - //print get_YMD_id(2014,3,28,1234) Convey("Test ID", t, func() { const shortForm = "2006-01-02 15:04" x, _ := time.Parse(shortForm, "1970-01-01 00:00:00") diff --git a/query/parser.go b/query/parser.go index a18bf51cd..c6db4f359 100644 --- a/query/parser.go +++ b/query/parser.go @@ -19,26 +19,6 @@ type QueryParser struct { pos int } -/* MASTER -if len(tokens) > 4 && tokens[2].Type == TYPE_FRAME && tokens[4].Type == TYPE_PROFILE { - frame_type = tokens[2].Text - profile_id, err = strconv.ParseUint(tokens[4].Text, 10, 64) - if err != nil { - panic(err) - } -} else if len(tokens) > 2 && tokens[2].Type == TYPE_FRAME { - frame_type = tokens[2].Text -} else if len(tokens) > 2 && tokens[2].Type == TYPE_PROFILE { - profile_id, err = strconv.ParseUint(tokens[2].Text, 10, 64) - if err != nil { - panic(err) - } -} -var filter int //TRAVIS the is for the category -filter = 0 -bm := db.Bitmap{bitmap_id, frame_type, filter} -return []QueryInput{&bm}, uint64(profile_id), 0 -*/ func (self *QueryParser) next() *Token { log.Trace("QueryParser.next") self.pos += 1 diff --git a/query/parser_test.go b/query/parser_test.go index 212741d58..1311c4d38 100644 --- a/query/parser_test.go +++ b/query/parser_test.go @@ -91,7 +91,6 @@ func TestQueryParser(t *testing.T) { }) Convey("Bracketed Lists", t, func() { tokens, err := Lex("plugin(get(99), [get(10), get(11)])") - //tokens, err := Lex("plugin(get(99), [1,3])") So(err, ShouldBeNil) query, err := Parse(tokens) diff --git a/query/planner.go b/query/planner.go index ef1294f8c..fefbe967c 100644 --- a/query/planner.go +++ b/query/planner.go @@ -76,7 +76,6 @@ func (self *BaseQueryStep) GetLocation() *db.Location { func (self *BaseQueryStep) LocIsDest() bool { log.Trace("BaseQueryStep.LocIsDest") - // if self.Location.ProcessId == self.Destination.ProcessId && if util.Equal(self.Location.ProcessId, self.Destination.ProcessId) && self.Location.FragmentId == self.Destination.FragmentId { log.Trace("BaseQueryStep.LocIsDest Return true") @@ -492,7 +491,6 @@ func (self *QueryPlanner) buildTree(query *Query, slice int) (QueryTree, error) var p Appendable if query.Operation == "stash" { - // log.Warn("STASH:", n) tree = &StashQueryTree{N: n} } else { tree = &CatQueryTree{N: n} @@ -627,14 +625,6 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca plan = append(plan, *subq_steps...) } plan = append(plan, step) - /*} else if recall, ok := qt.(*RecallQueryTree); ok { - //need to go through the stash and fetch - //from self.Database i can fetch the locations - step := RecallQueryStep{&BaseQueryStep{id, "recall", loc, location}, recall.Stash} - - plan := QueryPlan{step} - return &plan, nil - */ } else if stash, ok := qt.(*StashQueryTree); ok { inputs := make([]*util.GUID, len(stash.subqueries)) @@ -712,16 +702,6 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *util.GUID, location *db.Loca step := GetQueryStep{&BaseQueryStep{id, "get", loc, location}, get.bitmap, get.slice} plan := QueryPlan{step} return &plan, nil - /* - } else if mask, ok := qt.(*MaskQueryTree); ok { - loc, err := mask.getLocation(self.Database) - if err != nil { - return nil, err - } - step := MaskQueryStep{&BaseQueryStep{id, "mask", loc, location}, mask.start, mask.end} - plan := QueryPlan{step} - return &plan, nil - */ } else if rang, ok := qt.(*RangeQueryTree); ok { loc, err := rang.getLocation(self.Database) if err != nil { diff --git a/query/planner_test.go b/query/planner_test.go index e2cfc8e82..6c2695653 100644 --- a/query/planner_test.go +++ b/query/planner_test.go @@ -248,7 +248,6 @@ func TestQueryPlanner(t *testing.T) { So(len(p), ShouldNotEqual, 0) log.Println(len(p)) spew.Dump(p[0]) - //So(p[0].(StashQueryStep).Operation, ShouldEqual, "stash") }) } diff --git a/query/query_test.go b/query/query_test.go index 8e1fa5ec5..c8d0e42b1 100644 --- a/query/query_test.go +++ b/query/query_test.go @@ -13,12 +13,6 @@ func TestQuery(t *testing.T) { filter, filters := TokensToFilterStrings(tokens) So(filter, ShouldEqual, "get(88,general)") So(filters, ShouldResemble, []string{"get(12,general)", "get(13,general)"}) - /* - spew.Dump("*********************************************") - spew.Dump(filter) - spew.Dump(filters) - spew.Dump("*********************************************") - */ }) Convey("Bracketed Lists 2", t, func() { tokens, err := Lex("plugin(intersect(get(88, general, [0]), get(77, b.n)), [get(12, general), get(13, general)])") diff --git a/util/constants.go b/util/constants.go index b9ba72020..1f64bcd05 100644 --- a/util/constants.go +++ b/util/constants.go @@ -31,11 +31,7 @@ func ByteToInt64(data []byte) int64 { } func Uint64ToInt64(before uint64) int64 { return int64(before) - //buf := Uint64ToByte(before) - //return ByteToInt64(buf) } func Int64ToUint64(before int64) uint64 { return uint64(before) - //buf := Int64ToByte(before) - //return ByteToUint64(buf) }