From 91a1802b2c77dfd722a896bf75ea7df46679a600 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 6 Feb 2014 07:07:54 -0600 Subject: [PATCH] added category support for brands --- deps.json | 15 +++++--- index/brand.go | 18 +++++---- index/commands.go | 11 +++--- index/fragment_container.go | 76 ++++++++++++++++++++++++++++++++----- index/general.go | 2 +- 5 files changed, 95 insertions(+), 27 deletions(-) diff --git a/deps.json b/deps.json index c6038c07b..d42581f90 100644 --- a/deps.json +++ b/deps.json @@ -9,11 +9,6 @@ "version": "35c98e8c885757fab1824daa77cc0e24bf30e76b", "type": "git" }, - "log": { - "repo": "github.com/coreos/go-log/log", - "version": "70d039bee4b0e389e5be560491d8291708506f59", - "type": "git" - }, "gkvlite": { "repo": "github.com/steveyen/gkvlite", "version": "master", @@ -34,11 +29,21 @@ "version": "50", "type": "bzr" }, + "log": { + "repo": "github.com/coreos/go-log/log", + "version": "70d039bee4b0e389e5be560491d8291708506f59", + "type": "git" + }, "lru": { "repo": "github.com/golang/groupcache/lru", "version": "d781998583680cda80cf61e0b37dd0cd8da2eb52", "type": "git" }, + "mysql": { + "repo": "github.com/go-sql-driver/mysql", + "version": "master", + "type": "git" + }, "otto": { "repo": "github.com/robertkrimen/otto", "version": "f9e07770bd9b5142de517b05a85ffbb42d1895da", diff --git a/index/brand.go b/index/brand.go index e014ba594..ac2324682 100644 --- a/index/brand.go +++ b/index/brand.go @@ -1,5 +1,7 @@ package index +import _ "github.com/go-sql-driver/mysql" + import ( "encoding/json" "fmt" @@ -35,9 +37,10 @@ type Brand struct { threshold_length int threshold_idx int skip int + finder ICategoryFinder } -func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand { +func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int, c ICategoryFinder) *Brand { f := new(Brand) f.storage = s f.slice = slice @@ -47,6 +50,7 @@ func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, f.threshold_length = threshold_len f.threshold_idx = threshold f.skip = skipp + f.finder = c f.Clear() //alloc the cache return f @@ -63,13 +67,10 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap { //I should fetch the category here..need to come up with a good source b := self.storage.Fetch(bitmap_id, self.db, self.slice) - self.cache_it(b, bitmap_id, GetCategory(bitmap_id)) + self.cache_it(b, bitmap_id, self.finder.GetCategory(bitmap_id)) return b } -func GetCategory(bitmap_id uint64) int { - return 0 -} func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category int) { if bm.Count() >= self.threshold_value { @@ -170,14 +171,17 @@ func (self *Brand) Store(bitmap_id uint64, bm IBitmap) { //oldbm:=self.Get(bitmap_id) //nbm = Union(oldbm, bm) self.storage.Store(int64(bitmap_id), self.db, self.slice, bm.(*Bitmap)) - self.cache_it(bm, bitmap_id, GetCategory(bitmap_id)) + self.cache_it(bm, bitmap_id, self.finder.GetCategory(bitmap_id)) } -func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair { +func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []int) []Pair { self.rank_counter = 0 println("RANK") self.Rank() // TODO: TERRIBLE REMOVE THIS ASAP is := new(IntSet) + for _, v := range categories { + is.Add(v) + } return self.TopNCat(src_bitmap, n, is) } func dump(r RankList, n int) { diff --git a/index/commands.go b/index/commands.go index 1ffcc5c8b..3401fcc68 100644 --- a/index/commands.go +++ b/index/commands.go @@ -181,15 +181,16 @@ func (self *CmdGetList) Execute(f *Fragment) Calculation { type CmdTopN struct { *Responder - bitmap BitmapHandle - n int + bitmap BitmapHandle + n int + categories []int } -func NewTopN(b BitmapHandle, n int) *CmdTopN { - return &CmdTopN{NewResponder("TopN"), b, n} +func NewTopN(b BitmapHandle, n int, categories []int) *CmdTopN { + return &CmdTopN{NewResponder("TopN"), b, n, categories} } func (self *CmdTopN) Execute(f *Fragment) Calculation { - return f.TopN(self.bitmap, self.n) + return f.TopN(self.bitmap, self.n, self.categories) } type CmdClear struct { diff --git a/index/fragment_container.go b/index/fragment_container.go index 9f3aa43ac..5038d3d58 100644 --- a/index/fragment_container.go +++ b/index/fragment_container.go @@ -1,6 +1,8 @@ package index import ( + "database/sql" + "encoding/gob" "errors" "fmt" @@ -10,15 +12,70 @@ import ( "sync" "time" + _ "github.com/go-sql-driver/mysql" "github.com/golang/groupcache/lru" ) type FragmentContainer struct { fragments map[SUUID]*Fragment + finder *CategoryFinder +} + +func lookup(stmt *sql.Stmt, tile_id uint64) int { + var category int + err := stmt.QueryRow(tile_id).Scan(&category) // WHERE number = 13 + if err != nil { + log.Println(err.Error()) + return 0 + } + return category + +} + +type ICategoryFinder interface { + GetCategory(in uint64) int +} + +type CategoryFinder struct { + in chan uint64 + out chan int +} + +func NewCategoryFinder() *CategoryFinder { + ptr := new(CategoryFinder) + ptr.in = make(chan uint64) + ptr.out = make(chan int) + return ptr +} + +func (self *CategoryFinder) Start() { + connection := config.GetString("category_db_uri") + db, err := sql.Open("mysql", connection) + stmt, err := db.Prepare("select b.category_id from audience_brand b, accounts_tile t where b.id = t.object_id and t.id=?") + if err != nil { + panic(err.Error()) // proper error handling instead of panic in your app + } + defer stmt.Close() + for { + select { + case id := <-self.in: + category := lookup(stmt, id) + self.out <- category + } + } } func NewFragmentContainer() *FragmentContainer { - return &FragmentContainer{make(map[SUUID]*Fragment)} + f := new(FragmentContainer) + f.fragments = make(map[SUUID]*Fragment) + f.finder = NewCategoryFinder() + go f.finder.Start() + + return f +} +func (self *FragmentContainer) GetCategory(in uint64) int { + self.finder.in <- in + return <-self.finder.out } type BitmapHandle uint64 @@ -103,9 +160,9 @@ func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandl return 0, errors.New("Invalid Bitmap Handle") } -func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int) ([]Pair, error) { +func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []int) ([]Pair, error) { if fragment, found := self.GetFragment(frag_id); found { - request := NewTopN(bh, n) + request := NewTopN(bh, n, categories) fragment.requestChan <- request return request.Response().answer.([]Pair), nil } @@ -168,8 +225,9 @@ func (self *FragmentContainer) Clear(frag_id SUUID) (bool, error) { func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) { log.Println("ADD FRAGMENT", frame) - f := NewFragment(id, db, slice, frame) + f := NewFragment(id, db, slice, frame, self) self.fragments[id] = f + go f.ServeFragment() go f.Load() } @@ -177,7 +235,7 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i type Pilosa interface { Get(id uint64) IBitmap SetBit(id uint64, bit_pos uint64) bool - TopN(b IBitmap, n int) []Pair + TopN(b IBitmap, n int, categories []int) []Pair Clear() bool Store(bitmap_id uint64, bm IBitmap) Stats() interface{} @@ -224,13 +282,13 @@ func getStorage(db string, slice int, frame string) Storage { return nil } -func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment { +func NewFragment(frag_id SUUID, db string, slice int, frame string, p ICategoryFinder) *Fragment { var impl Pilosa log.Println(fmt.Sprintf("XXXXXXXXXXXXXXXXXXXXXXXXXXX(%s)", frame)) switch frame { case "brand": log.Println("Brand") - impl = NewBrand(db, slice, getStorage(db, slice, frame), 50000, 45000, 100) + impl = NewBrand(db, slice, getStorage(db, slice, frame), 50000, 45000, 100, p) default: log.Println("General") impl = NewGeneral(db, slice, getStorage(db, slice, frame)) @@ -251,11 +309,11 @@ func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) { return bm.(IBitmap), ok } -func (self *Fragment) TopN(bitmap BitmapHandle, n int) []Pair { +func (self *Fragment) TopN(bitmap BitmapHandle, n int, categories []int) []Pair { bm, ok := self.cache.Get(bitmap) if ok { - return self.impl.TopN(bm.(*Bitmap), n) + return self.impl.TopN(bm.(*Bitmap), n, categories) } return nil } diff --git a/index/general.go b/index/general.go index 818c10510..91f96dcf7 100644 --- a/index/general.go +++ b/index/general.go @@ -56,7 +56,7 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64) bool { } return change } -func (self *General) TopN(b IBitmap, n int) []Pair { +func (self *General) TopN(b IBitmap, n int, categories []int) []Pair { return nil }