scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added category support for brands

Browse source
This commit is contained in:
Todd Gruben 2014-02-06 07:07:54 -06:00
parent f55c092cc1
commit 91a1802b2c
5 changed files with 95 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
deps.json
View file

@ -9,11 +9,6 @@
"version": "35c98e8c885757fab1824daa77cc0e24bf30e76b",
"type": "git"
},
"log": {
"repo": "github.com/coreos/go-log/log",
"version": "70d039bee4b0e389e5be560491d8291708506f59",
"type": "git"
},
"gkvlite": {
"repo": "github.com/steveyen/gkvlite",
"version": "master",
@ -34,11 +29,21 @@
"version": "50",
"type": "bzr"
},
"log": {
"repo": "github.com/coreos/go-log/log",
"version": "70d039bee4b0e389e5be560491d8291708506f59",
"type": "git"
},
"lru": {
"repo": "github.com/golang/groupcache/lru",
"version": "d781998583680cda80cf61e0b37dd0cd8da2eb52",
"type": "git"
},
"mysql": {
"repo": "github.com/go-sql-driver/mysql",
"version": "master",
"type": "git"
},
"otto": {
"repo": "github.com/robertkrimen/otto",
"version": "f9e07770bd9b5142de517b05a85ffbb42d1895da",

18
index/brand.go
View file

@ -1,5 +1,7 @@
package index
import _ "github.com/go-sql-driver/mysql"
import (
"encoding/json"
"fmt"
@ -35,9 +37,10 @@ type Brand struct {
threshold_length int
threshold_idx int
skip int
finder ICategoryFinder
}
func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int, skipp int, c ICategoryFinder) *Brand {
f := new(Brand)
f.storage = s
f.slice = slice
@ -47,6 +50,7 @@ func NewBrand(db string, slice int, s Storage, threshold_len int, threshold int,
f.threshold_length = threshold_len
f.threshold_idx = threshold
f.skip = skipp
f.finder = c
f.Clear() //alloc the cache
return f
@ -63,13 +67,10 @@ func (self *Brand) Get(bitmap_id uint64) IBitmap {
//I should fetch the category here..need to come up with a good source
b := self.storage.Fetch(bitmap_id, self.db, self.slice)
self.cache_it(b, bitmap_id, GetCategory(bitmap_id))
self.cache_it(b, bitmap_id, self.finder.GetCategory(bitmap_id))
return b
}
func GetCategory(bitmap_id uint64) int {
return 0
}
func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category int) {
if bm.Count() >= self.threshold_value {
@ -170,14 +171,17 @@ func (self *Brand) Store(bitmap_id uint64, bm IBitmap) {
//oldbm:=self.Get(bitmap_id)
//nbm = Union(oldbm, bm)
self.storage.Store(int64(bitmap_id), self.db, self.slice, bm.(*Bitmap))
self.cache_it(bm, bitmap_id, GetCategory(bitmap_id))
self.cache_it(bm, bitmap_id, self.finder.GetCategory(bitmap_id))
}
func (self *Brand) TopN(src_bitmap IBitmap, n int) []Pair {
func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []int) []Pair {
self.rank_counter = 0
println("RANK")
self.Rank() // TODO: TERRIBLE REMOVE THIS ASAP
is := new(IntSet)
for _, v := range categories {
is.Add(v)
}
return self.TopNCat(src_bitmap, n, is)
}
func dump(r RankList, n int) {

11
index/commands.go
View file

@ -181,15 +181,16 @@ func (self *CmdGetList) Execute(f *Fragment) Calculation {
type CmdTopN struct {
*Responder
bitmap BitmapHandle
n int
bitmap BitmapHandle
n int
categories []int
}
func NewTopN(b BitmapHandle, n int) *CmdTopN {
return &CmdTopN{NewResponder("TopN"), b, n}
func NewTopN(b BitmapHandle, n int, categories []int) *CmdTopN {
return &CmdTopN{NewResponder("TopN"), b, n, categories}
}
func (self *CmdTopN) Execute(f *Fragment) Calculation {
return f.TopN(self.bitmap, self.n)
return f.TopN(self.bitmap, self.n, self.categories)
}
type CmdClear struct {

76
index/fragment_container.go
View file

@ -1,6 +1,8 @@
package index
import (
"database/sql"
"encoding/gob"
"errors"
"fmt"
@ -10,15 +12,70 @@ import (
"sync"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/golang/groupcache/lru"
)
type FragmentContainer struct {
fragments map[SUUID]*Fragment
finder *CategoryFinder
}
func lookup(stmt *sql.Stmt, tile_id uint64) int {
var category int
err := stmt.QueryRow(tile_id).Scan(&category) // WHERE number = 13
if err != nil {
log.Println(err.Error())
return 0
}
return category
}
type ICategoryFinder interface {
GetCategory(in uint64) int
}
type CategoryFinder struct {
in chan uint64
out chan int
}
func NewCategoryFinder() *CategoryFinder {
ptr := new(CategoryFinder)
ptr.in = make(chan uint64)
ptr.out = make(chan int)
return ptr
}
func (self *CategoryFinder) Start() {
connection := config.GetString("category_db_uri")
db, err := sql.Open("mysql", connection)
stmt, err := db.Prepare("select b.category_id from audience_brand b, accounts_tile t where b.id = t.object_id and t.id=?")
if err != nil {
panic(err.Error()) // proper error handling instead of panic in your app
}
defer stmt.Close()
for {
select {
case id := <-self.in:
category := lookup(stmt, id)
self.out <- category
}
}
}
func NewFragmentContainer() *FragmentContainer {
return &FragmentContainer{make(map[SUUID]*Fragment)}
f := new(FragmentContainer)
f.fragments = make(map[SUUID]*Fragment)
f.finder = NewCategoryFinder()
go f.finder.Start()
return f
}
func (self *FragmentContainer) GetCategory(in uint64) int {
self.finder.in <- in
return <-self.finder.out
}
type BitmapHandle uint64
@ -103,9 +160,9 @@ func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandl
return 0, errors.New("Invalid Bitmap Handle")
}
func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int) ([]Pair, error) {
func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categories []int) ([]Pair, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewTopN(bh, n)
request := NewTopN(bh, n, categories)
fragment.requestChan <- request
return request.Response().answer.([]Pair), nil
}
@ -168,8 +225,9 @@ func (self *FragmentContainer) Clear(frag_id SUUID) (bool, error) {
func (self *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) {
log.Println("ADD FRAGMENT", frame)
f := NewFragment(id, db, slice, frame)
f := NewFragment(id, db, slice, frame, self)
self.fragments[id] = f
go f.ServeFragment()
go f.Load()
}
@ -177,7 +235,7 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i
type Pilosa interface {
Get(id uint64) IBitmap
SetBit(id uint64, bit_pos uint64) bool
TopN(b IBitmap, n int) []Pair
TopN(b IBitmap, n int, categories []int) []Pair
Clear() bool
Store(bitmap_id uint64, bm IBitmap)
Stats() interface{}
@ -224,13 +282,13 @@ func getStorage(db string, slice int, frame string) Storage {
return nil
}
func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
func NewFragment(frag_id SUUID, db string, slice int, frame string, p ICategoryFinder) *Fragment {
var impl Pilosa
log.Println(fmt.Sprintf("XXXXXXXXXXXXXXXXXXXXXXXXXXX(%s)", frame))
switch frame {
case "brand":
log.Println("Brand")
impl = NewBrand(db, slice, getStorage(db, slice, frame), 50000, 45000, 100)
impl = NewBrand(db, slice, getStorage(db, slice, frame), 50000, 45000, 100, p)
default:
log.Println("General")
impl = NewGeneral(db, slice, getStorage(db, slice, frame))
@ -251,11 +309,11 @@ func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) {
return bm.(IBitmap), ok
}
func (self *Fragment) TopN(bitmap BitmapHandle, n int) []Pair {
func (self *Fragment) TopN(bitmap BitmapHandle, n int, categories []int) []Pair {
bm, ok := self.cache.Get(bitmap)
if ok {
return self.impl.TopN(bm.(*Bitmap), n)
return self.impl.TopN(bm.(*Bitmap), n, categories)
}
return nil
}

2
index/general.go
View file

@ -56,7 +56,7 @@ func (self *General) SetBit(bitmap_id uint64, bit_pos uint64) bool {
}
return change
}
func (self *General) TopN(b IBitmap, n int) []Pair {
func (self *General) TopN(b IBitmap, n int, categories []int) []Pair {
return nil
}