scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 11:27:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

trying an alternate backend leveldb

Browse source
This commit is contained in:
Todd Gruben 2014-04-10 15:31:58 -05:00
parent f0893f228c
commit b1271e7f4b
8 changed files with 78 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
deps.json
View file

@ -9,6 +9,11 @@
"version": "35c98e8c885757fab1824daa77cc0e24bf30e76b",
"type": "git"
},
"github.com/cactus/go-statsd-client/statsd": {
"repo": "github.com/cactus/go-statsd-client",
"version": "912f30c35e9cdf51f50bae24f071e227cca152fb",
"type": "git"
},
"gkvlite": {
"repo": "github.com/steveyen/gkvlite",
"version": "master",
@ -24,6 +29,11 @@
"version": "44eda643c1ae69e866e491b1c935c5b22e42350e",
"type": "git"
},
"goleveldb": {
"repo": "github.com/syndtr/goleveldb/leveldb",
"version": "ff3719c6816e2cd194f05058452d660608e178ac",
"type": "git"
},
"google-crypto": {
"repo": "code.google.com/p/go.crypto/ssh",
"version": "7aa593ce8cea",
@ -83,11 +93,5 @@
"repo": "github.com/gorilla/websocket",
"version": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c",
"type": "git"
},
"github.com/cactus/go-statsd-client/statsd": {
"repo": "github.com/cactus/go-statsd-client",
"version": "912f30c35e9cdf51f50bae24f071e227cca152fb",
"type": "git"
}
}

2
index/brand.go
View file

@ -85,6 +85,7 @@ func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category uint64) {
if bm.Count() >= self.threshold_value {
self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category}
if len(self.bitmap_cache) > self.threshold_length {
self.Rank()
self.trim()
}
}
@ -309,6 +310,7 @@ func (self *Brand) Persist() error {
return err
}
defer w.Close()
defer self.storage.Close()
asize := len(self.bitmap_cache)
var list RankList

10
index/fragment_container.go
View file

@ -233,12 +233,16 @@ type Fragment struct {
exit chan *sync.WaitGroup
}
func getStorage(db string, slice int, frame string) Storage {
func getStorage(db string, slice int, frame string, fid SUUID) Storage {
storage_method := config.GetString("storage_backend")
switch storage_method {
default:
return NewMemoryStorage()
case "leveldb":
base_path := config.GetString("level_db_path")
full_dir := fmt.Sprintf("%s/%s/%d/%s/%s", base_path, db, slice, frame, SUUID_to_Hex(fid))
return NewLevelDBStorage(full_dir)
case "cassandra":
host := config.GetString("cassandra_host")
if host == "" {
@ -258,10 +262,10 @@ func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
log.Println(fmt.Sprintf("XXXXXXXXXXXXXXXXXXXXXXXXXXX(%s)", frame))
if strings.HasSuffix(frame, ".n") {
log.Println(frame + "TOP")
impl = NewBrand(db, frame, slice, getStorage(db, slice, frame), 50000, 45000, 100)
impl = NewBrand(db, frame, slice, getStorage(db, slice, frame, frag_id), 50000, 45000, 100)
} else {
log.Println(frame)
impl = NewGeneral(db, frame, slice, getStorage(db, slice, frame))
impl = NewGeneral(db, frame, slice, getStorage(db, slice, frame, frag_id))
}
f := new(Fragment)

1
index/general.go
View file

@ -99,6 +99,7 @@ func (self *General) Persist() error {
return err
}
defer w.Close()
defer self.storage.Close()
encoder := json.NewEncoder(w)
return encoder.Encode(self.keys)

1
index/storage.go
View file

@ -6,4 +6,5 @@ type Storage interface {
StoreBlock(id int64, db string, frame string, slice int, filter uint64, chunk int64, block_index int32, block int64) error
BeginBatch()
EndBatch()
Close()
}

2
index/storage_cass.go
View file

@ -53,6 +53,8 @@ func NewCassStorage(host, keyspace string) Storage {
return obj
}
func (c *CassandraStorage) Close() {
}
func (c *CassandraStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) {
var dumb = COUNTERMASK
last_key := int64(dumb)

2
index/storage_mem.go
View file

@ -18,6 +18,8 @@ func NewMemoryStorage() Storage {
func (c *MemoryStorage) BeginBatch() {
}
func (c *MemoryStorage) Close() {
}
func (c *MemoryStorage) EndBatch() {
}
func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) {

88
index/storage_test.go
View file

@ -2,12 +2,11 @@ package index
import (
"fmt"
"net"
"testing"
"time"
// "io/ioutil"
// "time"
"github.com/davecgh/go-spew/spew"
. "github.com/smartystreets/goconvey/convey"
)
@ -18,42 +17,61 @@ func TestStorage(t *testing.T) {
filter := 10
bitmap_id := uint64(1234)
/* Convey("KV ", t, func() {
storage, _ := NewKVStorage("/tmp/", 0, db)
bm := storage.Fetch(bitmap_id, db, slice)
SetBit(bm, 0)
SetBit(bm, 1)
SetBit(bm, 2)
storage.Store(int64(bitmap_id), db, frame, slice, filter, bm.(*Bitmap))
bm2, _ := storage.Fetch(bitmap_id, db, slice)
So(BitCount(bm), ShouldEqual, BitCount(bm2))
So(BitCount(bm), ShouldEqual, bm.Count())
So(BitCount(bm), ShouldEqual, 3)
storage, _ := NewKVStorage("/tmp/", 0, db)
bm := storage.Fetch(bitmap_id, db, slice)
SetBit(bm, 0)
SetBit(bm, 1)
SetBit(bm, 2)
storage.Store(int64(bitmap_id), db, frame, slice, filter, bm.(*Bitmap))
bm2, _ := storage.Fetch(bitmap_id, db, slice)
So(BitCount(bm), ShouldEqual, BitCount(bm2))
So(BitCount(bm), ShouldEqual, bm.Count())
So(BitCount(bm), ShouldEqual, 3)
})
})
c, err := net.DialTimeout("tcp", "127.0.0.1:9042", 100*time.Millisecond)
if err != nil {
fmt.Println("NO cassandra. Skipping test.")
} else {
c.Close()
Convey("cassandra", t, func() {
fmt.Println("GO")
storage := NewCassStorage("127.0.0.1", "hotbox")
fmt.Println("FETCH")
bm, _ := storage.Fetch(bitmap_id, db, frame, slice)
SetBit(bm, 0)
SetBit(bm, 1)
SetBit(bm, 2)
fmt.Println("STORE")
storage.Store(int64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap))
fmt.Println("FETCH")
bm2, _ := storage.Fetch(bitmap_id, db, frame, slice)
So(BitCount(bm), ShouldEqual, BitCount(bm2))
So(BitCount(bm), ShouldEqual, bm.Count())
So(BitCount(bm), ShouldEqual, 3)
})
*/
c, err := net.DialTimeout("tcp", "127.0.0.1:9042", 100*time.Millisecond)
if err != nil {
fmt.Println("NO cassandra. Skipping test.")
} else {
c.Close()
Convey("cassandra", t, func() {
fmt.Println("GO")
storage := NewCassStorage("127.0.0.1", "hotbox")
Convey("leveldb", t, func() {
storage := NewLevelDBStorage("./basic/one")
fmt.Println("FETCH")
bm, _ := storage.Fetch(bitmap_id, db, frame, slice)
SetBit(bm, 0)
SetBit(bm, 1)
SetBit(bm, 2)
fmt.Println("STORE")
storage.Store(int64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap))
fmt.Println("FETCH")
bm2, _ := storage.Fetch(bitmap_id, db, frame, slice)
So(BitCount(bm), ShouldEqual, BitCount(bm2))
So(BitCount(bm), ShouldEqual, bm.Count())
So(BitCount(bm), ShouldEqual, 3)
fmt.Println("FETCH")
bm, _ := storage.Fetch(bitmap_id, db, frame, slice)
spew.Dump(bm)
SetBit(bm, 0)
SetBit(bm, 1)
SetBit(bm, 2)
fmt.Println("STORE")
storage.Store(int64(bitmap_id), db, frame, slice, uint64(filter), bm.(*Bitmap))
//storage.FlushBatch()
fmt.Println("FETCH")
bm2, _ := storage.Fetch(bitmap_id, db, frame, slice)
So(BitCount(bm), ShouldEqual, BitCount(bm2))
So(BitCount(bm), ShouldEqual, bm.Count())
So(BitCount(bm), ShouldEqual, 3)
storage.Close()
})
}
})
}