scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-11 23:31:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added file based storage backend

Browse source
This commit is contained in:
Todd Gruben 2013-12-18 20:14:07 -06:00
parent 05db8dcb7b
commit 7c08f8b97c
8 changed files with 73 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
deps.json
View file

@ -9,6 +9,11 @@
"version": "8a4461a",
"type": "git"
},
"gkvlite": {
"repo": "github.com/steveyen/gkvlite",
"version": "master",
"type": "git"
},
"goconvey": {
"repo": "github.com/smartystreets/goconvey/convey",
"version": "master",

13
index/bitmap.go
View file

@ -17,6 +17,15 @@ const (
COUNTERMASK = uint64(0xffffffffffffffff)
)
var (
COUNTER_KEY int64
)
func init() {
var dumb = COUNTERMASK
COUNTER_KEY = int64(dumb)
}
//
type IntSet struct {
set map[int]bool
@ -451,7 +460,7 @@ func deref(pos uint64) Address {
return Address{ChunkKey, BlockIndex, bit_offset}
}
func SetBit(b IBitmap, position uint64) bool {
func SetBit(b IBitmap, position uint64) (bool, *Chunk, Address) {
//Chunk,Chunk_index,bit_offset :=deref(position)
address := deref(position)
@ -467,7 +476,7 @@ func SetBit(b IBitmap, position uint64) bool {
if data_changed {
b.Inc()
}
return data_changed
return data_changed, node, address
}
func BitCount(b IBitmap) uint64 {
var total uint64

3
index/bitmap_test.go
View file

@ -1,9 +1,9 @@
package index
import (
. "github.com/smartystreets/goconvey/convey"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func TestBitmaps(t *testing.T) {
@ -15,6 +15,7 @@ func TestBitmaps(t *testing.T) {
bc1 := BitCount(bm)
bc2 := bm.Count()
So(bc1, ShouldEqual, bc2)
So(bc1, ShouldEqual, 4096)
})
Convey("function AND_NOT 1 and not 0 => true ", t, func() {
bm1 := CreateRBBitmap()

25
index/general.go
View file

@ -1,8 +1,6 @@
package index
import (
"github.com/golang/groupcache/lru"
)
import "github.com/golang/groupcache/lru"
type General struct {
bitmap_cache *lru.Cache
@ -21,16 +19,23 @@ func NewGeneral(db string, slice int, s Storage) *General {
}
func (f *General) Get(bitmap_id uint64) IBitmap {
bm, ok := f.bitmap_cache.Get(bitmap_id)
func (self *General) Get(bitmap_id uint64) IBitmap {
bm, ok := self.bitmap_cache.Get(bitmap_id)
if ok {
return bm.(*Bitmap)
}
bm = f.storage.Fetch(bitmap_id, f.db, f.slice)
f.bitmap_cache.Add(bitmap_id, bm)
bm = self.storage.Fetch(bitmap_id, self.db, self.slice)
self.bitmap_cache.Add(bitmap_id, bm)
return bm.(*Bitmap)
}
func (f *General) SetBit(bitmap_id uint64, bit_pos uint64) bool {
bm := f.Get(bitmap_id)
return SetBit(bm, bit_pos)
func (self *General) SetBit(bitmap_id uint64, bit_pos uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := SetBit(bm, bit_pos)
if change {
val := chunk.Value.Block[address.BlockIndex]
self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val))
self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count()))
}
return change
}

70
index/server.go
View file

@ -18,22 +18,22 @@ func NewFragmentContainer() *FragmentContainer {
type BitmapHandle uint64
func (a *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) {
func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) {
//lock
c, v := a.fragments[frag_id]
c, v := self.fragments[frag_id]
return c, v
}
func (a *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
if fragment, found := a.GetFragment(frag_id); found {
func (self *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewIntersect(bh)
fragment.requestChan <- request
return request.GetResponder().Response().answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle")
}
func (a *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
if fragment, found := a.GetFragment(frag_id); found {
func (self *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewUnion(bh)
fragment.requestChan <- request
return request.GetResponder().Response().answer.(BitmapHandle), nil
@ -41,16 +41,16 @@ func (a *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandl
return 0, errors.New("Invalid Bitmap Handle")
}
func (a *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandle, error) {
if fragment, found := a.GetFragment(frag_id); found {
func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandle, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewGet(bitmap_id)
fragment.requestChan <- request
return request.GetResponder().Response().answer.(BitmapHandle), nil
}
return 0, errors.New("Invalid Bitmap Handle")
}
func (a *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, error) {
if fragment, found := a.GetFragment(frag_id); found {
func (self *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewCount(bitmap)
fragment.requestChan <- request
return request.GetResponder().Response().answer.(uint64), nil
@ -58,8 +58,8 @@ func (a *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, e
return 0, errors.New("Invalid Bitmap Handle")
}
func (a *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64) (bool, error) {
if fragment, found := a.GetFragment(frag_id); found {
func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64) (bool, error) {
if fragment, found := self.GetFragment(frag_id); found {
request := NewSetBit(bitmap_id, pos)
fragment.requestChan <- request
return request.GetResponder().Response().answer.(bool), nil
@ -67,9 +67,9 @@ func (a *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64)
return false, errors.New("Invalid Bitmap Handle")
}
func (a *FragmentContainer) AddFragment(frame string, db string, slice int, id SUUID) {
func (self *FragmentContainer) AddFragment(frame string, db string, slice int, id SUUID) {
f := NewFragment(id, db, slice, frame)
a.fragments[id] = f
self.fragments[id] = f
go f.ServeFragment()
}
@ -97,62 +97,62 @@ func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment {
return f
}
func (f *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) {
bm, ok := f.cache.Get(bitmap)
func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) {
bm, ok := self.cache.Get(bitmap)
return bm.(IBitmap), ok
}
func (f *Fragment) NewHandle(bitmap_id uint64) BitmapHandle {
bm := f.impl.Get(bitmap_id)
return f.AllocHandle(bm)
func (self *Fragment) NewHandle(bitmap_id uint64) BitmapHandle {
bm := self.impl.Get(bitmap_id)
return self.AllocHandle(bm)
//given a bitmap_id return a newly allocated handle
}
func (f *Fragment) AllocHandle(bm IBitmap) BitmapHandle {
handle := f.nextHandle()
f.cache.Add(handle, bm)
func (self *Fragment) AllocHandle(bm IBitmap) BitmapHandle {
handle := self.nextHandle()
self.cache.Add(handle, bm)
return handle
}
func (f *Fragment) nextHandle() BitmapHandle {
func (self *Fragment) nextHandle() BitmapHandle {
millis := uint64(time.Now().UTC().UnixNano())
id := millis << (64 - 41)
id |= uint64(f.slice) << (64 - 41 - 13)
id |= f.counter % 1024
f.counter += 1
id |= uint64(self.slice) << (64 - 41 - 13)
id |= self.counter % 1024
self.counter += 1
return BitmapHandle(id)
}
func (f *Fragment) union(bitmaps []BitmapHandle) BitmapHandle {
func (self *Fragment) union(bitmaps []BitmapHandle) BitmapHandle {
result := NewBitmap()
for i, id := range bitmaps {
bm, _ := f.getBitmap(id)
bm, _ := self.getBitmap(id)
if i == 0 {
result = bm
} else {
result = Union(result, bm)
}
}
return f.AllocHandle(result)
return self.AllocHandle(result)
}
func (f *Fragment) intersect(bitmaps []BitmapHandle) BitmapHandle {
func (self *Fragment) intersect(bitmaps []BitmapHandle) BitmapHandle {
var result IBitmap
for i, id := range bitmaps {
bm, _ := f.getBitmap(id)
bm, _ := self.getBitmap(id)
if i == 0 {
result = Clone(bm)
} else {
result = Intersection(result, bm)
}
}
return f.AllocHandle(result)
return self.AllocHandle(result)
}
func (f *Fragment) ServeFragment() {
func (self *Fragment) ServeFragment() {
for {
req := <-f.requestChan
req := <-self.requestChan
start := time.Now()
responder := req.GetResponder()
answer := req.Execute(f)
answer := req.Execute(self)
delta := time.Since(start)
/*
var buffer bytes.Buffer

3
index/server_test.go
View file

@ -11,7 +11,8 @@ import (
func TestServer(t *testing.T) {
id := util.Id()
//id := util.Id()
id := util.Hex_to_SUUID("1")
dummy := NewFragmentContainer()
dummy.AddFragment("general", "25", 0, id)

1
index/storage_cass.go
View file

@ -4,6 +4,7 @@ package index
import (
"log"
"tux21b.org/v1/gocql"
)

11
index/storage_mem.go
View file

@ -29,19 +29,12 @@ func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, slice int) IBitmap {
}
func (c *MemoryStorage) Store(bitmap_id int64, db string, slice int, bitmap *Bitmap) error {
key := fmt.Sprintf("%d:%s:%d", bitmap_id, db, slice)
c.db[key] = bitmap
//only use the cache and throw away everything
return nil
}
func (c *MemoryStorage) StoreBlock(bitmap_id int64, db string, slice int, chunk_key int64, block_index int32, block int64) error {
bm := c.Fetch(uint64(bitmap_id), db, slice)
node := GetChunk(bm, uint64(chunk_key))
if node == nil {
node = &Chunk{uint64(chunk_key), BlockArray{}}
bm.AddChunk(node)
}
node.Value.Block[block_index] = uint64(block)
//only use the cache and throw away everything
return nil
}