From 7c08f8b97c00318d674ec636dabfed5b828ed94a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 18 Dec 2013 20:14:07 -0600 Subject: [PATCH] added file based storage backend --- deps.json | 5 ++++ index/bitmap.go | 13 ++++++-- index/bitmap_test.go | 3 +- index/general.go | 25 +++++++++------- index/server.go | 70 +++++++++++++++++++++---------------------- index/server_test.go | 3 +- index/storage_cass.go | 1 + index/storage_mem.go | 11 ++----- 8 files changed, 73 insertions(+), 58 deletions(-) diff --git a/deps.json b/deps.json index fc18b8318..a92ce3b19 100644 --- a/deps.json +++ b/deps.json @@ -9,6 +9,11 @@ "version": "8a4461a", "type": "git" }, + "gkvlite": { + "repo": "github.com/steveyen/gkvlite", + "version": "master", + "type": "git" + }, "goconvey": { "repo": "github.com/smartystreets/goconvey/convey", "version": "master", diff --git a/index/bitmap.go b/index/bitmap.go index 8a32f715b..bd73d86ff 100644 --- a/index/bitmap.go +++ b/index/bitmap.go @@ -17,6 +17,15 @@ const ( COUNTERMASK = uint64(0xffffffffffffffff) ) +var ( + COUNTER_KEY int64 +) + +func init() { + var dumb = COUNTERMASK + COUNTER_KEY = int64(dumb) +} + // type IntSet struct { set map[int]bool @@ -451,7 +460,7 @@ func deref(pos uint64) Address { return Address{ChunkKey, BlockIndex, bit_offset} } -func SetBit(b IBitmap, position uint64) bool { +func SetBit(b IBitmap, position uint64) (bool, *Chunk, Address) { //Chunk,Chunk_index,bit_offset :=deref(position) address := deref(position) @@ -467,7 +476,7 @@ func SetBit(b IBitmap, position uint64) bool { if data_changed { b.Inc() } - return data_changed + return data_changed, node, address } func BitCount(b IBitmap) uint64 { var total uint64 diff --git a/index/bitmap_test.go b/index/bitmap_test.go index 77f6c51d8..24f966beb 100644 --- a/index/bitmap_test.go +++ b/index/bitmap_test.go @@ -1,9 +1,9 @@ package index import ( - . "github.com/smartystreets/goconvey/convey" "testing" "time" + . "github.com/smartystreets/goconvey/convey" ) func TestBitmaps(t *testing.T) { @@ -15,6 +15,7 @@ func TestBitmaps(t *testing.T) { bc1 := BitCount(bm) bc2 := bm.Count() So(bc1, ShouldEqual, bc2) + So(bc1, ShouldEqual, 4096) }) Convey("function AND_NOT 1 and not 0 => true ", t, func() { bm1 := CreateRBBitmap() diff --git a/index/general.go b/index/general.go index 3b1a9d209..88621c9be 100644 --- a/index/general.go +++ b/index/general.go @@ -1,8 +1,6 @@ package index -import ( - "github.com/golang/groupcache/lru" -) +import "github.com/golang/groupcache/lru" type General struct { bitmap_cache *lru.Cache @@ -21,16 +19,23 @@ func NewGeneral(db string, slice int, s Storage) *General { } -func (f *General) Get(bitmap_id uint64) IBitmap { - bm, ok := f.bitmap_cache.Get(bitmap_id) +func (self *General) Get(bitmap_id uint64) IBitmap { + bm, ok := self.bitmap_cache.Get(bitmap_id) if ok { return bm.(*Bitmap) } - bm = f.storage.Fetch(bitmap_id, f.db, f.slice) - f.bitmap_cache.Add(bitmap_id, bm) + bm = self.storage.Fetch(bitmap_id, self.db, self.slice) + self.bitmap_cache.Add(bitmap_id, bm) return bm.(*Bitmap) } -func (f *General) SetBit(bitmap_id uint64, bit_pos uint64) bool { - bm := f.Get(bitmap_id) - return SetBit(bm, bit_pos) +func (self *General) SetBit(bitmap_id uint64, bit_pos uint64) bool { + bm := self.Get(bitmap_id) + change, chunk, address := SetBit(bm, bit_pos) + if change { + val := chunk.Value.Block[address.BlockIndex] + self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, int64(address.ChunkKey), int32(address.BlockIndex), int64(val)) + self.storage.StoreBlock(int64(bitmap_id), self.db, self.slice, COUNTER_KEY, 0, int64(bm.Count())) + + } + return change } diff --git a/index/server.go b/index/server.go index 37eec6c59..ef4d91a94 100644 --- a/index/server.go +++ b/index/server.go @@ -18,22 +18,22 @@ func NewFragmentContainer() *FragmentContainer { type BitmapHandle uint64 -func (a *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) { +func (self *FragmentContainer) GetFragment(frag_id SUUID) (*Fragment, bool) { //lock - c, v := a.fragments[frag_id] + c, v := self.fragments[frag_id] return c, v } -func (a *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { - if fragment, found := a.GetFragment(frag_id); found { +func (self *FragmentContainer) Intersect(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { + if fragment, found := self.GetFragment(frag_id); found { request := NewIntersect(bh) fragment.requestChan <- request return request.GetResponder().Response().answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle") } -func (a *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { - if fragment, found := a.GetFragment(frag_id); found { +func (self *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandle, error) { + if fragment, found := self.GetFragment(frag_id); found { request := NewUnion(bh) fragment.requestChan <- request return request.GetResponder().Response().answer.(BitmapHandle), nil @@ -41,16 +41,16 @@ func (a *FragmentContainer) Union(frag_id SUUID, bh []BitmapHandle) (BitmapHandl return 0, errors.New("Invalid Bitmap Handle") } -func (a *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandle, error) { - if fragment, found := a.GetFragment(frag_id); found { +func (self *FragmentContainer) Get(frag_id SUUID, bitmap_id uint64) (BitmapHandle, error) { + if fragment, found := self.GetFragment(frag_id); found { request := NewGet(bitmap_id) fragment.requestChan <- request return request.GetResponder().Response().answer.(BitmapHandle), nil } return 0, errors.New("Invalid Bitmap Handle") } -func (a *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, error) { - if fragment, found := a.GetFragment(frag_id); found { +func (self *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, error) { + if fragment, found := self.GetFragment(frag_id); found { request := NewCount(bitmap) fragment.requestChan <- request return request.GetResponder().Response().answer.(uint64), nil @@ -58,8 +58,8 @@ func (a *FragmentContainer) Count(frag_id SUUID, bitmap BitmapHandle) (uint64, e return 0, errors.New("Invalid Bitmap Handle") } -func (a *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64) (bool, error) { - if fragment, found := a.GetFragment(frag_id); found { +func (self *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64) (bool, error) { + if fragment, found := self.GetFragment(frag_id); found { request := NewSetBit(bitmap_id, pos) fragment.requestChan <- request return request.GetResponder().Response().answer.(bool), nil @@ -67,9 +67,9 @@ func (a *FragmentContainer) SetBit(frag_id SUUID, bitmap_id uint64, pos uint64) return false, errors.New("Invalid Bitmap Handle") } -func (a *FragmentContainer) AddFragment(frame string, db string, slice int, id SUUID) { +func (self *FragmentContainer) AddFragment(frame string, db string, slice int, id SUUID) { f := NewFragment(id, db, slice, frame) - a.fragments[id] = f + self.fragments[id] = f go f.ServeFragment() } @@ -97,62 +97,62 @@ func NewFragment(frag_id SUUID, db string, slice int, frame string) *Fragment { return f } -func (f *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) { - bm, ok := f.cache.Get(bitmap) +func (self *Fragment) getBitmap(bitmap BitmapHandle) (IBitmap, bool) { + bm, ok := self.cache.Get(bitmap) return bm.(IBitmap), ok } -func (f *Fragment) NewHandle(bitmap_id uint64) BitmapHandle { - bm := f.impl.Get(bitmap_id) - return f.AllocHandle(bm) +func (self *Fragment) NewHandle(bitmap_id uint64) BitmapHandle { + bm := self.impl.Get(bitmap_id) + return self.AllocHandle(bm) //given a bitmap_id return a newly allocated handle } -func (f *Fragment) AllocHandle(bm IBitmap) BitmapHandle { - handle := f.nextHandle() - f.cache.Add(handle, bm) +func (self *Fragment) AllocHandle(bm IBitmap) BitmapHandle { + handle := self.nextHandle() + self.cache.Add(handle, bm) return handle } -func (f *Fragment) nextHandle() BitmapHandle { +func (self *Fragment) nextHandle() BitmapHandle { millis := uint64(time.Now().UTC().UnixNano()) id := millis << (64 - 41) - id |= uint64(f.slice) << (64 - 41 - 13) - id |= f.counter % 1024 - f.counter += 1 + id |= uint64(self.slice) << (64 - 41 - 13) + id |= self.counter % 1024 + self.counter += 1 return BitmapHandle(id) } -func (f *Fragment) union(bitmaps []BitmapHandle) BitmapHandle { +func (self *Fragment) union(bitmaps []BitmapHandle) BitmapHandle { result := NewBitmap() for i, id := range bitmaps { - bm, _ := f.getBitmap(id) + bm, _ := self.getBitmap(id) if i == 0 { result = bm } else { result = Union(result, bm) } } - return f.AllocHandle(result) + return self.AllocHandle(result) } -func (f *Fragment) intersect(bitmaps []BitmapHandle) BitmapHandle { +func (self *Fragment) intersect(bitmaps []BitmapHandle) BitmapHandle { var result IBitmap for i, id := range bitmaps { - bm, _ := f.getBitmap(id) + bm, _ := self.getBitmap(id) if i == 0 { result = Clone(bm) } else { result = Intersection(result, bm) } } - return f.AllocHandle(result) + return self.AllocHandle(result) } -func (f *Fragment) ServeFragment() { +func (self *Fragment) ServeFragment() { for { - req := <-f.requestChan + req := <-self.requestChan start := time.Now() responder := req.GetResponder() - answer := req.Execute(f) + answer := req.Execute(self) delta := time.Since(start) /* var buffer bytes.Buffer diff --git a/index/server_test.go b/index/server_test.go index de1efc0d2..eb930ec36 100644 --- a/index/server_test.go +++ b/index/server_test.go @@ -11,7 +11,8 @@ import ( func TestServer(t *testing.T) { - id := util.Id() + //id := util.Id() + id := util.Hex_to_SUUID("1") dummy := NewFragmentContainer() dummy.AddFragment("general", "25", 0, id) diff --git a/index/storage_cass.go b/index/storage_cass.go index 4717f1acf..ee878f138 100644 --- a/index/storage_cass.go +++ b/index/storage_cass.go @@ -4,6 +4,7 @@ package index import ( "log" + "tux21b.org/v1/gocql" ) diff --git a/index/storage_mem.go b/index/storage_mem.go index 03473b6c2..00b4f13da 100644 --- a/index/storage_mem.go +++ b/index/storage_mem.go @@ -29,19 +29,12 @@ func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, slice int) IBitmap { } func (c *MemoryStorage) Store(bitmap_id int64, db string, slice int, bitmap *Bitmap) error { - key := fmt.Sprintf("%d:%s:%d", bitmap_id, db, slice) - c.db[key] = bitmap + //only use the cache and throw away everything return nil } func (c *MemoryStorage) StoreBlock(bitmap_id int64, db string, slice int, chunk_key int64, block_index int32, block int64) error { - bm := c.Fetch(uint64(bitmap_id), db, slice) - node := GetChunk(bm, uint64(chunk_key)) - if node == nil { - node = &Chunk{uint64(chunk_key), BlockArray{}} - bm.AddChunk(node) - } - node.Value.Block[block_index] = uint64(block) + //only use the cache and throw away everything return nil }