diff --git a/cruncher/cruncher.go b/cruncher/cruncher.go deleted file mode 100644 index cda812ed6..000000000 --- a/cruncher/cruncher.go +++ /dev/null @@ -1,26 +0,0 @@ -package cruncher - -import ( - "github.com/umbel/pilosa/core" - "github.com/umbel/pilosa/dispatch" - "github.com/umbel/pilosa/executor" - "github.com/umbel/pilosa/transport" -) - -type Cruncher struct { - *core.Service - close_chan chan bool -} - -func (cruncher *Cruncher) Run() { - cruncher.Service.Run() -} - -func NewCruncher() *Cruncher { - service := core.NewService() - cruncher := Cruncher{service, make(chan bool)} - cruncher.Transport = transport.NewTcpTransport(service) - cruncher.Dispatch = dispatch.NewDispatch(service) - cruncher.Executor = executor.NewExecutor(service) - return &cruncher -} diff --git a/cruncher/cruncher_test.go b/cruncher/cruncher_test.go deleted file mode 100644 index b9f4544f9..000000000 --- a/cruncher/cruncher_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package cruncher - -import ( - "testing" - - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" -) - -func TestCruncher(t *testing.T) { - Convey("Basic Cruncher Tests", t, func() { - spew.Dump("cruncher test") - }) -} diff --git a/db/topology_test.go b/db/topology_test.go index 1b6ff3cbf..54495ba01 100644 --- a/db/topology_test.go +++ b/db/topology_test.go @@ -1,26 +1,17 @@ -package db +package db_test import ( - "log" "testing" - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" + "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/util" ) -func TestTopology(t *testing.T) { - Convey("Basic DB structures", t, func() { - log.Println("topology test") +func TestCluster(t *testing.T) { + c := db.NewCluster() + d := c.GetOrCreateDatabase("main") - cluster := NewCluster() - database := cluster.GetOrCreateDatabase("main") - - frame := database.GetOrCreateFrame("general") - slice := database.GetOrCreateSlice(0) - - fragment_id := util.Id() - spew.Dump(fragment_id) - database.GetOrCreateFragment(frame, slice, fragment_id) - }) + f := d.GetOrCreateFrame("general") + sl := d.GetOrCreateSlice(0) + d.GetOrCreateFragment(f, sl, util.Id()) } diff --git a/hold/hold.go b/hold/hold.go index d212c6ac8..33fbe810a 100644 --- a/hold/hold.go +++ b/hold/hold.go @@ -23,6 +23,14 @@ type Holder struct { delchan chan delhold } +func NewHolder() *Holder { + return &Holder{ + data: make(map[GUID]holdchan), + getchan: make(chan gethold), + delchan: make(chan delhold), + } +} + func (self *Holder) DelChan(id *GUID) { log.Trace("Holder.DelChan", id) req := delhold{id} @@ -78,8 +86,3 @@ func (self *Holder) Run() { } } } - -func NewHolder() *Holder { - h := Holder{make(map[GUID]holdchan), make(chan gethold), make(chan delhold)} - return &h -} diff --git a/hold/hold_test.go b/hold/hold_test.go index ffe83929d..8ef3a1bd5 100644 --- a/hold/hold_test.go +++ b/hold/hold_test.go @@ -1,31 +1,41 @@ -package hold +package hold_test import ( "testing" "time" - . "github.com/smartystreets/goconvey/convey" + "github.com/umbel/pilosa/hold" "github.com/umbel/pilosa/util" ) -func TestHoldChan(t *testing.T) { +// Ensure hold can get a value that has been set. +func TestHold_Get(t *testing.T) { + h := hold.NewHolder() + go h.Run() - Hold := Holder{make(map[util.GUID]holdchan), make(chan gethold), make(chan delhold)} - go Hold.Run() - - Convey("set then get", t, func() { - id := util.RandomUUID() - Hold.Set(&id, "derp", 10) - derp, _ := Hold.Get(&id, 10) - So(derp, ShouldEqual, "derp") - }) - Convey("get then set", t, func() { - id := util.RandomUUID() - go func() { - Hold.Set(&id, "derpsy", 10) - }() - time.Sleep(time.Second / 10) - derp, _ := Hold.Get(&id, 10) - So(derp, ShouldEqual, "derpsy") - }) + id := util.RandomUUID() + h.Set(&id, "derp", 10) + if v, err := h.Get(&id, 10); err != nil { + t.Fatal(err) + } else if v != "derp" { + t.Fatalf("unexpected value: %v", v) + } +} + +// Ensure hold can wait for a value that has not been set yet. +func TestHold_Get_Delay(t *testing.T) { + h := hold.NewHolder() + go h.Run() + + id := util.RandomUUID() + go func() { + time.Sleep(500 * time.Millisecond) + h.Set(&id, "derpsy", 10) + }() + + if v, err := h.Get(&id, 10); err != nil { + t.Fatal(err) + } else if v != "derpsy" { + t.Fatalf("unexpected value: %v", v) + } } diff --git a/index/brand_test.go b/index/brand_test.go index 59aaddaa7..646d121a4 100644 --- a/index/brand_test.go +++ b/index/brand_test.go @@ -3,6 +3,8 @@ package index import ( "math/rand" "testing" + + "github.com/umbel/pilosa/util" ) var ( @@ -12,17 +14,19 @@ var ( ) func init() { - println("GO") size = 1000 + util.SetupStatsd() + // SetupCassandra() + membrand = NewBrand("db", "frame", 0, NewMemoryStorage(), size, size, 0) for i := uint64(0); i < uint64(size); i++ { membrand.SetBit(i, 0, 1) } - cassbrand = NewBrand("db", "frame", 0, NewCassStorage(), size, size, 0) - for i := uint64(0); i < uint64(size); i++ { - cassbrand.SetBit(i, 0, 1) - } + // cassbrand = NewBrand("db", "frame", 0, NewCassStorage(), size, size, 0) + // for i := uint64(0); i < uint64(size); i++ { + // cassbrand.SetBit(i, 0, 1) + // } } func benchmarkBrand(b *testing.B, size int, fill int, brand *Brand) { diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index f22dbaee4..577ebe173 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -1,158 +1,295 @@ -package index +package index_test -/* import ( "testing" + + "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa/util" ) -func TestFragment(t *testing.T) { - general := util.Hex_to_SUUID("1") - brand := util.Hex_to_SUUID("2") - dummy := NewFragmentContainer() - dummy.AddFragment("25", "general", 0, general) - dummy.AddFragment("25", "b.n", 0, brand) - - Convey("Get ", t, func() { - bh, _ := dummy.Get(general, 1234) - So(bh, ShouldNotEqual, 0) - }) - - Convey("SetBit/Count 1 1", t, func() { - bi1 := uint64(1234) - changed, _ := dummy.SetBit(general, bi1, 1, 0) - So(changed, ShouldEqual, true) - changed, _ = dummy.SetBit(general, bi1, 1, 0) - So(changed, ShouldEqual, false) - bh, _ := dummy.Get(general, bi1) - num, _ := dummy.Count(general, bh) - So(num, ShouldEqual, 1) - }) - - Convey("Union/Intersect/Difference", t, func() { - bi1 := uint64(1234) - bi2 := uint64(4321) - - dummy.SetBit(general, bi2, 65537, 0) //set_bit creates the bitmap - - bh1, _ := dummy.Get(general, bi1) - bh2, _ := dummy.Get(general, bi2) - - handles := []BitmapHandle{bh1, bh2} - result, _ := dummy.Union(general, handles) - - num, _ := dummy.Count(general, result) - So(num, ShouldEqual, 2) - result, _ = dummy.Intersect(general, handles) - - num, _ = dummy.Count(general, result) - So(num, ShouldEqual, 0) - - result, _ = dummy.Difference(general, handles) - num, _ = dummy.Count(general, result) - So(num, ShouldEqual, 1) - - }) - Convey("Union Empty", t, func() { - - bi1 := uint64(1234) - bh1, _ := dummy.Get(general, bi1) - bh2, _ := dummy.Empty(general) //set_bit creates the bitmap - - handles := []BitmapHandle{bh1, bh2} - result, _ := dummy.Union(general, handles) - - num, _ := dummy.Count(general, result) - - So(num, ShouldEqual, 1) - }) - Convey("Bytes", t, func() { - bi1 := uint64(1234) - bh1, _ := dummy.Get(general, bi1) - before, _ := dummy.Count(general, bh1) - - bytes, _ := dummy.GetBytes(general, bh1) - bh2, _ := dummy.FromBytes(general, bytes) - - after, _ := dummy.Count(general, bh2) - So(before, ShouldEqual, after) - }) - Convey("Empty ", t, func() { - bh, _ := dummy.Empty(general) - before, _ := dummy.Count(general, bh) - So(before, ShouldEqual, 0) - }) - - Convey("GetList ", t, func() { - bhs, _ := dummy.GetList(general, []uint64{1234, 4321, 789}) - result, _ := dummy.Union(general, bhs) - num, _ := dummy.Count(general, result) - So(num, ShouldEqual, 2) - }) - - Convey("Brand SetBit Small", t, func() { - bi1 := uint64(1029) - for x := uint64(0); x < 1000; x++ { - dummy.SetBit(brand, bi1, x, 0) - } - So(1, ShouldEqual, 1) - }) - - // Convey("Brand SetBit Big", t, func() { - // bi1 := uint64(1231) - // bi2 := uint64(1232) - // bi3 := uint64(1233) - // bi4 := uint64(1234) - // for x := uint64(0); x < 60000; x++ { - // if x < 100 { - // dummy.SetBit(brand, bi1, x) - // dummy.SetBit(brand, bi4, x) - // } - // if x < 500 { - // dummy.SetBit(brand, bi2, x) - // } - // if x%3 == 0 && x < 1000 { - // dummy.SetBit(brand, bi3, x) - // } - // if x > 700 && x < 1000 { - // dummy.SetBit(brand, bi4, x) - // } - // if x > 1000 { - // dummy.SetBit(brand, x, x) - // } - // } - // bh1, _ := dummy.Get(brand, bi1) - // // dummy.Rank() - // log.Println(dummy.TopN(brand, bh1, 4)) - // log.Println(dummy.Stats(brand)) - // So(1, ShouldEqual, 1) - // }) - - Convey("Brand TopN", t, func() { - dummy.SetBit(brand, uint64(1), 1, 2) - dummy.SetBit(brand, uint64(1), 2, 2) - dummy.SetBit(brand, uint64(1), 3, 2) - dummy.SetBit(brand, uint64(2), 1, 2) - dummy.SetBit(brand, uint64(2), 2, 2) - dummy.SetBit(brand, uint64(3), 1, 2) - bh1, _ := dummy.Get(brand, uint64(1)) - c := []uint64{2} - results, _ := dummy.TopN(brand, bh1, 4, c) - pair := results[0] - So(pair.Key, ShouldEqual, 1) - So(pair.Count, ShouldEqual, 3) - }) - Convey("Clear ", t, func() { - res, _ := dummy.Clear(general) - So(res, ShouldEqual, true) - }) - Convey("store ", t, func() { - b := uint64(1029) - compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" - dummy.LoadBitmap(brand, b, compressed, 0) - bh1, _ := dummy.Get(brand, b) - before, _ := dummy.Count(brand, bh1) - So(4096, ShouldEqual, before) - }) - +func init() { + index.Backend = "memory" +} + +// Ensure a fragment can be retrieved from the container. +func TestFragmentContainer_Get(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustClear(1) + + if bh, err := fc.Get(util.SUUID(1), 1234); err != nil { + t.Fatal(err) + } else if bh == 0 { + t.Fatal("expected non-zero bitmap handle") + } +} + +// Ensure a bit can be set on a bitmap. +func TestFragmentContainer_SetBit(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustClear(1) + + // Set a bit on the bitmap. + if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil { + t.Fatal(err) + } else if changed == false { + t.Fatal("expected change") + } + + // Set the same bit on the bitmap. No change should be indicated. + if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil { + t.Fatal(err) + } else if changed == true { + t.Fatal("expected no change") + } +} + +// Ensure the number of bits on a bitmap can be counted. +func TestFragmentContainer_Count(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, util.SUUID(1)) + + // Set a bit on the bitmap. + bi1 := uint64(1234) + if changed, err := fc.SetBit(util.SUUID(1), bi1, 1, 0); err != nil { + t.Fatal(err) + } else if changed == false { + t.Fatal("expected change") + } + + // Verify that one bit is set. + if bh, err := fc.Get(util.SUUID(1), bi1); err != nil { + t.Fatal(err) + } else if n, err := fc.Count(util.SUUID(1), bh); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal("unexpected count: %d", n) + } +} + +// Ensure the bits in two bitmaps can be unioned. +func TestFragmentContainer_Union(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + // Union the handles together. + if result, err := fc.Union(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 2 { + t.Fatalf("unexpected union bit count: %d", n) + } +} + +// Ensure unioning a bitmap with an empty bitmap returns a single bit count. +func TestFragmentContainer_Union_Empty(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + + // Union the handles together. + if result, err := fc.Union(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 1 { + t.Fatalf("unexpected empty union bit count: %d", n) + } +} + +// Ensure the bits in two bitmaps can be intersected. +func TestFragmentContainer_Intersect(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + // Intersect the handles together. + if result, err := fc.Intersect(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 0 { + t.Fatal("unexpected intersect bit count: %d", n) + } +} + +// Ensure the bits in two bitmaps can be diffed. +func TestFragmentContainer_Difference(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + // Compute the difference between the handles. + if result, err := fc.Difference(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 1 { + t.Fatalf("unexpected difference bit count: %d", err) + } +} + +// Ensure bitmaps can be marshaled and unmarshaled to bytes. +func TestFragmentContainer_Bytes(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + + // Count bits and marshal to bytes. + beforeN := fc.MustCount(1, 1234) + buf, err := fc.GetBytes(1, 1234) + if err != nil { + t.Fatal(err) + } + + // Marshal bytes back to a bitmap and re-count. + bh2, err := fc.FromBytes(1, buf) + if err != nil { + t.Fatal(err) + } + afterN := fc.MustCount(1, bh2) + + // Ensure the original bit count matches the new bitmap's bit count. + if beforeN != afterN { + t.Fatalf("unexpected bit count: before=%d, after=%d", beforeN, afterN) + } +} + +// Ensure an empty bitmap can be returned. +func TestFragmentContainer_Empty(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + bh, err := fc.Empty(1) + if err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, bh); n != 0 { + t.Fatalf("unexpected bit count: %d", n) + } +} + +// Ensure a list of bitmap handles can be returned. +func TestFragmentContainer_GetList(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + a, err := fc.GetList(1, []uint64{1234, 4321, 789}) + if err != nil { + t.Fatal(err) + } + + // Compute the union to ensure they're the correct bitmaps. + if res, err := fc.Union(1, a); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, res); n != 2 { + t.Fatalf("unexpected bit count: %d", n) + } +} + +// Ensure brand bitmaps can perform a small number of set bits. +func TestFragmentContainer_SetBit_Brand_Small(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "b.n", 0, 2) + for i := uint64(0); i < 1000; i++ { + fc.SetBit(2, 1029, i, 0) + } +} + +// Ensure the top n can be computed for a brand. +func TestFragmentContainer_TopN_Brand(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "b.n", 0, 2) + + // Set bits on the bitmap. + fc.MustSetBit(2, uint64(1), 1, 2) + fc.MustSetBit(2, uint64(1), 2, 2) + fc.MustSetBit(2, uint64(1), 3, 2) + fc.MustSetBit(2, uint64(2), 1, 2) + fc.MustSetBit(2, uint64(2), 2, 2) + fc.MustSetBit(2, uint64(3), 1, 2) + + // Retrieve the bitmap handle for bitmap 1. + bh := fc.MustGet(2, uint64(1)) + + // Compute the top-n. + if results, err := fc.TopN(2, bh, 4, []uint64{2}); err != nil { + t.Fatal(err) + } else if results[0].Key != 1 { + t.Fatalf("unexpected key: %d", results[0].Key) + } else if results[0].Count != 3 { + t.Fatalf("unexpected value: %d", results[0].Count) + } +} + +// Ensure a fragment can be cleared. +func TestFragmentContainer_Clear(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + + // Compute the top-n. + if res, err := fc.Clear(1); err != nil { + t.Fatal(err) + } else if res != true { + t.Fatalf("unexpected result: %v", res) + } +} + +// Ensure a fragment can be loaded from a compressed form. +func TestFragmentContainer_LoadBitmap(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "b.n", 0, 2) + + // Load a bitmap from compressed data. + buf := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" + fc.LoadBitmap(2, 1029, buf, 0) + + // Load and count bits. + if n := fc.MustCount(2, fc.MustGet(2, 1029)); n != 4096 { + t.Fatalf("unexpected bit count: %d", n) + } +} + +// FragementContainer is a test wrapper for index.FragmentContainer. +type FragmentContainer struct { + *index.FragmentContainer +} + +// NewFragmentContainer returns a new instance of FragmentContainer. +func NewFragmentContainer() *FragmentContainer { + return &FragmentContainer{index.NewFragmentContainer()} +} + +// MustGet retrieves a bitmap by id. Panic on error. +func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) index.BitmapHandle { + bh, err := fc.Get(frag_id, bitmap_id) + if err != nil { + panic(err) + } + return bh +} + +// MustSetBit sets a bit in a bitmap. Panic on error. +func (fc *FragmentContainer) MustSetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) bool { + changed, err := fc.SetBit(frag_id, bitmap_id, pos, category) + if err != nil { + panic(err) + } + return changed +} + +// MustClear clears a fragment. Panic on error. +func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool { + v, err := fc.Clear(fragmentID) + if err != nil { + panic(err) + } + return v +} + +// MustCount returns the number of set bits in a bitmap. Panic on error. +func (fc *FragmentContainer) MustCount(frag_id util.SUUID, bitmap index.BitmapHandle) uint64 { + v, err := fc.Count(frag_id, bitmap) + if err != nil { + panic(err) + } + return v } -*/ diff --git a/index/storage_cass.go b/index/storage_cass.go index d2170bec1..eaecd9ebe 100644 --- a/index/storage_cass.go +++ b/index/storage_cass.go @@ -55,12 +55,12 @@ func SetupCassandra() { func BuildSchema() { /* - "CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1" - create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true; - CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true; + "CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1" + create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true; + CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true; - CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) ) - " + CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) ) + " */ } @@ -136,7 +136,7 @@ func (self *CassandraStorage) runBatch(batch *gocql.Batch) { if batch != nil { err := self.db.ExecuteBatch(batch) if err != nil { - log.Warn("Batch ERROR", err) + log.Warn("Batch ERROR: ", err) } } } diff --git a/index/storage_mem.go b/index/storage_mem.go index bd5a0801a..84579ba07 100644 --- a/index/storage_mem.go +++ b/index/storage_mem.go @@ -15,16 +15,15 @@ func NewMemoryStorage() Storage { return obj } -func (c *MemoryStorage) BeginBatch() { -} -func (c *MemoryStorage) Close() { -} -func (c *MemoryStorage) EndBatch() { -} -func (c *MemoryStorage) FlushBatch() { -} -func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { +func (c *MemoryStorage) BeginBatch() {} +func (c *MemoryStorage) Close() {} + +func (c *MemoryStorage) EndBatch() {} + +func (c *MemoryStorage) FlushBatch() {} + +func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { key := fmt.Sprintf("%d:%s:%s:%d", bitmap_id, db, frame, slice) bitmap, found := c.db[key] if !found { @@ -41,11 +40,12 @@ func (c *MemoryStorage) Store(bitmap_id uint64, db string, frame string, slice i func (c *MemoryStorage) StoreBlock(bitmap_id uint64, db string, frame string, slice int, filter uint64, chunk_key uint64, block_index int32, block uint64) error { //only use the cache and throw away everything - return nil } + func (self *MemoryStorage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, bchunk uint64, block_index int32, bblock, count uint64) { } + func (self *MemoryStorage) RemoveBit(id uint64, db string, frame string, slice int, filter uint64, chunk uint64, block_index int32, count uint64) { } diff --git a/index/storage_test.go b/index/storage_test.go index 617ff10f3..d6f3c83b4 100644 --- a/index/storage_test.go +++ b/index/storage_test.go @@ -20,19 +20,6 @@ func TestStorage(t *testing.T) { filter := 10 bitmap_id := uint64(999999) - // Convey("KV ", t, func() { - // storage, _ := NewKVStorage("/tmp/", 0, db) - // bm := storage.Fetch(bitmap_id, db, slice) - // SetBit(bm, 0) - // SetBit(bm, 1) - // SetBit(bm, 2) - // storage.Store(int64(bitmap_id), db, frame, slice, filter, bm.(*Bitmap)) - // bm2, _ := storage.Fetch(bitmap_id, db, slice) - // So(BitCount(bm), ShouldEqual, BitCount(bm2)) - // So(BitCount(bm), ShouldEqual, bm.Count()) - // So(BitCount(bm), ShouldEqual, 3) - // }) - c, err := net.DialTimeout("tcp", "127.0.0.1:9042", 100*time.Millisecond) if err != nil { fmt.Println("NO cassandra. Skipping test.") diff --git a/index/timeframe_test.go b/index/timeframe_test.go index 546f48b9d..51d8112cb 100644 --- a/index/timeframe_test.go +++ b/index/timeframe_test.go @@ -1,133 +1,134 @@ -package index +package index_test -/* import ( - "fmt" - "log" "testing" "time" - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" + "github.com/umbel/pilosa/index" ) -func getTime(id uint64, s string) { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, s) - log.Println("BEFORE") - - for i, v := range GetTimeIds(uint64(id), t1, YMDH) { - log.Println(i, v, s) +func TestGetRange_1h_0(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-08-11 14:00"), + MustParseTime("2014-08-11 16:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) } - log.Println("AFTER") } -func TestDemo(t *testing.T) { - Convey("Test ID", t, func() { - getTime(uint64(1), "2014-08-11 14:00") - So(1, ShouldEqual, 1) - }) -} -func TestTimeFrame(t *testing.T) { - Convey("Test 1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-08-11 14:00") - t2, _ := time.Parse(shortForm, "2014-08-11 16:00") - m := GetRange(t1, t2, uint64(1)) - getTime(uint64(1), "2014-08-11 14:00") - spew.Dump(m) - So(len(m), ShouldEqual, 2) - }) - if true { - return +func TestGetRange_1h_1(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 10:03"), + MustParseTime("2014-01-02 11:03"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) } - - Convey("Test ID", t, func() { - const shortForm = "2006-01-02 15:04" - x, _ := time.Parse(shortForm, "1970-01-01 00:00:00") - fmt.Println(x) - - m := GetTimeIds(uint64(15027), x, YMD) - spew.Dump(m) - fmt.Println("OK") - So(1, ShouldEqual, 1) - }) - Convey("Test 1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 10:03") - t2, _ := time.Parse(shortForm, "2014-01-02 11:03") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 1) - }) - Convey("Test 2H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 10:03") - t2, _ := time.Parse(shortForm, "2014-01-02 12:03") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 2) - }) - - Convey("Test 24H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 12:03") - t2, _ := time.Parse(shortForm, "2014-01-03 12:03") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 24) - }) - Convey("Test 1D", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 00:00") - t2, _ := time.Parse(shortForm, "2014-01-03 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 1) - }) - - Convey("Test 1D1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 00:00") - t2, _ := time.Parse(shortForm, "2014-01-03 01:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 2) - }) - - Convey("Test 1H1D", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 23:00") - t2, _ := time.Parse(shortForm, "2014-01-04 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 2) - }) - - Convey("Test 1H1D1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 23:00") - t2, _ := time.Parse(shortForm, "2014-01-04 01:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 3) - }) - - Convey("Test 1Y", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-01 00:00") - t2, _ := time.Parse(shortForm, "2015-01-01 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 1) - }) - Convey("Test 1H1D1M", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-30 23:00") - t2, _ := time.Parse(shortForm, "2014-03-01 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 3) - }) - - Convey("Test 1H1D1MD1H1", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-30 23:00") - t2, _ := time.Parse(shortForm, "2014-03-02 01:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 5) - }) - } -*/ + +func TestGetRange_2h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 10:03"), + MustParseTime("2014-01-02 12:03"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_24h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 12:03"), + MustParseTime("2014-01-03 12:03"), + uint64(1), + ); len(m) != 24 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1d(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 00:00"), + MustParseTime("2014-01-03 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1d1h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 00:00"), + MustParseTime("2014-01-03 01:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 23:00"), + MustParseTime("2014-01-04 00:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d1h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 23:00"), + MustParseTime("2014-01-04 01:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1y(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-01 00:00"), + MustParseTime("2015-01-01 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d1m(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-30 23:00"), + MustParseTime("2014-03-01 00:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d1m1d1h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-30 23:00"), + MustParseTime("2014-03-02 01:00"), + uint64(1), + ); len(m) != 5 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetTimeIds(t *testing.T) { + _ = index.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), index.YMD) +} + +// DefaultTimeLayout is the time layout used by the tests. +const DefaultTimeLayout = "2006-01-02 15:04" + +// MustParseTime parses value using DefaultTimeLayout. Panic on error. +func MustParseTime(value string) time.Time { + v, err := time.Parse(DefaultTimeLayout, value) + if err != nil { + panic(err) + } + return v +} diff --git a/util/id_test.go b/util/id_test.go new file mode 100644 index 000000000..8bd1aba2c --- /dev/null +++ b/util/id_test.go @@ -0,0 +1,49 @@ +package util + +import ( + "fmt" + "testing" +) + +// Ensure id can be parsed from string. +func TestId_Small(t *testing.T) { + if v := Hex_to_SUUID("1"); v != 1 { + t.Fatalf("unexpected SUUID: %v", v) + } +} + +// Ensure generated IDs are unique. +func TestId_Unique(t *testing.T) { + a, b := Id(), Id() + if a == b { + t.Fatalf("ids should be unique: %v != %v", a, b) + } +} + +// Ensure ids can be converted to and from hex. +func TestId_Hex(t *testing.T) { + a := Id() + b := Hex_to_SUUID(SUUID_to_Hex(a)) + if a != b { + t.Fatalf("ids not equal: %v != %v", a, b) + } +} + +// Ensure ids can be generated in sequence. +func TestId_Multiple(t *testing.T) { + for i := 0; i < 10; i++ { + println(SUUID_to_Hex(Id())) + } +} + +// Ensure a random UUID can be converted to a string. +func TestRandomUUID_String(t *testing.T) { + fmt.Println(RandomUUID().String()) +} + +func BenchmarkId(b *testing.B) { + // run the Fib function b.N times + for n := 0; n < b.N; n++ { + Id() + } +} diff --git a/util/util_test.go b/util/util_test.go deleted file mode 100644 index 7fdfa4fcc..000000000 --- a/util/util_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package util - -import ( - "fmt" - "testing" - - "github.com/gocql/gocql" - . "github.com/smartystreets/goconvey/convey" -) - -/* -var ( - array [1000000]int - muid = make(map[SUUID]int) - muuid = make(map[*GUID]int) - r int -) - -func init() { - for i, _ := range array { - muid[Id()] = i - id := util.RandomUUID() - muuid[&id] = i - } - -} -*/ -func TestId(t *testing.T) { - Convey("Test Small", t, func() { - s := "1" - //s := "0000000000000001" - //s := "000000000000001" - b2 := Hex_to_SUUID(s) - So(1, ShouldEqual, b2) - }) - Convey("Basic Usage", t, func() { - bc1 := Id() - println(SUUID_to_Hex(bc1)) - println(SUUID_to_Hex(bc1)) - bc2 := Id() - println(SUUID_to_Hex(bc2)) - So(bc1, ShouldNotEqual, bc2) - }) - Convey("Hex Encoded Usage", t, func() { - b1 := Id() - s := SUUID_to_Hex(b1) - b2 := Hex_to_SUUID(s) - So(b1, ShouldEqual, b2) - }) - Convey("Gen 10", t, func() { - for i := 0; i < 10; i++ { - bc1 := Id() - println(SUUID_to_Hex(bc1)) - } - So(1, ShouldEqual, 1) - }) - -} - -func BenchmarkId(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - Id() - } -} - -func BenchmarkUUID(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - gocql.RandomUUID() - } -} -func TestGUID(t *testing.T) { - fmt.Println(RandomUUID().String()) - -} - -/* -func BenchmarkLookupId(b *testing.B) { - x := Id() - for i := 0; i < b.N; i++ { - if a, found := muid[x]; found { - muid[x] = a + 1 - } - } -} -func BenchmarkLookupUUID(b *testing.B) { - x := util.RandomUUID() - for i := 0; i < b.N; i++ { - if a, found := muuid[&x]; found { - muuid[&x] = a + 1 - } - } -} -*/