From 802970bd80a2b216d48f592f9b90c54bc535fa4e Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 15 Aug 2015 20:39:38 -0600 Subject: [PATCH] Convert tests to standard library testing. This commit converts the goconvey tests to use the standard library testing format. The query pkg has not be converted yet because it is more difficult to convert. --- cruncher/cruncher.go | 26 -- cruncher/cruncher_test.go | 14 - db/topology_test.go | 25 +- hold/hold.go | 13 +- hold/hold_test.go | 52 ++-- index/brand_test.go | 14 +- index/fragment_container_test.go | 441 ++++++++++++++++++++----------- index/storage_cass.go | 12 +- index/storage_mem.go | 20 +- index/storage_test.go | 13 - index/timeframe_test.go | 243 ++++++++--------- util/id_test.go | 49 ++++ util/util_test.go | 95 ------- 13 files changed, 532 insertions(+), 485 deletions(-) delete mode 100644 cruncher/cruncher.go delete mode 100644 cruncher/cruncher_test.go create mode 100644 util/id_test.go delete mode 100644 util/util_test.go diff --git a/cruncher/cruncher.go b/cruncher/cruncher.go deleted file mode 100644 index cda812ed6..000000000 --- a/cruncher/cruncher.go +++ /dev/null @@ -1,26 +0,0 @@ -package cruncher - -import ( - "github.com/umbel/pilosa/core" - "github.com/umbel/pilosa/dispatch" - "github.com/umbel/pilosa/executor" - "github.com/umbel/pilosa/transport" -) - -type Cruncher struct { - *core.Service - close_chan chan bool -} - -func (cruncher *Cruncher) Run() { - cruncher.Service.Run() -} - -func NewCruncher() *Cruncher { - service := core.NewService() - cruncher := Cruncher{service, make(chan bool)} - cruncher.Transport = transport.NewTcpTransport(service) - cruncher.Dispatch = dispatch.NewDispatch(service) - cruncher.Executor = executor.NewExecutor(service) - return &cruncher -} diff --git a/cruncher/cruncher_test.go b/cruncher/cruncher_test.go deleted file mode 100644 index b9f4544f9..000000000 --- a/cruncher/cruncher_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package cruncher - -import ( - "testing" - - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" -) - -func TestCruncher(t *testing.T) { - Convey("Basic Cruncher Tests", t, func() { - spew.Dump("cruncher test") - }) -} diff --git a/db/topology_test.go b/db/topology_test.go index 1b6ff3cbf..54495ba01 100644 --- a/db/topology_test.go +++ b/db/topology_test.go @@ -1,26 +1,17 @@ -package db +package db_test import ( - "log" "testing" - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" + "github.com/umbel/pilosa/db" "github.com/umbel/pilosa/util" ) -func TestTopology(t *testing.T) { - Convey("Basic DB structures", t, func() { - log.Println("topology test") +func TestCluster(t *testing.T) { + c := db.NewCluster() + d := c.GetOrCreateDatabase("main") - cluster := NewCluster() - database := cluster.GetOrCreateDatabase("main") - - frame := database.GetOrCreateFrame("general") - slice := database.GetOrCreateSlice(0) - - fragment_id := util.Id() - spew.Dump(fragment_id) - database.GetOrCreateFragment(frame, slice, fragment_id) - }) + f := d.GetOrCreateFrame("general") + sl := d.GetOrCreateSlice(0) + d.GetOrCreateFragment(f, sl, util.Id()) } diff --git a/hold/hold.go b/hold/hold.go index d212c6ac8..33fbe810a 100644 --- a/hold/hold.go +++ b/hold/hold.go @@ -23,6 +23,14 @@ type Holder struct { delchan chan delhold } +func NewHolder() *Holder { + return &Holder{ + data: make(map[GUID]holdchan), + getchan: make(chan gethold), + delchan: make(chan delhold), + } +} + func (self *Holder) DelChan(id *GUID) { log.Trace("Holder.DelChan", id) req := delhold{id} @@ -78,8 +86,3 @@ func (self *Holder) Run() { } } } - -func NewHolder() *Holder { - h := Holder{make(map[GUID]holdchan), make(chan gethold), make(chan delhold)} - return &h -} diff --git a/hold/hold_test.go b/hold/hold_test.go index ffe83929d..8ef3a1bd5 100644 --- a/hold/hold_test.go +++ b/hold/hold_test.go @@ -1,31 +1,41 @@ -package hold +package hold_test import ( "testing" "time" - . "github.com/smartystreets/goconvey/convey" + "github.com/umbel/pilosa/hold" "github.com/umbel/pilosa/util" ) -func TestHoldChan(t *testing.T) { +// Ensure hold can get a value that has been set. +func TestHold_Get(t *testing.T) { + h := hold.NewHolder() + go h.Run() - Hold := Holder{make(map[util.GUID]holdchan), make(chan gethold), make(chan delhold)} - go Hold.Run() - - Convey("set then get", t, func() { - id := util.RandomUUID() - Hold.Set(&id, "derp", 10) - derp, _ := Hold.Get(&id, 10) - So(derp, ShouldEqual, "derp") - }) - Convey("get then set", t, func() { - id := util.RandomUUID() - go func() { - Hold.Set(&id, "derpsy", 10) - }() - time.Sleep(time.Second / 10) - derp, _ := Hold.Get(&id, 10) - So(derp, ShouldEqual, "derpsy") - }) + id := util.RandomUUID() + h.Set(&id, "derp", 10) + if v, err := h.Get(&id, 10); err != nil { + t.Fatal(err) + } else if v != "derp" { + t.Fatalf("unexpected value: %v", v) + } +} + +// Ensure hold can wait for a value that has not been set yet. +func TestHold_Get_Delay(t *testing.T) { + h := hold.NewHolder() + go h.Run() + + id := util.RandomUUID() + go func() { + time.Sleep(500 * time.Millisecond) + h.Set(&id, "derpsy", 10) + }() + + if v, err := h.Get(&id, 10); err != nil { + t.Fatal(err) + } else if v != "derpsy" { + t.Fatalf("unexpected value: %v", v) + } } diff --git a/index/brand_test.go b/index/brand_test.go index 59aaddaa7..646d121a4 100644 --- a/index/brand_test.go +++ b/index/brand_test.go @@ -3,6 +3,8 @@ package index import ( "math/rand" "testing" + + "github.com/umbel/pilosa/util" ) var ( @@ -12,17 +14,19 @@ var ( ) func init() { - println("GO") size = 1000 + util.SetupStatsd() + // SetupCassandra() + membrand = NewBrand("db", "frame", 0, NewMemoryStorage(), size, size, 0) for i := uint64(0); i < uint64(size); i++ { membrand.SetBit(i, 0, 1) } - cassbrand = NewBrand("db", "frame", 0, NewCassStorage(), size, size, 0) - for i := uint64(0); i < uint64(size); i++ { - cassbrand.SetBit(i, 0, 1) - } + // cassbrand = NewBrand("db", "frame", 0, NewCassStorage(), size, size, 0) + // for i := uint64(0); i < uint64(size); i++ { + // cassbrand.SetBit(i, 0, 1) + // } } func benchmarkBrand(b *testing.B, size int, fill int, brand *Brand) { diff --git a/index/fragment_container_test.go b/index/fragment_container_test.go index f22dbaee4..577ebe173 100644 --- a/index/fragment_container_test.go +++ b/index/fragment_container_test.go @@ -1,158 +1,295 @@ -package index +package index_test -/* import ( "testing" + + "github.com/umbel/pilosa/index" + "github.com/umbel/pilosa/util" ) -func TestFragment(t *testing.T) { - general := util.Hex_to_SUUID("1") - brand := util.Hex_to_SUUID("2") - dummy := NewFragmentContainer() - dummy.AddFragment("25", "general", 0, general) - dummy.AddFragment("25", "b.n", 0, brand) - - Convey("Get ", t, func() { - bh, _ := dummy.Get(general, 1234) - So(bh, ShouldNotEqual, 0) - }) - - Convey("SetBit/Count 1 1", t, func() { - bi1 := uint64(1234) - changed, _ := dummy.SetBit(general, bi1, 1, 0) - So(changed, ShouldEqual, true) - changed, _ = dummy.SetBit(general, bi1, 1, 0) - So(changed, ShouldEqual, false) - bh, _ := dummy.Get(general, bi1) - num, _ := dummy.Count(general, bh) - So(num, ShouldEqual, 1) - }) - - Convey("Union/Intersect/Difference", t, func() { - bi1 := uint64(1234) - bi2 := uint64(4321) - - dummy.SetBit(general, bi2, 65537, 0) //set_bit creates the bitmap - - bh1, _ := dummy.Get(general, bi1) - bh2, _ := dummy.Get(general, bi2) - - handles := []BitmapHandle{bh1, bh2} - result, _ := dummy.Union(general, handles) - - num, _ := dummy.Count(general, result) - So(num, ShouldEqual, 2) - result, _ = dummy.Intersect(general, handles) - - num, _ = dummy.Count(general, result) - So(num, ShouldEqual, 0) - - result, _ = dummy.Difference(general, handles) - num, _ = dummy.Count(general, result) - So(num, ShouldEqual, 1) - - }) - Convey("Union Empty", t, func() { - - bi1 := uint64(1234) - bh1, _ := dummy.Get(general, bi1) - bh2, _ := dummy.Empty(general) //set_bit creates the bitmap - - handles := []BitmapHandle{bh1, bh2} - result, _ := dummy.Union(general, handles) - - num, _ := dummy.Count(general, result) - - So(num, ShouldEqual, 1) - }) - Convey("Bytes", t, func() { - bi1 := uint64(1234) - bh1, _ := dummy.Get(general, bi1) - before, _ := dummy.Count(general, bh1) - - bytes, _ := dummy.GetBytes(general, bh1) - bh2, _ := dummy.FromBytes(general, bytes) - - after, _ := dummy.Count(general, bh2) - So(before, ShouldEqual, after) - }) - Convey("Empty ", t, func() { - bh, _ := dummy.Empty(general) - before, _ := dummy.Count(general, bh) - So(before, ShouldEqual, 0) - }) - - Convey("GetList ", t, func() { - bhs, _ := dummy.GetList(general, []uint64{1234, 4321, 789}) - result, _ := dummy.Union(general, bhs) - num, _ := dummy.Count(general, result) - So(num, ShouldEqual, 2) - }) - - Convey("Brand SetBit Small", t, func() { - bi1 := uint64(1029) - for x := uint64(0); x < 1000; x++ { - dummy.SetBit(brand, bi1, x, 0) - } - So(1, ShouldEqual, 1) - }) - - // Convey("Brand SetBit Big", t, func() { - // bi1 := uint64(1231) - // bi2 := uint64(1232) - // bi3 := uint64(1233) - // bi4 := uint64(1234) - // for x := uint64(0); x < 60000; x++ { - // if x < 100 { - // dummy.SetBit(brand, bi1, x) - // dummy.SetBit(brand, bi4, x) - // } - // if x < 500 { - // dummy.SetBit(brand, bi2, x) - // } - // if x%3 == 0 && x < 1000 { - // dummy.SetBit(brand, bi3, x) - // } - // if x > 700 && x < 1000 { - // dummy.SetBit(brand, bi4, x) - // } - // if x > 1000 { - // dummy.SetBit(brand, x, x) - // } - // } - // bh1, _ := dummy.Get(brand, bi1) - // // dummy.Rank() - // log.Println(dummy.TopN(brand, bh1, 4)) - // log.Println(dummy.Stats(brand)) - // So(1, ShouldEqual, 1) - // }) - - Convey("Brand TopN", t, func() { - dummy.SetBit(brand, uint64(1), 1, 2) - dummy.SetBit(brand, uint64(1), 2, 2) - dummy.SetBit(brand, uint64(1), 3, 2) - dummy.SetBit(brand, uint64(2), 1, 2) - dummy.SetBit(brand, uint64(2), 2, 2) - dummy.SetBit(brand, uint64(3), 1, 2) - bh1, _ := dummy.Get(brand, uint64(1)) - c := []uint64{2} - results, _ := dummy.TopN(brand, bh1, 4, c) - pair := results[0] - So(pair.Key, ShouldEqual, 1) - So(pair.Count, ShouldEqual, 3) - }) - Convey("Clear ", t, func() { - res, _ := dummy.Clear(general) - So(res, ShouldEqual, true) - }) - Convey("store ", t, func() { - b := uint64(1029) - compressed := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" - dummy.LoadBitmap(brand, b, compressed, 0) - bh1, _ := dummy.Get(brand, b) - before, _ := dummy.Count(brand, bh1) - So(4096, ShouldEqual, before) - }) - +func init() { + index.Backend = "memory" +} + +// Ensure a fragment can be retrieved from the container. +func TestFragmentContainer_Get(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustClear(1) + + if bh, err := fc.Get(util.SUUID(1), 1234); err != nil { + t.Fatal(err) + } else if bh == 0 { + t.Fatal("expected non-zero bitmap handle") + } +} + +// Ensure a bit can be set on a bitmap. +func TestFragmentContainer_SetBit(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustClear(1) + + // Set a bit on the bitmap. + if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil { + t.Fatal(err) + } else if changed == false { + t.Fatal("expected change") + } + + // Set the same bit on the bitmap. No change should be indicated. + if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil { + t.Fatal(err) + } else if changed == true { + t.Fatal("expected no change") + } +} + +// Ensure the number of bits on a bitmap can be counted. +func TestFragmentContainer_Count(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, util.SUUID(1)) + + // Set a bit on the bitmap. + bi1 := uint64(1234) + if changed, err := fc.SetBit(util.SUUID(1), bi1, 1, 0); err != nil { + t.Fatal(err) + } else if changed == false { + t.Fatal("expected change") + } + + // Verify that one bit is set. + if bh, err := fc.Get(util.SUUID(1), bi1); err != nil { + t.Fatal(err) + } else if n, err := fc.Count(util.SUUID(1), bh); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal("unexpected count: %d", n) + } +} + +// Ensure the bits in two bitmaps can be unioned. +func TestFragmentContainer_Union(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + // Union the handles together. + if result, err := fc.Union(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 2 { + t.Fatalf("unexpected union bit count: %d", n) + } +} + +// Ensure unioning a bitmap with an empty bitmap returns a single bit count. +func TestFragmentContainer_Union_Empty(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + + // Union the handles together. + if result, err := fc.Union(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 1 { + t.Fatalf("unexpected empty union bit count: %d", n) + } +} + +// Ensure the bits in two bitmaps can be intersected. +func TestFragmentContainer_Intersect(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + // Intersect the handles together. + if result, err := fc.Intersect(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 0 { + t.Fatal("unexpected intersect bit count: %d", n) + } +} + +// Ensure the bits in two bitmaps can be diffed. +func TestFragmentContainer_Difference(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + // Compute the difference between the handles. + if result, err := fc.Difference(1, []index.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, result); n != 1 { + t.Fatalf("unexpected difference bit count: %d", err) + } +} + +// Ensure bitmaps can be marshaled and unmarshaled to bytes. +func TestFragmentContainer_Bytes(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + + // Count bits and marshal to bytes. + beforeN := fc.MustCount(1, 1234) + buf, err := fc.GetBytes(1, 1234) + if err != nil { + t.Fatal(err) + } + + // Marshal bytes back to a bitmap and re-count. + bh2, err := fc.FromBytes(1, buf) + if err != nil { + t.Fatal(err) + } + afterN := fc.MustCount(1, bh2) + + // Ensure the original bit count matches the new bitmap's bit count. + if beforeN != afterN { + t.Fatalf("unexpected bit count: before=%d, after=%d", beforeN, afterN) + } +} + +// Ensure an empty bitmap can be returned. +func TestFragmentContainer_Empty(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + bh, err := fc.Empty(1) + if err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, bh); n != 0 { + t.Fatalf("unexpected bit count: %d", n) + } +} + +// Ensure a list of bitmap handles can be returned. +func TestFragmentContainer_GetList(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + fc.MustSetBit(1, 1234, 1, 0) + fc.MustSetBit(1, 4321, 65537, 0) + + a, err := fc.GetList(1, []uint64{1234, 4321, 789}) + if err != nil { + t.Fatal(err) + } + + // Compute the union to ensure they're the correct bitmaps. + if res, err := fc.Union(1, a); err != nil { + t.Fatal(err) + } else if n := fc.MustCount(1, res); n != 2 { + t.Fatalf("unexpected bit count: %d", n) + } +} + +// Ensure brand bitmaps can perform a small number of set bits. +func TestFragmentContainer_SetBit_Brand_Small(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "b.n", 0, 2) + for i := uint64(0); i < 1000; i++ { + fc.SetBit(2, 1029, i, 0) + } +} + +// Ensure the top n can be computed for a brand. +func TestFragmentContainer_TopN_Brand(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "b.n", 0, 2) + + // Set bits on the bitmap. + fc.MustSetBit(2, uint64(1), 1, 2) + fc.MustSetBit(2, uint64(1), 2, 2) + fc.MustSetBit(2, uint64(1), 3, 2) + fc.MustSetBit(2, uint64(2), 1, 2) + fc.MustSetBit(2, uint64(2), 2, 2) + fc.MustSetBit(2, uint64(3), 1, 2) + + // Retrieve the bitmap handle for bitmap 1. + bh := fc.MustGet(2, uint64(1)) + + // Compute the top-n. + if results, err := fc.TopN(2, bh, 4, []uint64{2}); err != nil { + t.Fatal(err) + } else if results[0].Key != 1 { + t.Fatalf("unexpected key: %d", results[0].Key) + } else if results[0].Count != 3 { + t.Fatalf("unexpected value: %d", results[0].Count) + } +} + +// Ensure a fragment can be cleared. +func TestFragmentContainer_Clear(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "general", 0, 1) + + // Compute the top-n. + if res, err := fc.Clear(1); err != nil { + t.Fatal(err) + } else if res != true { + t.Fatalf("unexpected result: %v", res) + } +} + +// Ensure a fragment can be loaded from a compressed form. +func TestFragmentContainer_LoadBitmap(t *testing.T) { + fc := NewFragmentContainer() + fc.AddFragment("25", "b.n", 0, 2) + + // Load a bitmap from compressed data. + buf := "H4sIAAAJbogA/2JmYWBR+9/IzMjI6pxRmpfN+L+JgZGJkdk7tZKRjYGRNSwxpzSV8X8LAwOD8v9moDIup5z85GzHoqLESpAwI1AjWITxfxtQjdj/ViZGRo7o2NLMvBIzE5Ag0BiGf4zq/5uYGBV+/IeCUQZWBiikNP83AQN1NKwIMRgYAAAAAP//AQAA//9U05AivAIAAA==" + fc.LoadBitmap(2, 1029, buf, 0) + + // Load and count bits. + if n := fc.MustCount(2, fc.MustGet(2, 1029)); n != 4096 { + t.Fatalf("unexpected bit count: %d", n) + } +} + +// FragementContainer is a test wrapper for index.FragmentContainer. +type FragmentContainer struct { + *index.FragmentContainer +} + +// NewFragmentContainer returns a new instance of FragmentContainer. +func NewFragmentContainer() *FragmentContainer { + return &FragmentContainer{index.NewFragmentContainer()} +} + +// MustGet retrieves a bitmap by id. Panic on error. +func (fc *FragmentContainer) MustGet(frag_id util.SUUID, bitmap_id uint64) index.BitmapHandle { + bh, err := fc.Get(frag_id, bitmap_id) + if err != nil { + panic(err) + } + return bh +} + +// MustSetBit sets a bit in a bitmap. Panic on error. +func (fc *FragmentContainer) MustSetBit(frag_id util.SUUID, bitmap_id uint64, pos uint64, category uint64) bool { + changed, err := fc.SetBit(frag_id, bitmap_id, pos, category) + if err != nil { + panic(err) + } + return changed +} + +// MustClear clears a fragment. Panic on error. +func (fc *FragmentContainer) MustClear(fragmentID util.SUUID) bool { + v, err := fc.Clear(fragmentID) + if err != nil { + panic(err) + } + return v +} + +// MustCount returns the number of set bits in a bitmap. Panic on error. +func (fc *FragmentContainer) MustCount(frag_id util.SUUID, bitmap index.BitmapHandle) uint64 { + v, err := fc.Count(frag_id, bitmap) + if err != nil { + panic(err) + } + return v } -*/ diff --git a/index/storage_cass.go b/index/storage_cass.go index d2170bec1..eaecd9ebe 100644 --- a/index/storage_cass.go +++ b/index/storage_cass.go @@ -55,12 +55,12 @@ func SetupCassandra() { func BuildSchema() { /* - "CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1" - create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true; - CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true; + "CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1" + create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true; + CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true; - CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) ) - " + CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) ) + " */ } @@ -136,7 +136,7 @@ func (self *CassandraStorage) runBatch(batch *gocql.Batch) { if batch != nil { err := self.db.ExecuteBatch(batch) if err != nil { - log.Warn("Batch ERROR", err) + log.Warn("Batch ERROR: ", err) } } } diff --git a/index/storage_mem.go b/index/storage_mem.go index bd5a0801a..84579ba07 100644 --- a/index/storage_mem.go +++ b/index/storage_mem.go @@ -15,16 +15,15 @@ func NewMemoryStorage() Storage { return obj } -func (c *MemoryStorage) BeginBatch() { -} -func (c *MemoryStorage) Close() { -} -func (c *MemoryStorage) EndBatch() { -} -func (c *MemoryStorage) FlushBatch() { -} -func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { +func (c *MemoryStorage) BeginBatch() {} +func (c *MemoryStorage) Close() {} + +func (c *MemoryStorage) EndBatch() {} + +func (c *MemoryStorage) FlushBatch() {} + +func (c *MemoryStorage) Fetch(bitmap_id uint64, db string, frame string, slice int) (IBitmap, uint64) { key := fmt.Sprintf("%d:%s:%s:%d", bitmap_id, db, frame, slice) bitmap, found := c.db[key] if !found { @@ -41,11 +40,12 @@ func (c *MemoryStorage) Store(bitmap_id uint64, db string, frame string, slice i func (c *MemoryStorage) StoreBlock(bitmap_id uint64, db string, frame string, slice int, filter uint64, chunk_key uint64, block_index int32, block uint64) error { //only use the cache and throw away everything - return nil } + func (self *MemoryStorage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, bchunk uint64, block_index int32, bblock, count uint64) { } + func (self *MemoryStorage) RemoveBit(id uint64, db string, frame string, slice int, filter uint64, chunk uint64, block_index int32, count uint64) { } diff --git a/index/storage_test.go b/index/storage_test.go index 617ff10f3..d6f3c83b4 100644 --- a/index/storage_test.go +++ b/index/storage_test.go @@ -20,19 +20,6 @@ func TestStorage(t *testing.T) { filter := 10 bitmap_id := uint64(999999) - // Convey("KV ", t, func() { - // storage, _ := NewKVStorage("/tmp/", 0, db) - // bm := storage.Fetch(bitmap_id, db, slice) - // SetBit(bm, 0) - // SetBit(bm, 1) - // SetBit(bm, 2) - // storage.Store(int64(bitmap_id), db, frame, slice, filter, bm.(*Bitmap)) - // bm2, _ := storage.Fetch(bitmap_id, db, slice) - // So(BitCount(bm), ShouldEqual, BitCount(bm2)) - // So(BitCount(bm), ShouldEqual, bm.Count()) - // So(BitCount(bm), ShouldEqual, 3) - // }) - c, err := net.DialTimeout("tcp", "127.0.0.1:9042", 100*time.Millisecond) if err != nil { fmt.Println("NO cassandra. Skipping test.") diff --git a/index/timeframe_test.go b/index/timeframe_test.go index 546f48b9d..51d8112cb 100644 --- a/index/timeframe_test.go +++ b/index/timeframe_test.go @@ -1,133 +1,134 @@ -package index +package index_test -/* import ( - "fmt" - "log" "testing" "time" - "github.com/davecgh/go-spew/spew" - . "github.com/smartystreets/goconvey/convey" + "github.com/umbel/pilosa/index" ) -func getTime(id uint64, s string) { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, s) - log.Println("BEFORE") - - for i, v := range GetTimeIds(uint64(id), t1, YMDH) { - log.Println(i, v, s) +func TestGetRange_1h_0(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-08-11 14:00"), + MustParseTime("2014-08-11 16:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) } - log.Println("AFTER") } -func TestDemo(t *testing.T) { - Convey("Test ID", t, func() { - getTime(uint64(1), "2014-08-11 14:00") - So(1, ShouldEqual, 1) - }) -} -func TestTimeFrame(t *testing.T) { - Convey("Test 1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-08-11 14:00") - t2, _ := time.Parse(shortForm, "2014-08-11 16:00") - m := GetRange(t1, t2, uint64(1)) - getTime(uint64(1), "2014-08-11 14:00") - spew.Dump(m) - So(len(m), ShouldEqual, 2) - }) - if true { - return +func TestGetRange_1h_1(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 10:03"), + MustParseTime("2014-01-02 11:03"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) } - - Convey("Test ID", t, func() { - const shortForm = "2006-01-02 15:04" - x, _ := time.Parse(shortForm, "1970-01-01 00:00:00") - fmt.Println(x) - - m := GetTimeIds(uint64(15027), x, YMD) - spew.Dump(m) - fmt.Println("OK") - So(1, ShouldEqual, 1) - }) - Convey("Test 1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 10:03") - t2, _ := time.Parse(shortForm, "2014-01-02 11:03") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 1) - }) - Convey("Test 2H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 10:03") - t2, _ := time.Parse(shortForm, "2014-01-02 12:03") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 2) - }) - - Convey("Test 24H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 12:03") - t2, _ := time.Parse(shortForm, "2014-01-03 12:03") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 24) - }) - Convey("Test 1D", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 00:00") - t2, _ := time.Parse(shortForm, "2014-01-03 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 1) - }) - - Convey("Test 1D1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 00:00") - t2, _ := time.Parse(shortForm, "2014-01-03 01:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 2) - }) - - Convey("Test 1H1D", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 23:00") - t2, _ := time.Parse(shortForm, "2014-01-04 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 2) - }) - - Convey("Test 1H1D1H", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-02 23:00") - t2, _ := time.Parse(shortForm, "2014-01-04 01:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 3) - }) - - Convey("Test 1Y", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-01 00:00") - t2, _ := time.Parse(shortForm, "2015-01-01 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 1) - }) - Convey("Test 1H1D1M", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-30 23:00") - t2, _ := time.Parse(shortForm, "2014-03-01 00:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 3) - }) - - Convey("Test 1H1D1MD1H1", t, func() { - const shortForm = "2006-01-02 15:04" - t1, _ := time.Parse(shortForm, "2014-01-30 23:00") - t2, _ := time.Parse(shortForm, "2014-03-02 01:00") - m := GetRange(t1, t2, uint64(1)) - So(len(m), ShouldEqual, 5) - }) - } -*/ + +func TestGetRange_2h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 10:03"), + MustParseTime("2014-01-02 12:03"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_24h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 12:03"), + MustParseTime("2014-01-03 12:03"), + uint64(1), + ); len(m) != 24 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1d(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 00:00"), + MustParseTime("2014-01-03 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1d1h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 00:00"), + MustParseTime("2014-01-03 01:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 23:00"), + MustParseTime("2014-01-04 00:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d1h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-02 23:00"), + MustParseTime("2014-01-04 01:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1y(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-01 00:00"), + MustParseTime("2015-01-01 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d1m(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-30 23:00"), + MustParseTime("2014-03-01 00:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetRange_1h1d1m1d1h(t *testing.T) { + if m := index.GetRange( + MustParseTime("2014-01-30 23:00"), + MustParseTime("2014-03-02 01:00"), + uint64(1), + ); len(m) != 5 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestGetTimeIds(t *testing.T) { + _ = index.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), index.YMD) +} + +// DefaultTimeLayout is the time layout used by the tests. +const DefaultTimeLayout = "2006-01-02 15:04" + +// MustParseTime parses value using DefaultTimeLayout. Panic on error. +func MustParseTime(value string) time.Time { + v, err := time.Parse(DefaultTimeLayout, value) + if err != nil { + panic(err) + } + return v +} diff --git a/util/id_test.go b/util/id_test.go new file mode 100644 index 000000000..8bd1aba2c --- /dev/null +++ b/util/id_test.go @@ -0,0 +1,49 @@ +package util + +import ( + "fmt" + "testing" +) + +// Ensure id can be parsed from string. +func TestId_Small(t *testing.T) { + if v := Hex_to_SUUID("1"); v != 1 { + t.Fatalf("unexpected SUUID: %v", v) + } +} + +// Ensure generated IDs are unique. +func TestId_Unique(t *testing.T) { + a, b := Id(), Id() + if a == b { + t.Fatalf("ids should be unique: %v != %v", a, b) + } +} + +// Ensure ids can be converted to and from hex. +func TestId_Hex(t *testing.T) { + a := Id() + b := Hex_to_SUUID(SUUID_to_Hex(a)) + if a != b { + t.Fatalf("ids not equal: %v != %v", a, b) + } +} + +// Ensure ids can be generated in sequence. +func TestId_Multiple(t *testing.T) { + for i := 0; i < 10; i++ { + println(SUUID_to_Hex(Id())) + } +} + +// Ensure a random UUID can be converted to a string. +func TestRandomUUID_String(t *testing.T) { + fmt.Println(RandomUUID().String()) +} + +func BenchmarkId(b *testing.B) { + // run the Fib function b.N times + for n := 0; n < b.N; n++ { + Id() + } +} diff --git a/util/util_test.go b/util/util_test.go deleted file mode 100644 index 7fdfa4fcc..000000000 --- a/util/util_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package util - -import ( - "fmt" - "testing" - - "github.com/gocql/gocql" - . "github.com/smartystreets/goconvey/convey" -) - -/* -var ( - array [1000000]int - muid = make(map[SUUID]int) - muuid = make(map[*GUID]int) - r int -) - -func init() { - for i, _ := range array { - muid[Id()] = i - id := util.RandomUUID() - muuid[&id] = i - } - -} -*/ -func TestId(t *testing.T) { - Convey("Test Small", t, func() { - s := "1" - //s := "0000000000000001" - //s := "000000000000001" - b2 := Hex_to_SUUID(s) - So(1, ShouldEqual, b2) - }) - Convey("Basic Usage", t, func() { - bc1 := Id() - println(SUUID_to_Hex(bc1)) - println(SUUID_to_Hex(bc1)) - bc2 := Id() - println(SUUID_to_Hex(bc2)) - So(bc1, ShouldNotEqual, bc2) - }) - Convey("Hex Encoded Usage", t, func() { - b1 := Id() - s := SUUID_to_Hex(b1) - b2 := Hex_to_SUUID(s) - So(b1, ShouldEqual, b2) - }) - Convey("Gen 10", t, func() { - for i := 0; i < 10; i++ { - bc1 := Id() - println(SUUID_to_Hex(bc1)) - } - So(1, ShouldEqual, 1) - }) - -} - -func BenchmarkId(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - Id() - } -} - -func BenchmarkUUID(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - gocql.RandomUUID() - } -} -func TestGUID(t *testing.T) { - fmt.Println(RandomUUID().String()) - -} - -/* -func BenchmarkLookupId(b *testing.B) { - x := Id() - for i := 0; i < b.N; i++ { - if a, found := muid[x]; found { - muid[x] = a + 1 - } - } -} -func BenchmarkLookupUUID(b *testing.B) { - x := util.RandomUUID() - for i := 0; i < b.N; i++ { - if a, found := muuid[&x]; found { - muuid[&x] = a + 1 - } - } -} -*/