From 85b1a73b733c23c8f5c827ca127af955a73bd6dd Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 23 Jun 2017 09:10:15 -0500 Subject: [PATCH 01/17] Refactor test utilities into importable package. Remove duplicate instances of test utilities from pilosa.ctl. Now subpackages such as pilosa.ctl may import test utilities. --- attr_test.go | 78 +--------- client_test.go | 49 +++--- cluster_test.go | 39 +---- ctl/backup_test.go | 144 +---------------- ctl/bench_test.go | 3 +- ctl/export_test.go | 16 +- ctl/import_test.go | 12 +- ctl/restore_test.go | 12 +- executor_test.go | 109 ++++++------- fragment_test.go | 163 +++---------------- frame_test.go | 89 +---------- handler_test.go | 369 ++++++++++++++------------------------------ holder_test.go | 149 +++--------------- index_test.go | 113 ++------------ stats_test.go | 43 +++--- test/attr.go | 75 +++++++++ test/cluster.go | 40 +++++ test/fragment.go | 127 +++++++++++++++ test/frame.go | 92 +++++++++++ test/handler.go | 144 +++++++++++++++++ test/holder.go | 115 ++++++++++++++ test/index.go | 91 +++++++++++ view_test.go | 5 +- 23 files changed, 1010 insertions(+), 1067 deletions(-) create mode 100644 test/attr.go create mode 100644 test/cluster.go create mode 100644 test/fragment.go create mode 100644 test/frame.go create mode 100644 test/handler.go create mode 100644 test/holder.go create mode 100644 test/index.go diff --git a/attr_test.go b/attr_test.go index d9884cedc..8253c8e1d 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,19 +15,15 @@ package pilosa_test import ( - "io/ioutil" - "os" "reflect" - "runtime" - "sync" "testing" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) // Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { - s := MustOpenAttrStore() + s := test.MustOpenAttrStore() defer s.Close() // Set attributes. @@ -56,7 +52,7 @@ func TestAttrStore_Attrs(t *testing.T) { // Ensure database returns a non-nil empty map if unset. func TestAttrStore_Attrs_Empty(t *testing.T) { - s := MustOpenAttrStore() + s := test.MustOpenAttrStore() defer s.Close() if m, err := s.Attrs(100); err != nil { @@ -68,7 +64,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) { // Ensure database can unset attributes if explicitly set to nil. func TestAttrStore_Attrs_Unset(t *testing.T) { - s := MustOpenAttrStore() + s := test.MustOpenAttrStore() defer s.Close() // Set attributes. @@ -88,7 +84,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) { // Ensure attribute block checksums can be returned. func TestAttrStore_Blocks(t *testing.T) { - s := MustOpenAttrStore() + s := test.MustOpenAttrStore() defer s.Close() // Set attributes. @@ -127,67 +123,3 @@ func TestAttrStore_Blocks(t *testing.T) { t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2]) } } - -// AttrStore represents a test wrapper for pilosa.AttrStore. -type AttrStore struct { - *pilosa.AttrStore -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore() *AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") - if err != nil { - panic(err) - } - f.Close() - os.Remove(f.Name()) - - return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())} -} - -func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - const n = 5 - for i := 0; i < n; i++ { - if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - - // Update attributes with an existing subset. - cpuN := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - for i := 0; i < cpuN; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < b.N/cpuN; j++ { - if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { - b.Fatal(err) - } - } - }() - } - wg.Wait() -} - -// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() *AttrStore { - s := NewAttrStore() - if err := s.Open(); err != nil { - panic(err) - } - return s -} - -// Close closes the database and removes the underlying data. -func (s *AttrStore) Close() error { - defer os.RemoveAll(s.Path()) - return s.AttrStore.Close() -} diff --git a/client_test.go b/client_test.go index d2716c711..f4b4548d9 100644 --- a/client_test.go +++ b/client_test.go @@ -25,15 +25,16 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/test" ) -func createCluster(c *pilosa.Cluster) ([]*Server, []*Holder) { +func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { numNodes := len(c.Nodes) - hldr := make([]*Holder, numNodes) - server := make([]*Server, numNodes) + hldr := make([]*test.Holder, numNodes) + server := make([]*test.Server, numNodes) for i := 0; i < numNodes; i++ { - hldr[i] = MustOpenHolder() - server[i] = NewServer() + hldr[i] = test.MustOpenHolder() + server[i] = test.NewServer() server[i].Handler.Host = server[i].Host() server[i].Handler.Cluster = c server[i].Handler.Cluster.Nodes[i].Host = server[i].Host() @@ -44,7 +45,7 @@ func createCluster(c *pilosa.Cluster) ([]*Server, []*Holder) { // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { - cluster := NewCluster(3) + cluster := test.NewCluster(3) s, hldr := createCluster(cluster) for i := 0; i < len(cluster.Nodes); i++ { @@ -197,17 +198,17 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Load bitmap into cache to ensure cache gets updated. f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) f.Row(0) - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder @@ -232,7 +233,7 @@ func TestClient_Import(t *testing.T) { // Ensure client can bulk import data to an inverse frame. func TestClient_ImportInverseEnabled(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -255,10 +256,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. f.Row(0) - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder @@ -287,7 +288,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { // Ensure client backup and restore a frame. func TestClient_BackupRestore(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) @@ -295,10 +296,10 @@ func TestClient_BackupRestore(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder @@ -335,7 +336,7 @@ func TestClient_BackupRestore(t *testing.T) { // Ensure client backup and restore a frame with inverse view. func TestClient_BackupInverseView(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -360,10 +361,10 @@ func TestClient_BackupInverseView(t *testing.T) { f.SetBit(100, 3) f.SetBit(100, SliceWidth-1) - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder @@ -392,15 +393,15 @@ func TestClient_BackupInverseView(t *testing.T) { // backup returns error with invalid view func TestClient_BackupInvalidView(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder @@ -416,7 +417,7 @@ func TestClient_BackupInvalidView(t *testing.T) { // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Set two bits on blocks 0 & 3. @@ -426,10 +427,10 @@ func TestClient_FragmentBlocks(t *testing.T) { // Set a bit on a different slice. hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder diff --git a/cluster_test.go b/cluster_test.go index 5e4668fe2..d6b9dd487 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -15,7 +15,6 @@ package pilosa_test import ( - "fmt" "math/rand" "reflect" "testing" @@ -24,6 +23,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/httpbroadcast" + "github.com/pilosa/pilosa/test" ) // Ensure the cluster can fairly distribute partitions across the nodes. @@ -34,7 +34,7 @@ func TestCluster_Owners(t *testing.T) { {Host: "serverB:1000"}, {Host: "serverC:1000"}, }, - Hasher: NewModHasher(), + Hasher: test.NewModHasher(), ReplicaN: 2, } @@ -134,43 +134,10 @@ func TestCluster_NodeStates(t *testing.T) { // Ensure OwnsSlices can find the actual slice list for node and index func TestCluster_OwnsSlices(t *testing.T) { - c := NewCluster(5) + c := test.NewCluster(5) slices := c.OwnsSlices("test", 10, "host2") if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { t.Fatalf("unexpected slices for node's index: %v", slices) } } - -// NewCluster returns a cluster with n nodes and uses a mod-based hasher. -func NewCluster(n int) *pilosa.Cluster { - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - - for i := 0; i < n; i++ { - c.Nodes = append(c.Nodes, &pilosa.Node{ - Host: fmt.Sprintf("host%d", i), - }) - } - - return c -} - -// ModHasher represents a simple, mod-based hashing. -type ModHasher struct{} - -// NewModHasher returns a new instance of ModHasher with n buckets. -func NewModHasher() *ModHasher { return &ModHasher{} } - -func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } - -// ConstHasher represents hash that always returns the same index. -type ConstHasher struct { - i int -} - -// NewConstHasher returns a new instance of ConstHasher that always returns i. -func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } - -func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 512093a20..1c292685b 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -17,13 +17,11 @@ package ctl import ( "bytes" "context" - "fmt" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" "io/ioutil" - "net/http/httptest" - "net/url" "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) func TestBackupCommand_FileRequired(t *testing.T) { @@ -43,13 +41,13 @@ func TestBackupCommand_Run(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder cm := NewBackupCommand(stdin, stdout, stderr) @@ -65,133 +63,3 @@ func TestBackupCommand_Run(t *testing.T) { t.Fatalf("Command not working, error: '%s'", err) } } - -// Server represents a test wrapper for httptest.Server. -type Server struct { - *httptest.Server - Handler *Handler -} - -// NewServer returns a test server running on a random port. -func NewServer() *Server { - s := &Server{ - Handler: NewHandler(), - } - s.Server = httptest.NewServer(s.Handler.Handler) - - // Update handler to use hostname. - s.Handler.Host = s.Host() - - // Handler test messages can no-op. - s.Handler.Broadcaster = pilosa.NopBroadcaster - // Create a default cluster on the handler - s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - - return s -} - -// Handler represents a test wrapper for pilosa.Handler. -type Handler struct { - *pilosa.Handler - Executor HandlerExecutor -} - -// NewHandler returns a new instance of Handler. -func NewHandler() *Handler { - h := &Handler{ - Handler: pilosa.NewHandler(), - } - h.Handler.Executor = &h.Executor - h.Handler.LogOutput = ioutil.Discard - - // Handler test messages can no-op. - h.Broadcaster = pilosa.NopBroadcaster - - return h -} - -// HandlerExecutor is a mock implementing pilosa.Handler.Executor. -type HandlerExecutor struct { - cluster *pilosa.Cluster - ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) -} - -func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } - -func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return c.ExecuteFn(ctx, index, query, slices, opt) -} - -// Host returns the hostname of the running server. -func (s *Server) Host() string { return MustParseURLHost(s.URL) } - -// MustParseURLHost parses rawurl and returns the hostname. Panic on error. -func MustParseURLHost(rawurl string) string { - u, err := url.Parse(rawurl) - if err != nil { - panic(err) - } - return u.Host -} - -func NewCluster(n int) *pilosa.Cluster { - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - - for i := 0; i < n; i++ { - c.Nodes = append(c.Nodes, &pilosa.Node{ - Host: fmt.Sprintf("host%d", i), - }) - } - - return c -} - -// ModHasher represents a simple, mod-based hashing. -type ModHasher struct{} - -// NewModHasher returns a new instance of ModHasher with n buckets. -func NewModHasher() *ModHasher { return &ModHasher{} } - -func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } - -// ConstHasher represents hash that always returns the same index. -type ConstHasher struct { - i int -} - -// NewConstHasher returns a new instance of ConstHasher that always returns i. -func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } - -func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } - -// Holder is a test wrapper for pilosa.Holder. -type Holder struct { - *pilosa.Holder - LogOutput bytes.Buffer -} - -// NewHolder returns a new instance of Holder with a temporary path. -func NewHolder() *Holder { - path, err := ioutil.TempDir("", "pilosa-") - if err != nil { - panic(err) - } - - h := &Holder{Holder: pilosa.NewHolder()} - h.Path = path - h.Holder.LogOutput = &h.LogOutput - - return h -} - -// MustOpenHolder creates and opens a holder at a temporary path. Panic on error. -func MustOpenHolder() *Holder { - h := NewHolder() - if err := h.Open(); err != nil { - panic(err) - } - return h -} diff --git a/ctl/bench_test.go b/ctl/bench_test.go index c607e3301..c3014e6ca 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -18,10 +18,11 @@ import ( "bytes" "context" "fmt" - "github.com/pilosa/pilosa" "io" "os" "testing" + + "github.com/pilosa/pilosa" ) func TestBenchCommand_InvalidOption(t *testing.T) { diff --git a/ctl/export_test.go b/ctl/export_test.go index 20fef5cea..433078374 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -16,11 +16,13 @@ package ctl import ( "bytes" - "github.com/pilosa/pilosa" - "golang.org/x/net/context" "net/http" "strings" "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" + "golang.org/x/net/context" ) func TestExportCommand_Validation(t *testing.T) { @@ -53,18 +55,18 @@ func TestExportCommand_Run(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewExportCommand(stdin, stdout, stderr) - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder cm.Host = s.Host() - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) cm.Index = "i" cm.Frame = "f" diff --git a/ctl/import_test.go b/ctl/import_test.go index e80a0ce5c..8c4d3793e 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -16,13 +16,15 @@ package ctl import ( "bytes" - "github.com/pilosa/pilosa" - "golang.org/x/net/context" "io" "io/ioutil" "net/http" "strings" "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" + "golang.org/x/net/context" ) func TestImportCommand_Validation(t *testing.T) { @@ -59,12 +61,12 @@ func TestImportCommand_Run(t *testing.T) { t.Fatal(err) } - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder cm.Host = s.Host() diff --git a/ctl/restore_test.go b/ctl/restore_test.go index 093ded22f..5fd5af1d8 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -17,11 +17,13 @@ package ctl import ( "bufio" "bytes" - "github.com/pilosa/pilosa" - "golang.org/x/net/context" "io" "io/ioutil" "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" + "golang.org/x/net/context" ) func TestRestoreCommand_FileRequired(t *testing.T) { @@ -41,13 +43,13 @@ func TestRestoreCommand_Run(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Host = s.Host() - s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder diff --git a/executor_test.go b/executor_test.go index b9b4c0bd7..6a36babbf 100644 --- a/executor_test.go +++ b/executor_test.go @@ -25,12 +25,13 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/test" ) // Ensure a bitmap query can be executed. func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Row", func(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) @@ -38,7 +39,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", MustParse(``+ @@ -62,14 +63,14 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { }) t.Run("Column", func(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", MustParse(``+ @@ -95,7 +96,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) @@ -103,7 +104,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { @@ -113,11 +114,11 @@ func TestExecutor_Execute_Difference(t *testing.T) { // Ensure an empty difference query behaves properly. func TestExecutor_Execute_Empty_Difference(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } @@ -125,7 +126,7 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) @@ -135,7 +136,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { @@ -145,10 +146,10 @@ func TestExecutor_Execute_Intersect(t *testing.T) { // Ensure an empty intersect query behaves properly. func TestExecutor_Execute_Empty_Intersect(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } @@ -156,7 +157,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) @@ -165,7 +166,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { @@ -175,11 +176,11 @@ func TestExecutor_Execute_Union(t *testing.T) { // Ensure an empty union query behaves properly. func TestExecutor_Execute_Empty_Union(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { @@ -189,13 +190,13 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { @@ -205,10 +206,10 @@ func TestExecutor_Execute_Count(t *testing.T) { // Ensure a set query can be executed. func TestExecutor_Execute_SetBit(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) if n := f.Row(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) @@ -236,7 +237,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { // Ensure a SetRowAttrs() query can be executed. func TestExecutor_Execute_SetRowAttrs(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Create frames. @@ -249,7 +250,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } @@ -273,9 +274,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) // Set bits for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -326,7 +327,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. @@ -338,7 +339,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) // Execute query. - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -350,7 +351,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Ensure func TestExecutor_Execute_TopN_fill_small(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) @@ -372,7 +373,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) // Execute query. - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -384,7 +385,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Ensure a TopN() query with a source bitmap can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. @@ -407,7 +408,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -422,7 +423,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { //Ensure TopN handles Attribute filters func TestExecutor_Execute_TopN_Attr(t *testing.T) { // - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) @@ -431,7 +432,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -445,7 +446,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { //Ensure TopN handles Attribute filters with source bitmap func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) @@ -454,7 +455,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -467,9 +468,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -507,7 +508,7 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("Inverse", func(t *testing.T) { - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { @@ -518,10 +519,10 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure a remote query can return a bitmap. func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { - c := NewCluster(2) + c := test.NewCluster(2) // Create secondary server and update second cluster node. - s := NewServer() + s := test.NewServer() defer s.Close() c.Nodes[1].Host = s.Host() @@ -546,7 +547,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // Create local executor data. // The local node owns slice 1. - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) @@ -561,10 +562,10 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // Ensure a remote query can return a count. func TestExecutor_Execute_Remote_Count(t *testing.T) { - c := NewCluster(2) + c := test.NewCluster(2) // Create secondary server and update second cluster node. - s := NewServer() + s := test.NewServer() defer s.Close() c.Nodes[1].Host = s.Host() @@ -574,7 +575,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { } // Create local executor data. The local node owns slice 1. - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) @@ -590,11 +591,11 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Ensure a remote query can set bits on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - c := NewCluster(2) + c := test.NewCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. - s := NewServer() + s := test.NewServer() defer s.Close() c.Nodes[1].Host = s.Host() @@ -611,7 +612,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Create local executor data. - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.Holder = hldr.Holder @@ -636,11 +637,11 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Ensure a remote query can set bits on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - c := NewCluster(2) + c := test.NewCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. - s := NewServer() + s := test.NewServer() defer s.Close() c.Nodes[1].Host = s.Host() @@ -657,7 +658,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Create local executor data. - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.Holder = hldr.Holder @@ -684,10 +685,10 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Ensure a remote query can return a top-n query. func TestExecutor_Execute_Remote_TopN(t *testing.T) { - c := NewCluster(2) + c := test.NewCluster(2) // Create secondary server and update second cluster node. - s := NewServer() + s := test.NewServer() defer s.Close() c.Nodes[1].Host = s.Host() @@ -725,7 +726,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { } // Create local executor data on slice 2 & 4. - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.Holder = hldr.Holder hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) @@ -745,9 +746,9 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Ensure executor returns an error if too many writes are in a single request. func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) diff --git a/fragment_test.go b/fragment_test.go index f6b36f11e..bd1416257 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -17,14 +17,13 @@ package pilosa_test import ( "bytes" "flag" - "io/ioutil" "math" - "os" "reflect" "testing" "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) // Test flags @@ -37,7 +36,7 @@ const SliceWidth = pilosa.SliceWidth // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. @@ -68,7 +67,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. @@ -95,7 +94,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. @@ -124,7 +123,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. @@ -153,7 +152,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) @@ -175,7 +174,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. @@ -205,7 +204,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() // Create an intersecting input row. @@ -236,7 +235,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() // Create an intersecting input row. @@ -274,7 +273,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() // Set bits on various rows. @@ -299,7 +298,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { cacheSize := uint32(3) // Create Index. - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create frame. @@ -322,9 +321,9 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Close the storage so we can re-open it without encountering a flock. frag.Close() - f := &Fragment{ + f := &test.Fragment{ Fragment: frag, - RowAttrStore: MustOpenAttrStore(), + RowAttrStore: test.MustOpenAttrStore(), } f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore if err := f.Open(); err != nil { @@ -362,7 +361,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Retrieve checksum and set bits. @@ -381,7 +380,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Retrieve initial checksum. @@ -419,7 +418,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set bits on a different block. @@ -437,7 +436,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) defer f.Close() // Set bits on the fragment. @@ -469,7 +468,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create frame. @@ -522,7 +521,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f0.Close() // Set and then clear bits on the fragment. @@ -547,7 +546,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f1 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -596,7 +595,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() f.MaxOpN = math.MaxInt32 @@ -626,124 +625,8 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { } } -// Fragment is a test wrapper for pilosa.Fragment. -type Fragment struct { - *pilosa.Fragment - RowAttrStore *AttrStore -} - -// NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { - file, err := ioutil.TempFile("", "pilosa-fragment-") - if err != nil { - panic(err) - } - file.Close() - - f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice), - RowAttrStore: MustOpenAttrStore(), - } - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore - return f -} - -// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { - if cacheType == "" { - cacheType = pilosa.DefaultCacheType - } - f := NewFragment(index, frame, view, slice, cacheType) - - if err := f.Open(); err != nil { - panic(err) - } - return f -} - -// Close closes the fragment and removes all underlying data. -func (f *Fragment) Close() error { - defer os.Remove(f.Path()) - defer os.Remove(f.CachePath()) - defer f.RowAttrStore.Close() - return f.Fragment.Close() -} - -// Reopen closes the fragment and reopens it as a new instance. -func (f *Fragment) Reopen() error { - cacheType := f.Fragment.CacheType - path := f.Path() - if err := f.Fragment.Close(); err != nil { - return err - } - - f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore - if err := f.Open(); err != nil { - return err - } - return nil -} - -// MustSetBits sets bits on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// MustClearBits clears bits on a row. Panic on error. -func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// RowAttrStore provides simple storage for attributes. -type RowAttrStore struct { - attrs map[uint64]map[string]interface{} -} - -// NewRowAttrStore returns a new instance of RowAttrStore. -func NewRowAttrStore() *RowAttrStore { - return &RowAttrStore{ - attrs: make(map[uint64]map[string]interface{}), - } -} - -// RowAttrs returns the attributes set to a row id. -func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { - return s.attrs[id], nil -} - -// SetRowAttrs assigns a set of attributes to a row id. -func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { - s.attrs[id] = m -} - -// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk. -func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { - ipct := int(pct * 100) - for i := 0; i < SliceWidth*rowN; i++ { - if i%100 >= ipct { - continue - } - - rowIDs = append(rowIDs, uint64(i%SliceWidth)) - columnIDs = append(columnIDs, uint64(i/SliceWidth)) - } - return -} - func TestFragment_Tanimoto(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) @@ -766,7 +649,7 @@ func TestFragment_Tanimoto(t *testing.T) { } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) diff --git a/frame_test.go b/frame_test.go index fc5c01570..70e2a8b0b 100644 --- a/frame_test.go +++ b/frame_test.go @@ -16,16 +16,15 @@ package pilosa_test import ( "io/ioutil" - "os" "testing" - "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) // Ensure frame can open and retrieve a view. func TestFrame_CreateViewIfNotExists(t *testing.T) { - f := MustOpenFrame() + f := test.MustOpenFrame() defer f.Close() // Create view. @@ -51,7 +50,7 @@ func TestFrame_CreateViewIfNotExists(t *testing.T) { // Ensure frame can set its time quantum. func TestFrame_SetTimeQuantum(t *testing.T) { - f := MustOpenFrame() + f := test.MustOpenFrame() defer f.Close() // Set & retrieve time quantum. @@ -161,85 +160,3 @@ func TestFrame_RowLabelValidation(t *testing.T) { } } - -// Frame represents a test wrapper for pilosa.Frame. -type Frame struct { - *pilosa.Frame -} - -// NewFrame returns a new instance of Frame d/0. -func NewFrame() *Frame { - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - frame, err := pilosa.NewFrame(path, "i", "f") - if err != nil { - panic(err) - } - return &Frame{Frame: frame} -} - -// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. -func MustOpenFrame() *Frame { - f := NewFrame() - if err := f.Open(); err != nil { - panic(err) - } - return f -} - -// Close closes the frame and removes the underlying data. -func (f *Frame) Close() error { - defer os.RemoveAll(f.Path()) - return f.Frame.Close() -} - -// Reopen closes the index and reopens it. -func (f *Frame) Reopen() error { - var err error - if err := f.Frame.Close(); err != nil { - return err - } - - path, index, name := f.Path(), f.Index(), f.Name() - f.Frame, err = pilosa.NewFrame(path, index, name) - if err != nil { - return err - } - - if err := f.Open(); err != nil { - return err - } - return nil -} - -// MustSetBit sets a bit on the frame. Panic on error. -func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { - changed, err := f.SetBit(view, rowID, columnID, t) - if err != nil { - panic(err) - } - return changed -} - -// Ensure frame can set its cache -func TestFrame_SetCacheSize(t *testing.T) { - f := MustOpenFrame() - defer f.Close() - cacheSize := uint32(100) - - // Set & retrieve frame cache size. - if err := f.SetCacheSize(cacheSize); err != nil { - t.Fatal(err) - } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size: %d", q) - } - - // Reload frame and verify that it is persisted. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size (reopen): %d", q) - } -} diff --git a/handler_test.go b/handler_test.go index ab65894fb..b96b755fc 100644 --- a/handler_test.go +++ b/handler_test.go @@ -17,13 +17,10 @@ package pilosa_test import ( "bytes" "context" - "encoding/json" "errors" - "io" "io/ioutil" "net/http" "net/http/httptest" - "net/url" "reflect" "strings" "testing" @@ -32,19 +29,20 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/test" ) // Ensure the handler returns "not found" for invalid paths. func TestHandler_NotFound(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) if w.Code != http.StatusNotFound { t.Fatalf("invalid status: %d", w.Code) } @@ -52,7 +50,7 @@ func TestHandler_NotFound(t *testing.T) { // Ensure the handler can return the schema. func TestHandler_Schema(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) @@ -74,11 +72,11 @@ func TestHandler_Schema(t *testing.T) { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { @@ -88,8 +86,8 @@ func TestHandler_Schema(t *testing.T) { // Ensure the handler can return the status. func TestHandler_Status(t *testing.T) { - s := NewServer() - hldr := MustOpenHolder() + s := test.NewServer() + hldr := test.MustOpenHolder() defer s.Close() defer hldr.Close() @@ -112,14 +110,14 @@ func TestHandler_Status(t *testing.T) { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) h.StatusHandler = s s.Handler = h w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/status", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" { @@ -129,7 +127,7 @@ func TestHandler_Status(t *testing.T) { // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) @@ -140,11 +138,11 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { @@ -154,7 +152,7 @@ func TestHandler_MaxSlices(t *testing.T) { // Ensure the handler can return the maxslice map for the inverse views. func TestHandler_MaxSlices_Inverse(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) @@ -181,11 +179,11 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { @@ -195,11 +193,11 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { @@ -213,7 +211,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -223,11 +221,11 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { @@ -250,7 +248,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { } // Generate protobuf request. - req := MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) + req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) req.Header.Set("Content-Type", "application/x-protobuf") w := httptest.NewRecorder() @@ -263,14 +261,14 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Ensure the handler returns an error when parsing bad arguments. func TestHandler_Query_Args_Err(t *testing.T) { w := httptest.NewRecorder() - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { @@ -279,7 +277,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { } func TestHandler_Query_Params_Err(t *testing.T) { w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) + test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" { @@ -290,18 +288,18 @@ func TestHandler_Query_Params_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -311,18 +309,18 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -339,11 +337,11 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) @@ -352,7 +350,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" { @@ -362,7 +360,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap with column attributes as JSON. func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { - hldr := NewHolder() + hldr := test.NewHolder() defer hldr.Close() // Create index and set column attributes. @@ -375,9 +373,9 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -385,7 +383,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { @@ -395,11 +393,11 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as protobuf. func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) @@ -408,7 +406,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -433,7 +431,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf. func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { - hldr := NewHolder() + hldr := test.NewHolder() defer hldr.Close() // Create index and set column attributes. @@ -444,9 +442,9 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -463,7 +461,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) r.Header.Set("Content-Type", "application/x-protobuf") r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) @@ -500,11 +498,11 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ @@ -514,7 +512,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { @@ -524,11 +522,11 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ @@ -538,7 +536,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -555,18 +553,18 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) if w.Code != http.StatusInternalServerError { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { @@ -576,18 +574,18 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusInternalServerError { @@ -604,14 +602,14 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { // Ensure the handler returns "method not allowed" for non-POST queries. func TestHandler_Query_MethodNotAllowed(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i/query", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) if w.Code != http.StatusMethodNotAllowed { t.Fatalf("invalid status: %d", w.Code) } @@ -619,14 +617,14 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { // Ensure the handler returns an error if there is a parsing error.. func TestHandler_Query_ErrParse(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { @@ -636,10 +634,10 @@ func TestHandler_Query_ErrParse(t *testing.T) { // Ensure the handler can delete an index. func TestHandler_Index_Delete(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() @@ -649,7 +647,7 @@ func TestHandler_Index_Delete(t *testing.T) { } // Send request to delete index. - resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) + resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if err != nil { t.Fatal(err) } @@ -672,18 +670,18 @@ func TestHandler_Index_Delete(t *testing.T) { // Ensure handler can delete a frame. func TestHandler_DeleteFrame(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -695,15 +693,15 @@ func TestHandler_DeleteFrame(t *testing.T) { // Ensure handler can set the Index time quantum. func TestHandler_SetIndexTimeQuantum(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -715,7 +713,7 @@ func TestHandler_SetIndexTimeQuantum(t *testing.T) { // Ensure handler can set the frame time quantum. func TestHandler_SetFrameTimeQuantum(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Create frame. @@ -723,11 +721,11 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { t.Fatal(err) } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) + h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -739,10 +737,10 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { // Ensure the handler can return data in differing blocks for an index. func TestHandler_Index_AttrStore_Diff(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() @@ -773,7 +771,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { resp, err := http.Post( s.URL+"/index/i/attr/diff", "application/json", - strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`), + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) if err != nil { t.Fatal(err) @@ -781,17 +779,17 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { defer resp.Body.Close() // Read and validate body. - if body := string(MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } // Ensure the handler can return data in differing blocks for a frame. func TestHandler_Frame_AttrStore_Diff(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() @@ -823,7 +821,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { resp, err := http.Post( s.URL+"/index/i/frame/meta/attr/diff", "application/json", - strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`), + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) if err != nil { t.Fatal(err) @@ -831,17 +829,17 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer resp.Body.Close() // Read and validate body. - if body := string(MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } // Ensure the handler can backup a fragment and then restore it. func TestHandler_Fragment_BackupRestore(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() @@ -887,15 +885,15 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder w := httptest.NewRecorder() - r := MustNewHTTPRequest("GET", "/version", nil) + r := test.MustNewHTTPRequest("GET", "/version", nil) h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -906,16 +904,16 @@ func TestHandler_Version(t *testing.T) { // Ensure the handler can return a list of nodes for a fragment. func TestHandler_Fragment_Nodes(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(3) + h.Cluster = test.NewCluster(3) h.Cluster.ReplicaN = 2 w := httptest.NewRecorder() - r := MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) + r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -926,143 +924,16 @@ func TestHandler_Fragment_Nodes(t *testing.T) { // Ensure the handler can return expvars without panicking. func TestHandler_Expvars(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() - h.Cluster = NewCluster(1) + h := test.NewHandler() + h.Cluster = test.NewCluster(1) h.Holder = hldr.Holder w := httptest.NewRecorder() - r := MustNewHTTPRequest("GET", "/debug/vars", nil) + r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } - -// Handler represents a test wrapper for pilosa.Handler. -type Handler struct { - *pilosa.Handler - Executor HandlerExecutor -} - -// NewHandler returns a new instance of Handler. -func NewHandler() *Handler { - h := &Handler{ - Handler: pilosa.NewHandler(), - } - h.Handler.Executor = &h.Executor - h.Handler.LogOutput = ioutil.Discard - - // Handler test messages can no-op. - h.Broadcaster = pilosa.NopBroadcaster - - return h -} - -// HandlerExecutor is a mock implementing pilosa.Handler.Executor. -type HandlerExecutor struct { - cluster *pilosa.Cluster - ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) -} - -func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } - -func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return c.ExecuteFn(ctx, index, query, slices, opt) -} - -// Server represents a test wrapper for httptest.Server. -type Server struct { - *httptest.Server - Handler *Handler -} - -// NewServer returns a test server running on a random port. -func NewServer() *Server { - s := &Server{ - Handler: NewHandler(), - } - s.Server = httptest.NewServer(s.Handler.Handler) - - // Update handler to use hostname. - s.Handler.Host = s.Host() - - // Handler test messages can no-op. - s.Handler.Broadcaster = pilosa.NopBroadcaster - // Create a default cluster on the handler - s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - - return s -} - -// LocalStatus returns the state of the local node as well as the -// holder (indexes/frames) according to the local node. -func (s *Server) LocalStatus() (proto.Message, error) { - if s.Handler.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Host: s.Handler.Handler.Host, - State: pilosa.NodeStateUp, - Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.Host) - } - - return &ns, nil -} - -// ClusterStatus returns the NodeState for all nodes in the cluster. -func (s *Server) ClusterStatus() (proto.Message, error) { - // Assuming we are only testing this with one Node - // So just return its status - return s.LocalStatus() -} - -// HandleRemoteStatus just need to implement a nop to complete the Interface -func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } - -// Host returns the hostname of the running server. -func (s *Server) Host() string { return MustParseURLHost(s.URL) } - -// MustParseURLHost parses rawurl and returns the hostname. Panic on error. -func MustParseURLHost(rawurl string) string { - u, err := url.Parse(rawurl) - if err != nil { - panic(err) - } - return u.Host -} - -// MustNewHTTPRequest creates a new HTTP request. Panic on error. -func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { - req, err := http.NewRequest(method, urlStr, body) - if err != nil { - panic(err) - } - return req -} - -// MustMarshalJSON marshals v to JSON. Panic on error. -func MustMarshalJSON(v interface{}) []byte { - buf, err := json.Marshal(v) - if err != nil { - panic(err) - } - return buf -} - -// MustReadAll reads a reader into a buffer and returns it. Panic on error. -func MustReadAll(r io.Reader) []byte { - buf, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - return buf -} diff --git a/holder_test.go b/holder_test.go index b6864ffc9..108349bb2 100644 --- a/holder_test.go +++ b/holder_test.go @@ -15,9 +15,7 @@ package pilosa_test import ( - "bytes" "context" - "io/ioutil" "os" "path/filepath" "reflect" @@ -26,11 +24,12 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/test" ) func TestHolder_Open(t *testing.T) { t.Run("ErrIndexName", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { @@ -45,7 +44,7 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrIndexPermission", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { @@ -60,7 +59,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { @@ -74,7 +73,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { @@ -89,7 +88,7 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFramePermission", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -106,7 +105,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -122,7 +121,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -139,7 +138,7 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrViewPermission", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -158,7 +157,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -178,7 +177,7 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentStoragePermission", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -199,7 +198,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -220,7 +219,7 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentCachePermission", func(t *testing.T) { - h := MustOpenHolder() + h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -246,7 +245,7 @@ func TestHolder_Open(t *testing.T) { // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() // Write bits to separate indexes. @@ -279,16 +278,16 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { - cluster := NewCluster(2) + cluster := test.NewCluster(2) // Create a local holder. - hldr0 := MustOpenHolder() + hldr0 := test.MustOpenHolder() defer hldr0.Close() // Create a remote holder wrapped by an HTTP - hldr1 := MustOpenHolder() + hldr1 := test.MustOpenHolder() defer hldr1.Close() - s := NewServer() + s := test.NewServer() defer s.Close() s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -302,10 +301,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 cluster.Nodes[0].Host = "localhost:0" - cluster.Nodes[1].Host = MustParseURLHost(s.URL) + cluster.Nodes[1].Host = test.MustParseURLHost(s.URL) // Create frames on nodes. - for _, hldr := range []*Holder{hldr0, hldr1} { + for _, hldr := range []*test.Holder{hldr0, hldr1} { hldr.MustCreateFrameIfNotExists("i", "f") hldr.MustCreateFrameIfNotExists("i", "f0") hldr.MustCreateFrameIfNotExists("y", "z") @@ -365,7 +364,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } // Verify data is the same on both nodes. - for i, hldr := range []*Holder{hldr0, hldr1} { + for i, hldr := range []*test.Holder{hldr0, hldr1} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) @@ -393,109 +392,3 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } } } - -// Holder is a test wrapper for pilosa.Holder. -type Holder struct { - *pilosa.Holder - LogOutput bytes.Buffer -} - -// NewHolder returns a new instance of Holder with a temporary path. -func NewHolder() *Holder { - path, err := ioutil.TempDir("", "pilosa-") - if err != nil { - panic(err) - } - - h := &Holder{Holder: pilosa.NewHolder()} - h.Path = path - h.Holder.LogOutput = &h.LogOutput - - return h -} - -// MustOpenHolder creates and opens a holder at a temporary path. Panic on error. -func MustOpenHolder() *Holder { - h := NewHolder() - if err := h.Open(); err != nil { - panic(err) - } - return h -} - -// Close closes the holder and removes all underlying data. -func (h *Holder) Close() error { - defer os.RemoveAll(h.Path) - return h.Holder.Close() -} - -// Reopen closes the holder and instantiates and opens a new holder. -func (h *Holder) Reopen() error { - if err := h.Holder.Close(); err != nil { - return err - } - - path, logOutput := h.Path, h.Holder.LogOutput - h.Holder = pilosa.NewHolder() - h.Holder.Path = path - h.Holder.LogOutput = logOutput - if err := h.Holder.Open(); err != nil { - return err - } - - return nil -} - -// MustCreateIndexIfNotExists returns a given index. Panic on error. -func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { - idx, err := h.Holder.CreateIndexIfNotExists(index, opt) - if err != nil { - panic(err) - } - return &Index{Index: idx} -} - -// MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) - if err != nil { - panic(err) - } - return f -} - -// MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { - idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) - if err != nil { - panic(err) - } - v, err := f.CreateViewIfNotExists(view) - if err != nil { - panic(err) - } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} -} - -// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { - idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) - if err != nil { - panic(err) - } - v, err := f.CreateViewIfNotExists(view) - if err != nil { - panic(err) - } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} -} diff --git a/index_test.go b/index_test.go index 1d8a0330b..2c23f7a40 100644 --- a/index_test.go +++ b/index_test.go @@ -15,17 +15,16 @@ package pilosa_test import ( - "io/ioutil" - "os" "reflect" "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) // Ensure index can open and retrieve a frame. func TestIndex_CreateFrameIfNotExists(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create frame. @@ -53,7 +52,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Ensure time quantum can be set appropriately on a new frame. t.Run("TimeQuantum", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Set index time quantum. @@ -71,7 +70,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("Inherited", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Set index time quantum. @@ -92,7 +91,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Ensure frame can include range columns. t.Run("RangeEnabled", func(t *testing.T) { t.Run("OK", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create frame with schema and verify it exists. @@ -127,7 +126,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -139,7 +138,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -151,7 +150,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -164,7 +163,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("ErrFieldNameRequired", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -178,7 +177,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("ErrInvalidFieldType", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -192,7 +191,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("ErrInvalidFieldRange", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -209,7 +208,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Ensure frame cannot be created with a matching row label. t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() _, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel}) @@ -219,7 +218,7 @@ func TestIndex_CreateFrame(t *testing.T) { }) t.Run("Default", func(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil { t.Fatal(err) @@ -235,7 +234,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Ensure index can delete a frame. func TestIndex_DeleteFrame(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create frame. @@ -258,7 +257,7 @@ func TestIndex_DeleteFrame(t *testing.T) { // Ensure index can set the default time quantum. func TestIndex_SetTimeQuantum(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Set & retrieve time quantum. @@ -275,85 +274,3 @@ func TestIndex_SetTimeQuantum(t *testing.T) { t.Fatalf("unexpected quantum (reopen): %s", q) } } - -// Index represents a test wrapper for pilosa.Index. -type Index struct { - *pilosa.Index -} - -// NewIndex returns a new instance of Index. -func NewIndex() *Index { - path, err := ioutil.TempDir("", "pilosa-index-") - if err != nil { - panic(err) - } - index, err := pilosa.NewIndex(path, "i") - if err != nil { - panic(err) - } - return &Index{Index: index} -} - -// MustOpenIndex returns a new, opened index at a temporary path. Panic on error. -func MustOpenIndex() *Index { - index := NewIndex() - if err := index.Open(); err != nil { - panic(err) - } - return index -} - -// Close closes the index and removes the underlying data. -func (i *Index) Close() error { - defer os.RemoveAll(i.Path()) - return i.Index.Close() -} - -// Reopen closes the index and reopens it. -func (i *Index) Reopen() error { - var err error - if err := i.Index.Close(); err != nil { - return err - } - - path, name := i.Path(), i.Name() - i.Index, err = pilosa.NewIndex(path, name) - if err != nil { - return err - } - - if err := i.Open(); err != nil { - return err - } - return nil -} - -// CreateFrame creates a frame with the given options. -func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := i.Index.CreateFrame(name, opt) - if err != nil { - return nil, err - } - return &Frame{Frame: f}, nil -} - -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := i.Index.CreateFrameIfNotExists(name, opt) - if err != nil { - return nil, err - } - return &Frame{Frame: f}, nil -} - -// Ensure index can delete a frame. -func TestIndex_InvalidName(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-index-") - if err != nil { - panic(err) - } - index, err := pilosa.NewIndex(path, "ABC") - if index != nil { - t.Fatalf("unexpected index name %s", index) - } -} diff --git a/stats_test.go b/stats_test.go index 33e882a20..9ca81bcda 100644 --- a/stats_test.go +++ b/stats_test.go @@ -10,12 +10,13 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) // TestMultiStatClient_Expvar run the multistat client with exp var // since the EXPVAR data is stored in a global we should run these in one test function func TestMultiStatClient_Expvar(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() c := pilosa.NewExpvarStatsClient() @@ -73,7 +74,7 @@ func TestMultiStatClient_Expvar(t *testing.T) { } func TestStatsCount_TopN(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) @@ -83,7 +84,7 @@ func TestStatsCount_TopN(t *testing.T) { // Execute query. called := false - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "TopN" { @@ -107,13 +108,13 @@ func TestStatsCount_TopN(t *testing.T) { } func TestStatsCount_Bitmap(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) called := false - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "Bitmap" { @@ -137,14 +138,14 @@ func TestStatsCount_Bitmap(t *testing.T) { } func TestStatsCount_SetBitmapAttrs(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) called := false - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) frame := e.Holder.Frame("d", "f") if frame == nil { t.Fatal("frame not found") @@ -168,14 +169,14 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { } func TestStatsCount_SetProfileAttrs(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) called := false - e := NewExecutor(hldr.Holder, NewCluster(1)) + e := NewExecutor(hldr.Holder, test.NewCluster(1)) idx := e.Holder.Index("d") if idx == nil { t.Fatal("idex not found") @@ -200,9 +201,9 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } func TestStatsCount_CreateIndex(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() called := false @@ -216,17 +217,17 @@ func TestStatsCount_CreateIndex(t *testing.T) { return }, } - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", nil)) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", nil)) if !called { t.Error("Count isn't called") } } func TestStatsCount_DeleteIndex(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() @@ -245,17 +246,17 @@ func TestStatsCount_DeleteIndex(t *testing.T) { return }, } - http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if !called { t.Error("Count isn't called") } } func TestStatsCount_CreateFrame(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() @@ -277,17 +278,17 @@ func TestStatsCount_CreateFrame(t *testing.T) { return }, } - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil)) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil)) if !called { t.Error("Count isn't called") } } func TestStatsCount_DeleteFrame(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - s := NewServer() + s := test.NewServer() s.Handler.Holder = hldr.Holder defer s.Close() called := false @@ -309,7 +310,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { return }, } - http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader(""))) if !called { t.Error("Count isn't called") } diff --git a/test/attr.go b/test/attr.go new file mode 100644 index 000000000..620848084 --- /dev/null +++ b/test/attr.go @@ -0,0 +1,75 @@ +package test + +import ( + "io/ioutil" + "os" + "runtime" + "sync" + "testing" + + "github.com/pilosa/pilosa" +) + +// AttrStore represents a test wrapper for pilosa.AttrStore. +type AttrStore struct { + *pilosa.AttrStore +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore() *AttrStore { + f, err := ioutil.TempFile("", "pilosa-attr-") + if err != nil { + panic(err) + } + f.Close() + os.Remove(f.Name()) + + return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())} +} + +func BenchmarkAttrStore_Duplicate(b *testing.B) { + s := MustOpenAttrStore() + defer s.Close() + + // Set attributes. + const n = 5 + for i := 0; i < n; i++ { + if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + // Update attributes with an existing subset. + cpuN := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + for i := 0; i < cpuN; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < b.N/cpuN; j++ { + if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { + b.Fatal(err) + } + } + }() + } + wg.Wait() +} + +// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. +func MustOpenAttrStore() *AttrStore { + s := NewAttrStore() + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +// Close closes the database and removes the underlying data. +func (s *AttrStore) Close() error { + defer os.RemoveAll(s.Path()) + return s.AttrStore.Close() +} diff --git a/test/cluster.go b/test/cluster.go new file mode 100644 index 000000000..557aff559 --- /dev/null +++ b/test/cluster.go @@ -0,0 +1,40 @@ +package test + +import ( + "fmt" + + "github.com/pilosa/pilosa" +) + +// NewCluster returns a cluster with n nodes and uses a mod-based hasher. +func NewCluster(n int) *pilosa.Cluster { + c := pilosa.NewCluster() + c.ReplicaN = 1 + c.Hasher = NewModHasher() + + for i := 0; i < n; i++ { + c.Nodes = append(c.Nodes, &pilosa.Node{ + Host: fmt.Sprintf("host%d", i), + }) + } + + return c +} + +// ModHasher represents a simple, mod-based hashing. +type ModHasher struct{} + +// NewModHasher returns a new instance of ModHasher with n buckets. +func NewModHasher() *ModHasher { return &ModHasher{} } + +func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } + +// ConstHasher represents hash that always returns the same index. +type ConstHasher struct { + i int +} + +// NewConstHasher returns a new instance of ConstHasher that always returns i. +func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } + +func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } diff --git a/test/fragment.go b/test/fragment.go new file mode 100644 index 000000000..36ebba5bf --- /dev/null +++ b/test/fragment.go @@ -0,0 +1,127 @@ +package test + +import ( + "io/ioutil" + "os" + + "github.com/pilosa/pilosa" +) + +// SliceWidth is a helper reference to use when testing. +const SliceWidth = pilosa.SliceWidth + +// Fragment is a test wrapper for pilosa.Fragment. +type Fragment struct { + *pilosa.Fragment + RowAttrStore *AttrStore +} + +// NewFragment returns a new instance of Fragment with a temporary path. +func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + f := &Fragment{ + Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice), + RowAttrStore: MustOpenAttrStore(), + } + f.Fragment.CacheType = cacheType + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + return f +} + +// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. +func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { + if cacheType == "" { + cacheType = pilosa.DefaultCacheType + } + f := NewFragment(index, frame, view, slice, cacheType) + + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Close closes the fragment and removes all underlying data. +func (f *Fragment) Close() error { + defer os.Remove(f.Path()) + defer os.Remove(f.CachePath()) + defer f.RowAttrStore.Close() + return f.Fragment.Close() +} + +// Reopen closes the fragment and reopens it as a new instance. +func (f *Fragment) Reopen() error { + cacheType := f.Fragment.CacheType + path := f.Path() + if err := f.Fragment.Close(); err != nil { + return err + } + + f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) + f.Fragment.CacheType = cacheType + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + if err := f.Open(); err != nil { + return err + } + return nil +} + +// MustSetBits sets bits on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.SetBit(rowID, columnID); err != nil { + panic(err) + } + } +} + +// MustClearBits clears bits on a row. Panic on error. +func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.ClearBit(rowID, columnID); err != nil { + panic(err) + } + } +} + +// RowAttrStore provides simple storage for attributes. +type RowAttrStore struct { + attrs map[uint64]map[string]interface{} +} + +// NewRowAttrStore returns a new instance of RowAttrStore. +func NewRowAttrStore() *RowAttrStore { + return &RowAttrStore{ + attrs: make(map[uint64]map[string]interface{}), + } +} + +// RowAttrs returns the attributes set to a row id. +func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { + return s.attrs[id], nil +} + +// SetRowAttrs assigns a set of attributes to a row id. +func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { + s.attrs[id] = m +} + +// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk. +func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { + ipct := int(pct * 100) + for i := 0; i < SliceWidth*rowN; i++ { + if i%100 >= ipct { + continue + } + + rowIDs = append(rowIDs, uint64(i%SliceWidth)) + columnIDs = append(columnIDs, uint64(i/SliceWidth)) + } + return +} diff --git a/test/frame.go b/test/frame.go new file mode 100644 index 000000000..6234c802e --- /dev/null +++ b/test/frame.go @@ -0,0 +1,92 @@ +package test + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/pilosa/pilosa" +) + +// Frame represents a test wrapper for pilosa.Frame. +type Frame struct { + *pilosa.Frame +} + +// NewFrame returns a new instance of Frame d/0. +func NewFrame() *Frame { + path, err := ioutil.TempDir("", "pilosa-frame-") + if err != nil { + panic(err) + } + frame, err := pilosa.NewFrame(path, "i", "f") + if err != nil { + panic(err) + } + return &Frame{Frame: frame} +} + +// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. +func MustOpenFrame() *Frame { + f := NewFrame() + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Close closes the frame and removes the underlying data. +func (f *Frame) Close() error { + defer os.RemoveAll(f.Path()) + return f.Frame.Close() +} + +// Reopen closes the index and reopens it. +func (f *Frame) Reopen() error { + var err error + if err := f.Frame.Close(); err != nil { + return err + } + + path, index, name := f.Path(), f.Index(), f.Name() + f.Frame, err = pilosa.NewFrame(path, index, name) + if err != nil { + return err + } + + if err := f.Open(); err != nil { + return err + } + return nil +} + +// MustSetBit sets a bit on the frame. Panic on error. +func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { + changed, err := f.SetBit(view, rowID, columnID, t) + if err != nil { + panic(err) + } + return changed +} + +// Ensure frame can set its cache +func TestFrame_SetCacheSize(t *testing.T) { + f := MustOpenFrame() + defer f.Close() + cacheSize := uint32(100) + + // Set & retrieve frame cache size. + if err := f.SetCacheSize(cacheSize); err != nil { + t.Fatal(err) + } else if q := f.CacheSize(); q != cacheSize { + t.Fatalf("unexpected frame cache size: %d", q) + } + + // Reload frame and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.CacheSize(); q != cacheSize { + t.Fatalf("unexpected frame cache size (reopen): %d", q) + } +} diff --git a/test/handler.go b/test/handler.go new file mode 100644 index 000000000..70ac9e4e0 --- /dev/null +++ b/test/handler.go @@ -0,0 +1,144 @@ +package test + +import ( + "context" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" +) + +// Handler represents a test wrapper for pilosa.Handler. +type Handler struct { + *pilosa.Handler + Executor HandlerExecutor +} + +// NewHandler returns a new instance of Handler. +func NewHandler() *Handler { + h := &Handler{ + Handler: pilosa.NewHandler(), + } + h.Handler.Executor = &h.Executor + h.Handler.LogOutput = ioutil.Discard + + // Handler test messages can no-op. + h.Broadcaster = pilosa.NopBroadcaster + + return h +} + +// HandlerExecutor is a mock implementing pilosa.Handler.Executor. +type HandlerExecutor struct { + cluster *pilosa.Cluster + ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) +} + +func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } + +func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return c.ExecuteFn(ctx, index, query, slices, opt) +} + +// Server represents a test wrapper for httptest.Server. +type Server struct { + *httptest.Server + Handler *Handler +} + +// NewServer returns a test server running on a random port. +func NewServer() *Server { + s := &Server{ + Handler: NewHandler(), + } + s.Server = httptest.NewServer(s.Handler.Handler) + + // Update handler to use hostname. + s.Handler.Host = s.Host() + + // Handler test messages can no-op. + s.Handler.Broadcaster = pilosa.NopBroadcaster + // Create a default cluster on the handler + s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster.Nodes[0].Host = s.Host() + + return s +} + +// LocalStatus returns the state of the local node as well as the +// holder (indexes/frames) according to the local node. +func (s *Server) LocalStatus() (proto.Message, error) { + if s.Handler.Holder == nil { + return nil, errors.New("Server.Holder is nil") + } + + ns := internal.NodeStatus{ + Host: s.Handler.Handler.Host, + State: pilosa.NodeStateUp, + Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()), + } + + // Append Slice list per this Node's indexes + for _, index := range ns.Indexes { + index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.Host) + } + + return &ns, nil +} + +// ClusterStatus returns the NodeState for all nodes in the cluster. +func (s *Server) ClusterStatus() (proto.Message, error) { + // Assuming we are only testing this with one Node + // So just return its status + return s.LocalStatus() +} + +// HandleRemoteStatus just need to implement a nop to complete the Interface +func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } + +// Host returns the hostname of the running server. +func (s *Server) Host() string { return MustParseURLHost(s.URL) } + +// MustParseURLHost parses rawurl and returns the hostname. Panic on error. +func MustParseURLHost(rawurl string) string { + u, err := url.Parse(rawurl) + if err != nil { + panic(err) + } + return u.Host +} + +// MustNewHTTPRequest creates a new HTTP request. Panic on error. +func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + panic(err) + } + return req +} + +// MustMarshalJSON marshals v to JSON. Panic on error. +func MustMarshalJSON(v interface{}) []byte { + buf, err := json.Marshal(v) + if err != nil { + panic(err) + } + return buf +} + +// MustReadAll reads a reader into a buffer and returns it. Panic on error. +func MustReadAll(r io.Reader) []byte { + buf, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + return buf +} diff --git a/test/holder.go b/test/holder.go new file mode 100644 index 000000000..6bb346ec1 --- /dev/null +++ b/test/holder.go @@ -0,0 +1,115 @@ +package test + +import ( + "bytes" + "io/ioutil" + "os" + + "github.com/pilosa/pilosa" +) + +// Holder is a test wrapper for pilosa.Holder. +type Holder struct { + *pilosa.Holder + LogOutput bytes.Buffer +} + +// NewHolder returns a new instance of Holder with a temporary path. +func NewHolder() *Holder { + path, err := ioutil.TempDir("", "pilosa-") + if err != nil { + panic(err) + } + + h := &Holder{Holder: pilosa.NewHolder()} + h.Path = path + h.Holder.LogOutput = &h.LogOutput + + return h +} + +// MustOpenHolder creates and opens a holder at a temporary path. Panic on error. +func MustOpenHolder() *Holder { + h := NewHolder() + if err := h.Open(); err != nil { + panic(err) + } + return h +} + +// Close closes the holder and removes all underlying data. +func (h *Holder) Close() error { + defer os.RemoveAll(h.Path) + return h.Holder.Close() +} + +// Reopen closes the holder and instantiates and opens a new holder. +func (h *Holder) Reopen() error { + if err := h.Holder.Close(); err != nil { + return err + } + + path, logOutput := h.Path, h.Holder.LogOutput + h.Holder = pilosa.NewHolder() + h.Holder.Path = path + h.Holder.LogOutput = logOutput + if err := h.Holder.Open(); err != nil { + return err + } + + return nil +} + +// MustCreateIndexIfNotExists returns a given index. Panic on error. +func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { + idx, err := h.Holder.CreateIndexIfNotExists(index, opt) + if err != nil { + panic(err) + } + return &Index{Index: idx} +} + +// MustCreateFrameIfNotExists returns a given frame. Panic on error. +func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { + f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + if err != nil { + panic(err) + } + return f +} + +// MustCreateFragmentIfNotExists returns a given fragment. Panic on error. +func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + if err != nil { + panic(err) + } + v, err := f.CreateViewIfNotExists(view) + if err != nil { + panic(err) + } + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + panic(err) + } + return &Fragment{Fragment: frag} +} + +// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. +func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + if err != nil { + panic(err) + } + v, err := f.CreateViewIfNotExists(view) + if err != nil { + panic(err) + } + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + panic(err) + } + return &Fragment{Fragment: frag} +} diff --git a/test/index.go b/test/index.go new file mode 100644 index 000000000..650aeb6bf --- /dev/null +++ b/test/index.go @@ -0,0 +1,91 @@ +package test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/pilosa/pilosa" +) + +// Index represents a test wrapper for pilosa.Index. +type Index struct { + *pilosa.Index +} + +// NewIndex returns a new instance of Index. +func NewIndex() *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := pilosa.NewIndex(path, "i") + if err != nil { + panic(err) + } + return &Index{Index: index} +} + +// MustOpenIndex returns a new, opened index at a temporary path. Panic on error. +func MustOpenIndex() *Index { + index := NewIndex() + if err := index.Open(); err != nil { + panic(err) + } + return index +} + +// Close closes the index and removes the underlying data. +func (i *Index) Close() error { + defer os.RemoveAll(i.Path()) + return i.Index.Close() +} + +// Reopen closes the index and reopens it. +func (i *Index) Reopen() error { + var err error + if err := i.Index.Close(); err != nil { + return err + } + + path, name := i.Path(), i.Name() + i.Index, err = pilosa.NewIndex(path, name) + if err != nil { + return err + } + + if err := i.Open(); err != nil { + return err + } + return nil +} + +// CreateFrame creates a frame with the given options. +func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := i.Index.CreateFrame(name, opt) + if err != nil { + return nil, err + } + return &Frame{Frame: f}, nil +} + +// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. +func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := i.Index.CreateFrameIfNotExists(name, opt) + if err != nil { + return nil, err + } + return &Frame{Frame: f}, nil +} + +// Ensure index can delete a frame. +func TestIndex_InvalidName(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := pilosa.NewIndex(path, "ABC") + if index != nil { + t.Fatalf("unexpected index name %s", index) + } +} diff --git a/view_test.go b/view_test.go index 53310c298..5f5ce94e2 100644 --- a/view_test.go +++ b/view_test.go @@ -19,12 +19,13 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" ) // View is a test wrapper for pilosa.View. type View struct { *pilosa.View - RowAttrStore *AttrStore + RowAttrStore *test.AttrStore } // NewView returns a new instance of View with a temporary path. @@ -37,7 +38,7 @@ func NewView(index, frame, name string) *View { v := &View{ View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize), - RowAttrStore: MustOpenAttrStore(), + RowAttrStore: test.MustOpenAttrStore(), } v.View.RowAttrStore = v.RowAttrStore.AttrStore return v From 12e6b22abe7f509654f99ba6a51b1aebd2593626 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 23 Jun 2017 17:27:57 -0500 Subject: [PATCH 02/17] WIP replace fmt prints with logger prints and Stderr with LogOutput --- cmd/server.go | 20 +++++++++++++++++--- fragment.go | 2 +- gossip/gossip.go | 5 ++--- holder.go | 2 +- server/server.go | 14 ++++++++++---- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 3e43869c5..9ef9aa1a8 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -17,6 +17,7 @@ package cmd import ( "fmt" "io" + "log" "os" "os/signal" "runtime/pprof" @@ -44,7 +45,20 @@ It will load existing data from the configured directory, and start listening client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - fmt.Fprintf(Server.Stderr, "Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) + // TODO this code is duplicated from server/server.go:Server.Run() because it hasnt run yet + var logOutput io.Writer + if Server.Config.LogPath == "" { + logOutput = stderr + } else { + var err error + logOutput, err = os.OpenFile(Server.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return err + } + } + logger := log.New(logOutput, "", log.LstdFlags) + logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) + // fmt.Fprintf(Server.Stderr, "Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) // Start CPU profiling. if Server.CPUProfile != "" { @@ -73,7 +87,7 @@ on the configured port.`, signal.Notify(c, os.Interrupt) select { case sig := <-c: - fmt.Fprintf(Server.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) + logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) // Second signal causes a hard shutdown. go func() { <-c; os.Exit(1) }() @@ -82,7 +96,7 @@ on the configured port.`, return err } case <-Server.Done: - fmt.Fprintf(Server.Stderr, "Server closed externally") + logger.Printf("Server closed externally") } return nil }, diff --git a/fragment.go b/fragment.go index 4758f18e8..278a23bcf 100644 --- a/fragment.go +++ b/fragment.go @@ -265,7 +265,7 @@ func (f *Fragment) openCache() error { // Unmarshal cache data. var pb internal.Cache if err := proto.Unmarshal(buf, &pb); err != nil { - log.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) + f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) return nil } diff --git a/gossip/gossip.go b/gossip/gossip.go index 1b15522a7..c8d9a616d 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -18,7 +18,6 @@ import ( "fmt" "io" "log" - "os" "golang.org/x/sync/errgroup" @@ -98,9 +97,9 @@ type gossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler, logOutput io.Writer) *GossipNodeSet { g := &GossipNodeSet{ - LogOutput: os.Stderr, + LogOutput: logOutput, } //TODO: pull memberlist config from pilosa.cfg file diff --git a/holder.go b/holder.go index 59fef1560..2fdb58e97 100644 --- a/holder.go +++ b/holder.go @@ -92,7 +92,7 @@ func (h *Holder) Open() error { continue } - h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) + h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) // TODO h.LogOutput not set until server.go:Server.Open() index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { diff --git a/server/server.go b/server/server.go index 7738f513c..dfa5b6e61 100644 --- a/server/server.go +++ b/server/server.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "log" "math/rand" "net" "os" @@ -100,7 +101,10 @@ func (m *Command) Run(args ...string) (err error) { if err = m.Server.Open(); err != nil { return fmt.Errorf("server.Open: %v", err) } - fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) + + logger := log.New(m.Server.LogOutput, "", log.LstdFlags) // TODO make this a function? + + logger.Printf("Listening as http://%s\n", m.Server.Host) return nil } @@ -132,8 +136,10 @@ func (m *Command) SetupServer() error { m.Server.LogOutput = logFile } + logger := log.New(m.Server.LogOutput, "", log.LstdFlags) + // Configure holder. - fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir) + logger.Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollingInterval) m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) @@ -160,7 +166,7 @@ func (m *Command) SetupServer() error { switch m.Config.Cluster.Type { case "http": m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, internalPortStr) - m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Stderr) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Server.LogOutput) m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) if err != nil { @@ -180,7 +186,7 @@ func (m *Command) SetupServer() error { if err != nil { gossipHost = m.Config.Host } - gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) + gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server, m.Server.LogOutput) m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet From 8c2d6e92c29f9e0e381546db39c99a7f30d1d344 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 23 Jun 2017 18:02:44 -0500 Subject: [PATCH 03/17] WIP Export Server.logger for use in server/server.go --- server.go | 14 +++++++------- server/server.go | 9 ++------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/server.go b/server.go index 37fadedd5..6154ed3e1 100644 --- a/server.go +++ b/server.go @@ -197,14 +197,14 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } -func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } +func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { t := time.Now() ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() - s.logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.Logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) for { // Wait for tick or a close. @@ -215,7 +215,7 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } - s.logger().Printf("holder sync beginning") + s.Logger().Printf("holder sync beginning") // Initialize syncer with local holder and remote client. var syncer HolderSyncer @@ -226,12 +226,12 @@ func (s *Server) monitorAntiEntropy() { // Sync holders. if err := syncer.SyncHolder(); err != nil { - s.logger().Printf("holder sync error: err=%s", err) + s.Logger().Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.logger().Printf("holder sync complete") + s.Logger().Printf("holder sync complete") } dif := time.Since(t) s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) @@ -267,7 +267,7 @@ func (s *Server) monitorMaxSlices() { localIndex.SetRemoteMaxSlice(newmax) } } else { - s.logger().Printf("Local Index not found: %s", index) + s.Logger().Printf("Local Index not found: %s", index) } } } @@ -472,7 +472,7 @@ func (s *Server) monitorRuntime() { gcn := gcnotifier.New() defer gcn.Close() - s.logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) + s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) for { // Wait for tick or a close. diff --git a/server/server.go b/server/server.go index dfa5b6e61..51bee64b2 100644 --- a/server/server.go +++ b/server/server.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "io" - "log" "math/rand" "net" "os" @@ -102,9 +101,7 @@ func (m *Command) Run(args ...string) (err error) { return fmt.Errorf("server.Open: %v", err) } - logger := log.New(m.Server.LogOutput, "", log.LstdFlags) // TODO make this a function? - - logger.Printf("Listening as http://%s\n", m.Server.Host) + m.Server.Logger().Printf("Listening as http://%s\n", m.Server.Host) return nil } @@ -136,10 +133,8 @@ func (m *Command) SetupServer() error { m.Server.LogOutput = logFile } - logger := log.New(m.Server.LogOutput, "", log.LstdFlags) - // Configure holder. - logger.Printf("Using data from: %s\n", m.Config.DataDir) + m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollingInterval) m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) From 2fe21231fc31e119171367de585276516edc225d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 23 Jun 2017 18:03:28 -0500 Subject: [PATCH 04/17] Deduplicate logfile opening code --- server/server.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/server/server.go b/server/server.go index 51bee64b2..88f4c4f5d 100644 --- a/server/server.go +++ b/server/server.go @@ -123,14 +123,9 @@ func (m *Command) SetupServer() error { m.Server.Cluster = cluster // Setup logging output. - if m.Config.LogPath == "" { - m.Server.LogOutput = m.Stderr - } else { - logFile, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return err - } - m.Server.LogOutput = logFile + m.Server.LogOutput, err = GetLogWriter(m.Config.LogPath, m.Stderr) + if err != nil { + return err } // Configure holder. @@ -203,6 +198,20 @@ func (m *Command) SetupServer() error { return nil } +// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path. +func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) { + // This is split out so it can be used in NewServeCmd as well as SetupServer + if path == "" { + return defaultWriter, nil + } else { + logFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + return logFile, nil + } +} + func normalizeHost(host string) (string, error) { if !strings.Contains(host, ":") { host = host + ":" From b9e853dbdbdcdafe95403f85654c25f138c8f48c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 23 Jun 2017 18:08:30 -0500 Subject: [PATCH 05/17] Deduplicate logfile opening code --- cmd/server.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 9ef9aa1a8..2cd2aa5a1 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -45,20 +45,12 @@ It will load existing data from the configured directory, and start listening client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - // TODO this code is duplicated from server/server.go:Server.Run() because it hasnt run yet - var logOutput io.Writer - if Server.Config.LogPath == "" { - logOutput = stderr - } else { - var err error - logOutput, err = os.OpenFile(Server.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return err - } + logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr) + if err != nil { + return err } logger := log.New(logOutput, "", log.LstdFlags) logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) - // fmt.Fprintf(Server.Stderr, "Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) // Start CPU profiling. if Server.CPUProfile != "" { From 855f224778c306d361e49df5b215aa84c6019ea2 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 23 Jun 2017 18:14:59 -0500 Subject: [PATCH 06/17] Pass logOutput to Holder.Open --- holder.go | 4 +++- server.go | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/holder.go b/holder.go index 2fdb58e97..854d778ef 100644 --- a/holder.go +++ b/holder.go @@ -70,7 +70,7 @@ func NewHolder() *Holder { } // Open initializes the root data directory for the holder. -func (h *Holder) Open() error { +func (h *Holder) Open(logOutput io.Writer) error { if err := os.MkdirAll(h.Path, 0777); err != nil { return err } @@ -87,6 +87,8 @@ func (h *Holder) Open() error { return err } + h.LogOutput = logOutput + for _, fi := range fis { if !fi.IsDir() { continue diff --git a/server.go b/server.go index 6154ed3e1..265a400f5 100644 --- a/server.go +++ b/server.go @@ -129,7 +129,7 @@ func (s *Server) Open() error { } // Open holder. - if err := s.Holder.Open(); err != nil { + if err := s.Holder.Open(s.LogOutput); err != nil { return fmt.Errorf("opening Holder: %v", err) } @@ -159,7 +159,6 @@ func (s *Server) Open() error { // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster - s.Holder.LogOutput = s.LogOutput // Serve HTTP. go func() { http.Serve(ln, s.Handler) }() From e1f44d4e319c2f1d2b267aa85aaff523ef2f1394 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 26 Jun 2017 11:23:37 -0500 Subject: [PATCH 07/17] Update tests --- ctl/backup_test.go | 3 ++- holder_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 512093a20..936576704 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "net/http/httptest" "net/url" + "os" "testing" ) @@ -190,7 +191,7 @@ func NewHolder() *Holder { // MustOpenHolder creates and opens a holder at a temporary path. Panic on error. func MustOpenHolder() *Holder { h := NewHolder() - if err := h.Open(); err != nil { + if err := h.Open(os.Stderr); err != nil { panic(err) } return h diff --git a/holder_test.go b/holder_test.go index b6864ffc9..78a9219b3 100644 --- a/holder_test.go +++ b/holder_test.go @@ -417,7 +417,7 @@ func NewHolder() *Holder { // MustOpenHolder creates and opens a holder at a temporary path. Panic on error. func MustOpenHolder() *Holder { h := NewHolder() - if err := h.Open(); err != nil { + if err := h.Open(&h.LogOutput); err != nil { panic(err) } return h @@ -439,7 +439,7 @@ func (h *Holder) Reopen() error { h.Holder = pilosa.NewHolder() h.Holder.Path = path h.Holder.LogOutput = logOutput - if err := h.Holder.Open(); err != nil { + if err := h.Holder.Open(&h.LogOutput); err != nil { return err } From 0631a63de2befda96accdd145754e4c950d7054f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 26 Jun 2017 16:30:04 -0500 Subject: [PATCH 08/17] Refactor additional test utilities into pilosa.test --- client_test.go | 34 ++++------- executor_test.go | 143 +++++++++++++++++++---------------------------- stats_test.go | 16 +++--- test/client.go | 19 +++++++ test/executor.go | 32 +++++++++++ 5 files changed, 128 insertions(+), 116 deletions(-) create mode 100644 test/client.go create mode 100644 test/executor.go diff --git a/client_test.go b/client_test.go index f4b4548d9..ff8e4f385 100644 --- a/client_test.go +++ b/client_test.go @@ -131,10 +131,10 @@ func TestClient_MultiNode(t *testing.T) { hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() // Connect to each node to compare results. - client := make([]*Client, 3) - client[0] = MustNewClient(s[0].Host()) - client[1] = MustNewClient(s[1].Host()) - client[2] = MustNewClient(s[2].Host()) + client := make([]*test.Client, 3) + client[0] = test.MustNewClient(s[0].Host()) + client[1] = test.MustNewClient(s[1].Host()) + client[2] = test.MustNewClient(s[2].Host()) topN := 4 q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN) @@ -213,7 +213,7 @@ func TestClient_Import(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := MustNewClient(s.Host()) + c := test.MustNewClient(s.Host()) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -264,7 +264,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s.Handler.Holder = hldr.Holder // Send import request. - c := MustNewClient(s.Host()) + c := test.MustNewClient(s.Host()) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -303,7 +303,7 @@ func TestClient_BackupRestore(t *testing.T) { s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder - c := MustNewClient(s.Host()) + c := test.MustNewClient(s.Host()) // Backup from frame. var buf bytes.Buffer @@ -368,7 +368,7 @@ func TestClient_BackupInverseView(t *testing.T) { s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder - c := MustNewClient(s.Host()) + c := test.MustNewClient(s.Host()) // Backup from frame. var buf bytes.Buffer @@ -405,7 +405,7 @@ func TestClient_BackupInvalidView(t *testing.T) { s.Handler.Cluster.Nodes[0].Host = s.Host() s.Handler.Holder = hldr.Holder - c := MustNewClient(s.Host()) + c := test.MustNewClient(s.Host()) // Backup from frame. var buf bytes.Buffer @@ -435,7 +435,7 @@ func TestClient_FragmentBlocks(t *testing.T) { s.Handler.Holder = hldr.Holder // Retrieve blocks. - c := MustNewClient(s.Host()) + c := test.MustNewClient(s.Host()) blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0) if err != nil { t.Fatal(err) @@ -452,17 +452,3 @@ func TestClient_FragmentBlocks(t *testing.T) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } - -// Client represents a test wrapper for pilosa.Client. -type Client struct { - *pilosa.Client -} - -// MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string) *Client { - c, err := pilosa.NewClient(host) - if err != nil { - panic(err) - } - return &Client{Client: c} -} diff --git a/executor_test.go b/executor_test.go index 6a36babbf..d4714b714 100644 --- a/executor_test.go +++ b/executor_test.go @@ -19,7 +19,6 @@ import ( "fmt" "reflect" "strconv" - "strings" "testing" "github.com/davecgh/go-spew/spew" @@ -39,10 +38,10 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // Set bits. - if _, err := e.Execute(context.Background(), "i", MustParse(``+ + if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), @@ -53,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -70,10 +69,10 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // Set bits. - if _, err := e.Execute(context.Background(), "i", MustParse(``+ + if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), @@ -84,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -104,8 +103,8 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { t.Fatalf("unexpected bits: %+v", bits) @@ -118,8 +117,8 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Difference()`), nil, nil); err == nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } } @@ -136,8 +135,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -149,8 +148,8 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect()`), nil, nil); err == nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } } @@ -166,8 +165,8 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -180,8 +179,8 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { defer hldr.Close() hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Union()`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected bits: %+v", bits) @@ -196,8 +195,8 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -209,13 +208,13 @@ func TestExecutor_Execute_SetBit(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) if n := f.Row(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -226,7 +225,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -250,17 +249,17 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -276,7 +275,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { func TestExecutor_Execute_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // Set bits for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -285,7 +284,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", MustParse(` + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, rowID=0, columnID=0) SetBit(frame=f, rowID=0, columnID=1) SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`) @@ -305,7 +304,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() t.Run("Standard", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {ID: 0, Count: 5}, @@ -316,7 +315,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { }) t.Run("Inverse", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {ID: SliceWidth, Count: 3}, @@ -339,8 +338,8 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) // Execute query. - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, @@ -373,8 +372,8 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) // Execute query. - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -408,8 +407,8 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -432,8 +431,8 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -455,8 +454,8 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -470,7 +469,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -484,7 +483,7 @@ func TestExecutor_Execute_Range(t *testing.T) { } // Set bits. - if _, err := e.Execute(context.Background(), "i", MustParse(` + if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00") SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00") SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00") @@ -500,7 +499,7 @@ func TestExecutor_Execute_Range(t *testing.T) { } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) @@ -508,8 +507,8 @@ func TestExecutor_Execute_Range(t *testing.T) { }) t.Run("Inverse", func(t *testing.T) { - e := NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { t.Fatalf("unexpected bits: %+v", bits) @@ -552,8 +551,8 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { s.Handler.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) - e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, c) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected bits: %+v", bits) @@ -581,8 +580,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) - e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, c) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -621,8 +620,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { t.Fatal(err) } - e := NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, c) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -669,8 +668,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { t.Fatal(err) } - e := NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, c) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -732,8 +731,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) - e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, c) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -748,33 +747,9 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) e.MaxWritesPerRequest = 3 - if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } - -// Executor represents a test wrapper for pilosa.Executor. -type Executor struct { - *pilosa.Executor -} - -// NewExecutor returns a new instance of Executor. -// The executor always matches the hostname of the first cluster node. -func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - e := &Executor{Executor: pilosa.NewExecutor()} - e.Holder = holder - e.Cluster = cluster - e.Host = cluster.Nodes[0].Host - return e -} - -// MustParse parses s into a PQL query. Panic on error. -func MustParse(s string) *pql.Query { - q, err := pql.NewParser(strings.NewReader(s)).Parse() - if err != nil { - panic(err) - } - return q -} diff --git a/stats_test.go b/stats_test.go index 9ca81bcda..7ed679f21 100644 --- a/stats_test.go +++ b/stats_test.go @@ -84,7 +84,7 @@ func TestStatsCount_TopN(t *testing.T) { // Execute query. called := false - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "TopN" { @@ -99,7 +99,7 @@ func TestStatsCount_TopN(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -114,7 +114,7 @@ func TestStatsCount_Bitmap(t *testing.T) { hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) called := false - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "Bitmap" { @@ -129,7 +129,7 @@ func TestStatsCount_Bitmap(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -145,7 +145,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) called := false - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) frame := e.Holder.Frame("d", "f") if frame == nil { t.Fatal("frame not found") @@ -160,7 +160,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -176,7 +176,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) called := false - e := NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) idx := e.Holder.Index("d") if idx == nil { t.Fatal("idex not found") @@ -192,7 +192,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { diff --git a/test/client.go b/test/client.go new file mode 100644 index 000000000..1afb8df1b --- /dev/null +++ b/test/client.go @@ -0,0 +1,19 @@ +package test + +import ( + "github.com/pilosa/pilosa" +) + +// Client represents a test wrapper for pilosa.Client. +type Client struct { + *pilosa.Client +} + +// MustNewClient returns a new instance of Client. Panic on error. +func MustNewClient(host string) *Client { + c, err := pilosa.NewClient(host) + if err != nil { + panic(err) + } + return &Client{Client: c} +} diff --git a/test/executor.go b/test/executor.go new file mode 100644 index 000000000..af5a248f6 --- /dev/null +++ b/test/executor.go @@ -0,0 +1,32 @@ +package test + +import ( + "strings" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/pql" +) + +// Executor represents a test wrapper for pilosa.Executor. +type Executor struct { + *pilosa.Executor +} + +// NewExecutor returns a new instance of Executor. +// The executor always matches the hostname of the first cluster node. +func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { + e := &Executor{Executor: pilosa.NewExecutor()} + e.Holder = holder + e.Cluster = cluster + e.Host = cluster.Nodes[0].Host + return e +} + +// MustParse parses s into a PQL query. Panic on error. +func MustParse(s string) *pql.Query { + q, err := pql.NewParser(strings.NewReader(s)).Parse() + if err != nil { + panic(err) + } + return q +} From 1cb94baa318293097896465afea95fba7679c484 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 26 Jun 2017 16:33:48 -0500 Subject: [PATCH 09/17] Add `-n` to tail command for GNU-tail compatibility --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ef21fda0..d26d6b42c 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ cover-pkg: mkdir -p build/coverage touch build/coverage/$(subst /,-,$(PKG)).out go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) - tail +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out cover-viz: cover go tool cover -html=build/coverage/all.out From d96a121347ca9ff714feda4453aeb9ad138a2c13 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 27 Jun 2017 09:52:40 -0500 Subject: [PATCH 10/17] Simplify function arguments --- gossip/gossip.go | 6 +++--- server/server.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index c8d9a616d..92ec94c97 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -97,9 +97,9 @@ type gossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler, logOutput io.Writer) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server) *GossipNodeSet { g := &GossipNodeSet{ - LogOutput: logOutput, + LogOutput: server.LogOutput, } //TODO: pull memberlist config from pilosa.cfg file @@ -114,7 +114,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.AdvertisePort = gossipPort g.config.memberlistConfig.Delegate = g - g.statusHandler = sh + g.statusHandler = server return g } diff --git a/server/server.go b/server/server.go index 88f4c4f5d..f66b88870 100644 --- a/server/server.go +++ b/server/server.go @@ -176,7 +176,7 @@ func (m *Command) SetupServer() error { if err != nil { gossipHost = m.Config.Host } - gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server, m.Server.LogOutput) + gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet From 271dd5a63b07f8902018c7984abd2669f6db8b1c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 27 Jun 2017 21:43:26 -0500 Subject: [PATCH 11/17] Simplify holder logOutput --- ctl/backup_test.go | 3 +-- holder.go | 4 +--- holder_test.go | 4 ++-- server.go | 3 ++- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 936576704..512093a20 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "net/http/httptest" "net/url" - "os" "testing" ) @@ -191,7 +190,7 @@ func NewHolder() *Holder { // MustOpenHolder creates and opens a holder at a temporary path. Panic on error. func MustOpenHolder() *Holder { h := NewHolder() - if err := h.Open(os.Stderr); err != nil { + if err := h.Open(); err != nil { panic(err) } return h diff --git a/holder.go b/holder.go index 854d778ef..2fdb58e97 100644 --- a/holder.go +++ b/holder.go @@ -70,7 +70,7 @@ func NewHolder() *Holder { } // Open initializes the root data directory for the holder. -func (h *Holder) Open(logOutput io.Writer) error { +func (h *Holder) Open() error { if err := os.MkdirAll(h.Path, 0777); err != nil { return err } @@ -87,8 +87,6 @@ func (h *Holder) Open(logOutput io.Writer) error { return err } - h.LogOutput = logOutput - for _, fi := range fis { if !fi.IsDir() { continue diff --git a/holder_test.go b/holder_test.go index 78a9219b3..b6864ffc9 100644 --- a/holder_test.go +++ b/holder_test.go @@ -417,7 +417,7 @@ func NewHolder() *Holder { // MustOpenHolder creates and opens a holder at a temporary path. Panic on error. func MustOpenHolder() *Holder { h := NewHolder() - if err := h.Open(&h.LogOutput); err != nil { + if err := h.Open(); err != nil { panic(err) } return h @@ -439,7 +439,7 @@ func (h *Holder) Reopen() error { h.Holder = pilosa.NewHolder() h.Holder.Path = path h.Holder.LogOutput = logOutput - if err := h.Holder.Open(&h.LogOutput); err != nil { + if err := h.Holder.Open(); err != nil { return err } diff --git a/server.go b/server.go index 265a400f5..1f09119e1 100644 --- a/server.go +++ b/server.go @@ -129,7 +129,8 @@ func (s *Server) Open() error { } // Open holder. - if err := s.Holder.Open(s.LogOutput); err != nil { + s.Holder.LogOutput = s.LogOutput + if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } From 374dfd1fa048a73bd679e4e8076cbc369fb1239a Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 27 Jun 2017 21:44:17 -0500 Subject: [PATCH 12/17] Remove TODO comment --- holder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holder.go b/holder.go index 2fdb58e97..59fef1560 100644 --- a/holder.go +++ b/holder.go @@ -92,7 +92,7 @@ func (h *Holder) Open() error { continue } - h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) // TODO h.LogOutput not set until server.go:Server.Open() + h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { From 124fd6a351fd9d34a10713da6aa77403606ced3b Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 28 Jun 2017 15:44:39 -0500 Subject: [PATCH 13/17] Add brief explanations for query parameters on the query endpoint --- docs/api-reference.md | 4 ++++ docs/data-model.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 39f66683f..afc4d9f1a 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -93,6 +93,10 @@ In order to send protobuf binaries in the request and response, set `Content-Typ The response doesn't include column attributes by default. To return them, set `columnAttrs` query argument to `true`. +The query is executed for all [slices](data-model#slice) by default. To use specified slices only, set `slices` query argument to a comma-separated list of slice indices. + +The time quantum can be specified with the `time_granularity`. Valid values match those for the [time-quantum](#/index//time-quantum) endpoint. + Request: ``` curl localhost:10101/index/user/query?columnAttrs=true \ diff --git a/docs/data-model.md b/docs/data-model.md index b4b98de9a..2779f17ad 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -54,7 +54,7 @@ Attributes are arbitrary key/value pairs that can be associated to both rows or ### Slice -Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. +Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. SliceWidth is a non-configurable constant set to 2^20. Columns are sharded on a preset width, and each shard is referred to as a Slice. Slices are operated on in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm. From f70c4e637b8201ddb110e7c790051fef6a0a5092 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 28 Jun 2017 16:08:07 -0500 Subject: [PATCH 14/17] Formatting and link tweaks --- docs/api-reference.md | 6 +++--- docs/data-model.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index afc4d9f1a..5871fd309 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -93,13 +93,13 @@ In order to send protobuf binaries in the request and response, set `Content-Typ The response doesn't include column attributes by default. To return them, set `columnAttrs` query argument to `true`. -The query is executed for all [slices](data-model#slice) by default. To use specified slices only, set `slices` query argument to a comma-separated list of slice indices. +The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set `slices` query argument to a comma-separated list of slice indices. -The time quantum can be specified with the `time_granularity`. Valid values match those for the [time-quantum](#/index//time-quantum) endpoint. +The time quantum can be specified with the `time_granularity`. Valid values match those for the `time-quantum` endpoint. Request: ``` -curl localhost:10101/index/user/query?columnAttrs=true \ +curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ -d 'Bitmap(frame="language", id=5)' ``` diff --git a/docs/data-model.md b/docs/data-model.md index 2779f17ad..f8987f87b 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -54,7 +54,7 @@ Attributes are arbitrary key/value pairs that can be associated to both rows or ### Slice -Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. SliceWidth is a non-configurable constant set to 2^20. +Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. SliceWidth is a non-configurable constant set to 220. Columns are sharded on a preset width, and each shard is referred to as a Slice. Slices are operated on in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm. From af0e53120dd6de66730cb9c2d7def2575e433c6c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 28 Jun 2017 16:09:15 -0500 Subject: [PATCH 15/17] Tweak wording --- docs/api-reference.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 5871fd309..96ca16229 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -91,11 +91,11 @@ Response: In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`. -The response doesn't include column attributes by default. To return them, set `columnAttrs` query argument to `true`. +The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`. -The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set `slices` query argument to a comma-separated list of slice indices. +The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. -The time quantum can be specified with the `time_granularity`. Valid values match those for the `time-quantum` endpoint. +The time quantum can be specified with the `time_granularity` query argument. Valid values match those for the `time-quantum` endpoint. Request: ``` From ee84d64dd5de02c2b909b3b77ff740a7fb3a5346 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 28 Jun 2017 16:31:44 -0500 Subject: [PATCH 16/17] Replace code headers with anchor headers --- docs/api-reference.md | 44 ++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 39f66683f..81c0be1f7 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -5,9 +5,9 @@ title = "API Reference" ## API Reference -### `/index` +### List all index schemas -#### `GET` +`GET /index` Returns the schema of all indexes in JSON. @@ -21,9 +21,9 @@ Response: {"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} ``` -### `/index/` +### List index schema -#### `GET` +`GET /index/` Returns the schema of the specified index in JSON. @@ -37,7 +37,9 @@ Response: {"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} ``` -#### `POST` +### Create index + +`POST /index/` Creates an index with the given name. @@ -57,7 +59,9 @@ Response: {} ``` -#### `DELETE` +### Remove index + +`DELETE /index/index-name` Removes the given index. @@ -71,9 +75,9 @@ Response: {} ``` -### `/index//query` +### Query index -#### `POST` +`POST /index//query` Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. @@ -107,9 +111,9 @@ Response: } ``` -### `/index//time-quantum` +### Change index time quantum -#### `PATCH` +`PATCH /index//time-quantum` Changes the time quantum for the given index. This endpoint should be called at most once right after creating a database. @@ -139,9 +143,9 @@ Response: {} ``` -### `/index//frame/` +### Create frame -#### `POST` +`POST /index//frame/` Creates a frame in the given index with the given name. @@ -165,7 +169,9 @@ Response: {} ``` -#### `DELETE` +### Remove frame + +`DELETE POST /index//frame/` Removes the given frame. @@ -179,9 +185,9 @@ Response: {} ``` -### `/index//frame//time-quantum` +### Change frame time quantum -#### `PATCH` +`PATCH /index//frame//time-quantum` Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame. @@ -211,9 +217,9 @@ Response: {} ``` -### `/hosts` +### List hosts -#### `GET` +`GET /hosts` Returns the hosts in the cluster. @@ -227,9 +233,9 @@ Response: [{"host":":10101","internalHost":""}] ``` -### `/version` +### Get version -#### `GET` +`GET /version` Returns the version of the Pilosa server. From 264c657500c3f03ff3667479db63ca214a9ed500 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 28 Jun 2017 17:26:12 -0500 Subject: [PATCH 17/17] De-document unused query argument --- docs/api-reference.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 96ca16229..adea42ff4 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -95,8 +95,6 @@ The response doesn't include column attributes by default. To return them, set t The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. -The time quantum can be specified with the `time_granularity` query argument. Valid values match those for the `time-quantum` endpoint. - Request: ``` curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \