From 92466fda8dee5e03dee92b2b29bb777976d47288 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Jun 2018 21:59:15 -0500 Subject: [PATCH 1/6] Remove Apache2 license from enterprise build directory --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 363bcd6b6..90e4700df 100644 --- a/Makefile +++ b/Makefile @@ -48,8 +48,8 @@ build: vendor # Create a single release build under the build directory release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) - $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID)) + cp NOTICE README.md build/pilosa-$(VERSION_ID) + $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID)) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz From 5766f572b19812b28cf8e61baf681337d8ff32b9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 29 Jun 2018 12:21:33 -0500 Subject: [PATCH 2/6] fix skipped client TopN test had to remove the first check which was for the wrong result - because maxShard is no longer wrong since the broadcaster is actually working. --- field.go | 1 - http/client_test.go | 90 ++++++++++++++++++--------------------------- server.go | 14 +++++++ test/cluster.go | 15 +++----- test/holder.go | 10 ++++- 5 files changed, 64 insertions(+), 66 deletions(-) diff --git a/field.go b/field.go index 9eb4f75b8..dd5bc8d03 100644 --- a/field.go +++ b/field.go @@ -583,7 +583,6 @@ func (f *Field) RecalculateCaches() { // CreateViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. func (f *Field) CreateViewIfNotExists(name string) (*View, error) { - view, created, err := f.createViewIfNotExistsBase(name) if err != nil { return nil, err diff --git a/http/client_test.go b/http/client_test.go index e5a595605..fa9d94e4f 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -50,41 +51,26 @@ func init() { } +// modHasher represents a simple, mod-based hashing. +type modHasher struct{} + +func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } + // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { - t.Skip() // Until test.NewServer() works + c := test.MustRunCluster(t, 3, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&modHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&modHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&modHasher{}))}, + ) + defer c.Close() - cluster := test.NewCluster(3) - s, hldr := createCluster(cluster) - - for i := 0; i < len(cluster.Nodes); i++ { - defer hldr[i].Close() - defer s[i].Close() - } - - s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr[0].Holder - e.Node = cluster.Nodes[0] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) - } - s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr[1].Holder - e.Node = cluster.Nodes[1] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) - } - s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr[2].Holder - e.Node = cluster.Nodes[2] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) + hldr := []test.Holder{} + for _, command := range c { + hldr = append(hldr, test.Holder{Holder: command.Server.Holder()}) } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN. @@ -106,7 +92,7 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, s[i].Host(), owns) + t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns) } } @@ -120,13 +106,21 @@ func TestClient_MultiNode(t *testing.T) { maxShard = x } } + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } hldr[0].MustSetBits("i", "f", 100, baseBit0+10) hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12) hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2) hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) @@ -145,39 +139,27 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[0]).RecalculateCache() - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[1]).RecalculateCache() - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[2]).RecalculateCache() + c[0].RecalculateCaches() + c[1].RecalculateCaches() + c[2].RecalculateCaches() // Connect to each node to compare results. client := make([]*Client, 3) - client[0] = MustNewClient(s[0].Host(), defaultClient) - client[1] = MustNewClient(s[1].Host(), defaultClient) - client[2] = MustNewClient(s[2].Host(), defaultClient) + client[0] = MustNewClient(c[0].URL(), defaultClient) + client[1] = MustNewClient(c[1].URL(), defaultClient) + client[2] = MustNewClient(c[2].URL(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), Remote: false, } + result, err := client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } - // Check the results before every node has the correct max shard value. - pairs := result.Results[0].Pairs - for _, pair := range pairs { - if pair.ID == 22 && pair.Count != 3 { - t.Fatalf("Invalid Cluster wide MaxShard prevents accurate calculation of %s", pair) - } - } - - // Set max shard to correct value. - hldr[0].Index("i").SetRemoteMaxShard(maxShard) - hldr[1].Index("i").SetRemoteMaxShard(maxShard) - hldr[2].Index("i").SetRemoteMaxShard(maxShard) - result, err = client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) @@ -189,7 +171,7 @@ func TestClient_MultiNode(t *testing.T) { } p := []*internal.Pair{ {ID: 100, Count: 12}, - {ID: 22, Count: 11}, + {ID: 22, Count: 10}, {ID: 98, Count: 8}, {ID: 99, Count: 7}} diff --git a/server.go b/server.go index 457a9123a..e1ad60327 100644 --- a/server.go +++ b/server.go @@ -209,6 +209,20 @@ func OptServerIsCoordinator(is bool) ServerOption { } } +func OptServerNodeID(nodeID string) ServerOption { + return func(s *Server) error { + s.nodeID = nodeID + return nil + } +} + +func OptServerClusterHasher(h Hasher) ServerOption { + return func(s *Server) error { + s.cluster.Hasher = h + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ diff --git a/test/cluster.go b/test/cluster.go index d1ee6af80..8c1e3791e 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -30,7 +30,7 @@ func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() c.ReplicaN = 1 - c.Hasher = newModHasher() + c.Hasher = &modHasher{} c.Path = path c.Topology = pilosa.NewTopology() @@ -48,14 +48,6 @@ func NewCluster(n int) *pilosa.Cluster { return c } -// modHasher represents a simple, mod-based hashing. -type modHasher struct{} - -// newModHasher returns a new instance of ModHasher with n buckets. -func newModHasher() *modHasher { return &modHasher{} } - -func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } - // newURI is a test URI creator that intentionally swallows errors. func newURI(scheme, host string, port uint16) pilosa.URI { uri := pilosa.DefaultURI() @@ -64,3 +56,8 @@ func newURI(scheme, host string, port uint16) pilosa.URI { uri.SetPort(port) return *uri } + +// modHasher represents a simple, mod-based hashing. +type modHasher struct{} + +func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } diff --git a/test/holder.go b/test/holder.go index ff308062c..277c774dd 100644 --- a/test/holder.go +++ b/test/holder.go @@ -142,7 +142,10 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - f.SetBit(rowID, columnID, nil) + _, err = f.SetBit(rowID, columnID, nil) + if err != nil { + panic(err) + } } // ClearBit clears a bit on the given field. @@ -152,7 +155,10 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - f.ClearBit(rowID, columnID) + _, err = f.ClearBit(rowID, columnID) + if err != nil { + panic(err) + } } // MustSetBits sets columns on a row. Panic on error. From ea77db895a4bb1802e71c32ca254693960e27bf7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 29 Jun 2018 14:43:50 -0500 Subject: [PATCH 3/6] fix syncholder test and a few bugs The http internal client's FragmentBlocks and Blockdata methods were being used incorrectly, and incorrect respectively. One was not being passed a node URI by monitorAntiEntropy, and the other was always using the defaultURI regardless of what was passed to it. Antientropy was doubling not working because of this. I think this crept in pretty recently, so hasn't actually affected anyone. I exposed a SyncData method on Server so that we can invoke the anti entropy task manually instead of trying to set up the interval so that it will run and then sleeping and waiting for it to run. Now that this test works the way it does, the other anti entropy test is obsolete, and I deleted it. --- fragment.go | 2 +- holder.go | 4 ++ holder_test.go | 118 +++++++++++++++++++------------------------- http/client.go | 5 +- http/client_test.go | 14 ------ server.go | 29 +++++++---- server_test.go | 51 ------------------- 7 files changed, 80 insertions(+), 143 deletions(-) delete mode 100644 server_test.go diff --git a/fragment.go b/fragment.go index 891c1570e..a5a40d68f 100644 --- a/fragment.go +++ b/fragment.go @@ -1752,7 +1752,7 @@ func (s *FragmentSyncer) syncFragment() error { } // Retrieve remote blocks. - blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.shard) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.shard) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } diff --git a/holder.go b/holder.go index bfb2ae758..eb6eacd57 100644 --- a/holder.go +++ b/holder.go @@ -564,6 +564,8 @@ func (h *Holder) logStartup() error { // HolderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. type HolderSyncer struct { + mu sync.Mutex + Holder *Holder Node *Node @@ -588,6 +590,8 @@ func (s *HolderSyncer) IsClosing() bool { // SyncHolder compares the holder on host with the local holder and resolves differences. func (s *HolderSyncer) SyncHolder() error { + s.mu.Lock() // only allow one instance of SyncHolder to be running at a time + defer s.mu.Unlock() ti := time.Now() // Iterate over schema in sorted order. for _, di := range s.Holder.Schema() { diff --git a/holder_test.go b/holder_test.go index f758798a4..fe6a1a9f8 100644 --- a/holder_test.go +++ b/holder_test.go @@ -24,8 +24,6 @@ import ( "testing" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" ) @@ -350,49 +348,40 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { - t.Skip() // Until test.NewServer() works - - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.URL) + c := test.MustNewCluster(t, 2) + c[0].Config.Cluster.ReplicaN = 2 + c[0].Config.AntiEntropy.Interval = 0 + c[1].Config.Cluster.ReplicaN = 2 + c[1].Config.AntiEntropy.Interval = 0 + err := c.Start() if err != nil { - t.Fatal(err) + t.Fatalf("starting cluster: %v", err) + } + defer c.Close() + + _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index i: %v", err) + } + _, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index y: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field f: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field f0: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field z in y: %v", err) } - cluster := test.NewCluster(2) - client := http.GetHTTPClient(nil) - httpClient := http.NewInternalClientFromURI(uri, client) - cluster.InternalClient = httpClient - - // Create a local holder. - hldr0 := test.MustOpenHolder() - defer hldr0.Close() - - // Create a remote holder wrapped by an HTTP - hldr1 := test.MustOpenHolder() - defer hldr1.Close() - s.Handler.API.Holder = hldr1.Holder - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr1.Holder - e.Node = cluster.Nodes[1] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) - } - - // Mock 2-node, fully replicated cluster. - cluster.ReplicaN = 2 - - cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) - cluster.Nodes[1].URI = *uri - - // Create fields on nodes. - for _, hldr := range []*test.Holder{hldr0, hldr1} { - hldr.MustCreateFieldIfNotExists("i", "f") - hldr.MustCreateFieldIfNotExists("i", "f0") - hldr.MustCreateFieldIfNotExists("y", "z") - } + hldr0 := &test.Holder{Holder: c[0].Server.Holder()} + hldr1 := &test.Holder{Holder: c[1].Server.Holder()} // Set data on the local holder. hldr0.SetBit("i", "f", 0, 10) @@ -414,42 +403,39 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5) hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7) - // Set highest shard. - hldr0.Index("i").SetRemoteMaxShard(1) - hldr0.Index("y").SetRemoteMaxShard(3) - - // Set up syncer. - syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - Node: cluster.Nodes[0], - Cluster: cluster, - Stats: pilosa.NopStatsClient, + err = c[0].Server.SyncData() + if err != nil { + t.Fatalf("syncing node 0: %v", err) } - - if err := syncer.SyncHolder(); err != nil { - t.Fatal(err) + err = c[1].Server.SyncData() + if err != nil { + t.Fatalf("syncing node 1: %v", err) } // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) + t.Errorf("unexpected columns(%d/0): %+v", i, a) + } + if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Errorf("unexpected columns(%d/2): %+v", i, a) + } + if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Errorf("unexpected columns(%d/3): %+v", i, a) + } + if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Errorf("unexpected columns(%d/120): %+v", i, a) + } + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Errorf("unexpected columns(%d/200): %+v", i, a) } if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { - t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) + t.Errorf("unexpected columns(%d/d/f0): %+v", i, a) } if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + t.Errorf("unexpected columns(%d/y/z): %+v", i, a) } } } diff --git a/http/client.go b/http/client.go index 59efb4f72..345c572c1 100644 --- a/http/client.go +++ b/http/client.go @@ -717,6 +717,9 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in // BlockData returns row/column id pairs for a block. func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { + if uri == nil { + panic("need to pass a URI to BlockData") + } buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Field: field, @@ -727,7 +730,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, return nil, nil, errors.Wrap(err, "marshaling") } - u := uriPathToURL(c.defaultURI, "/fragment/block/data") + u := uriPathToURL(uri, "/fragment/block/data") req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) if err != nil { return nil, nil, errors.Wrap(err, "creating request") diff --git a/http/client_test.go b/http/client_test.go index fa9d94e4f..ec849ba10 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -30,20 +30,6 @@ import ( "github.com/pilosa/pilosa/test" ) -func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { - numNodes := len(c.Nodes) - hldr := make([]*test.Holder, numNodes) - server := make([]*test.Server, numNodes) - for i := 0; i < numNodes; i++ { - hldr[i] = test.MustOpenHolder() - server[i] = test.NewServer() - server[i].Handler.API.Cluster = c - server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() - server[i].Handler.API.Holder = hldr[i].Holder - } - return server, hldr -} - var defaultClient *gohttp.Client func init() { diff --git a/server.go b/server.go index e1ad60327..d64b9efa7 100644 --- a/server.go +++ b/server.go @@ -70,6 +70,7 @@ type Server struct { diagnosticInterval time.Duration maxWritesPerRequest int isCoordinator bool + syncer HolderSyncer primaryTranslateStore TranslateStore @@ -342,6 +343,12 @@ func (s *Server) Open() error { // buffered channel. s.cluster.listenForJoins() + s.syncer.Holder = s.holder + s.syncer.Node = s.cluster.Node + s.syncer.Cluster = s.cluster + s.syncer.Closing = s.closing + s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") + // Start background monitoring. s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() @@ -384,12 +391,22 @@ func (s *Server) loadNodeID() string { return nodeID } +// SyncData manually invokes the anti entropy process which makes sure that this +// node has the data from all replicas across the cluster. +func (s *Server) SyncData() error { + return errors.Wrap(s.syncer.SyncHolder(), "syncing holder") +} + func (s *Server) monitorAntiEntropy() { + if s.antiEntropyInterval == 0 { + return // anti entropy disabled + } ticker := time.NewTicker(s.antiEntropyInterval) defer ticker.Stop() s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval) + // Initialize syncer with local holder and remote client. for { // Wait for tick or a close. select { @@ -399,18 +416,10 @@ func (s *Server) monitorAntiEntropy() { s.holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.logger.Printf("holder sync beginning") - - // Initialize syncer with local holder and remote client. - var syncer HolderSyncer - syncer.Holder = s.holder - syncer.Node = s.cluster.Node - syncer.Cluster = s.cluster - syncer.Closing = s.closing - syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") // Sync holders. - if err := syncer.SyncHolder(); err != nil { + s.logger.Printf("holder sync beginning") + if err := s.syncer.SyncHolder(); err != nil { s.logger.Printf("holder sync error: err=%s", err) continue } diff --git a/server_test.go b/server_test.go deleted file mode 100644 index 04f8a6171..000000000 --- a/server_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "context" - "testing" - "time" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/server" - "github.com/pilosa/pilosa/test" -) - -// TestMonitorAntiEntropy is a regression test which which caught a bug where -// pilosa.Server was not having its remoteClient field set by an option and so -// it was using a nil client in monitorAntiEntropy. -func TestMonitorAntiEntropy(t *testing.T) { - cluster := test.MustRunCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)}) - client := cluster[1].Client() - err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - - err = client.CreateField(context.Background(), "balh", "fralh") - if err != nil { - t.Fatalf("creating field: %v", err) - } - - time.Sleep(time.Millisecond * 40) - for _, m := range cluster { - err := m.Close() - if err != nil { - t.Fatal(err) - } - } - -} From 3db1087bce8a210f5d3e9bf06ce8450b42b42faa Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 29 Jun 2018 15:33:23 -0500 Subject: [PATCH 4/6] more comprehensive time quantum tests --- executor_test.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/executor_test.go b/executor_test.go index 7a0991288..2746ab715 100644 --- a/executor_test.go +++ b/executor_test.go @@ -19,6 +19,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "testing" "github.com/davecgh/go-spew/spew" @@ -1500,3 +1501,64 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { } } + +func TestExecutor_Time_Clear_Quantums(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + + var rangeTests = []struct { + quantum pilosa.TimeQuantum + expected []uint64 + }{ + {quantum: "Y", expected: []uint64{3, 4, 5, 6}}, + {quantum: "M", expected: []uint64{3, 4, 6}}, + {quantum: "D", expected: []uint64{3, 4, 5, 6}}, + {quantum: "H", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "YM", expected: []uint64{3, 4, 5, 6}}, + {quantum: "YMD", expected: []uint64{3, 4, 5, 6}}, + {quantum: "YMDH", expected: []uint64{3, 4, 5, 6, 7}}, + } + populateBatch := test.MustParse(` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + `) + clearColumn := test.MustParse(`Clear( 2, f=1)`) + rangeCheckQuery := test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`) + + for i, tt := range rangeTests { + t.Run(fmt.Sprintf("#%d Quantum %s", i+1, tt.quantum), func(t *testing.T) { + // Create index. + indexName := strings.ToLower(string(tt.quantum)) + index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{}) + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeTime, + TimeQuantum: tt.quantum, + }); err != nil { + t.Fatal(err) + } + // Populate + if _, err := e.Execute(context.Background(), indexName, populateBatch, nil, nil); err != nil { + t.Fatal(err) + } + if _, err := e.Execute(context.Background(), indexName, clearColumn, nil, nil); err != nil { + t.Fatal(err) + } + if res, err := e.Execute(context.Background(), indexName, rangeCheckQuery, nil, nil); err != nil { + t.Fatal(err) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, tt.expected) { + t.Fatalf("unexpected columns: %+v", columns) + } + + }) + } + +} From 785adfec2a3c6350fc39b4eb99a5c83b2514f888 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 29 Jun 2018 17:35:40 -0500 Subject: [PATCH 5/6] more complete permutations --- executor_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/executor_test.go b/executor_test.go index 2746ab715..eea974c77 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1518,6 +1518,9 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { {quantum: "YM", expected: []uint64{3, 4, 5, 6}}, {quantum: "YMD", expected: []uint64{3, 4, 5, 6}}, {quantum: "YMDH", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "MD", expected: []uint64{3, 4, 5, 6}}, + {quantum: "MDH", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "DH", expected: []uint64{3, 4, 5, 6, 7}}, } populateBatch := test.MustParse(` Set(2, f=1, 1999-12-31T00:00) From 29ad1287d4b3ac90d47119bf87dcc1c22f5db065 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sat, 30 Jun 2018 08:41:19 -0500 Subject: [PATCH 6/6] refactor executor_test.go --- executor_test.go | 880 +++++++++++++++++--------------------------- http/client_test.go | 11 +- test/cluster.go | 12 +- test/holder.go | 9 + 4 files changed, 360 insertions(+), 552 deletions(-) diff --git a/executor_test.go b/executor_test.go index 7a0991288..0bf811bbe 100644 --- a/executor_test.go +++ b/executor_test.go @@ -25,7 +25,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" "github.com/pkg/errors" ) @@ -33,71 +33,70 @@ import ( // Ensure a bitmap query can be executed. func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Row", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) f, err := index.CreateField("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - ), nil, nil); err != nil { + }); err != nil { t.Fatal(err) } if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit column attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeColumns: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit row attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeRowAttrs: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("Column", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - ), nil, nil); err != nil { + }); err != nil { t.Fatal(err) } if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { @@ -106,28 +105,33 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { }) t.Run("Keys", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - `Set("foo", f="bar")`+"\n"+ - `Set("foo", f="baz")`+"\n"+ - `Set("bat", f="bar")`+"\n"+ - `Set("aaa", f="bbb")`+"\n", - ), nil, nil); err != nil { - t.Fatal(err) + _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `` + + `Set("foo", f="bar")` + "\n" + + `Set("foo", f="baz")` + "\n" + + `Set("bat", f="bar")` + "\n" + + `Set("aaa", f="bbb")` + "\n", + }) + if err != nil { + t.Fatalf("querying: %v", err) } - if results, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f="bar")`), nil, nil); err != nil { + if results, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `Row(f="bar")`, + }); err != nil { t.Fatal(err) - } else if diff := cmp.Diff(results, []interface{}{ + } else if diff := cmp.Diff(results.Results, []interface{}{ &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { t.Fatal(diff) @@ -137,38 +141,39 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, 2) hldr.SetBit("i", "general", 10, 3) hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, 4) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty difference query behaves properly. func TestExecutor_Execute_Empty_Difference(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 1) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference()`}); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } } // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -177,29 +182,28 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty intersect query behaves properly. func TestExecutor_Execute_Empty_Intersect(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect()`}); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } } // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 0) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -207,32 +211,33 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty union query behaves properly. func TestExecutor_Execute_Empty_Union(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 0) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union()`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure a xor query can be executed. func TestExecutor_Execute_Xor(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -240,27 +245,27 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "f", 10, 3) hldr.SetBit("i", "f", 10, ShardWidth+1) hldr.SetBit("i", "f", 10, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { t.Fatal(err) - } else if res[0] != uint64(3) { - t.Fatalf("unexpected n: %d", res[0]) + } else if res.Results[0] != uint64(3) { + t.Fatalf("unexpected n: %d", res.Results[0]) } } @@ -370,24 +375,24 @@ func TestExecutor_Execute_SetBit(t *testing.T) { // Ensure old PQL syntax doesn't break anything too badly. func TestExecutor_Execute_OldPQL(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // set a bit so the view gets created. hldr.SetBit("i", "f", 1, 0) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err == nil || err.Error() != "unknown call: SetBit" { - t.Fatal("Expected error: 'unknown call: SetBit'") + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { + t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err)) } } // Ensure a SetValue() query can be executed. func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create felds. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -402,10 +407,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil { + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil { t.Fatal(err) } @@ -428,8 +432,10 @@ func TestExecutor_Execute_SetValue(t *testing.T) { }) t.Run("", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, @@ -440,22 +446,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) } }) @@ -464,8 +467,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Ensure a SetRowAttrs() query can be executed. func TestExecutor_Execute_SetRowAttrs(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -477,17 +481,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other bitmaps and fields to test isolation. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 200, YYY=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(xxx, 10, YYY=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { t.Fatal(err) } @@ -502,9 +505,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { t.Run("ID", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -513,17 +516,17 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(1, f=0) - Set(`+strconv.Itoa(ShardWidth)+`, f=0) - Set(`+strconv.Itoa(ShardWidth+2)+`, f=0) - Set(`+strconv.Itoa((5*ShardWidth)+100)+`, f=0) + Set(` + strconv.Itoa(ShardWidth) + `, f=0) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=0) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=0) Set(0, f=10) - Set(`+strconv.Itoa(ShardWidth)+`, f=10) - Set(`+strconv.Itoa(ShardWidth)+`, f=20) + Set(` + strconv.Itoa(ShardWidth) + `, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=20) Set(0, other=0) - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } @@ -531,9 +534,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ {ID: 0, Count: 5}, {ID: 10, Count: 2}, }) { @@ -542,9 +545,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { }) t.Run("Keys", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { @@ -553,7 +556,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("a", f="foo") Set("b", f="foo") Set("c", f="foo") @@ -563,15 +566,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { Set("b", f="bar") Set("b", f="baz") Set("a", other="foo") - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if diff := cmp.Diff(result, []interface{}{ + } else if diff := cmp.Diff(result.Results, []interface{}{ []pilosa.Pair{ {Key: "foo", Count: 5}, {Key: "bar", Count: 2}, @@ -583,8 +586,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { } func TestExecutor_Execute_TopN_fill(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) @@ -595,10 +599,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.SetBit("i", "f", 1, ShardWidth) // Execute query. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -607,8 +610,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Ensure func TestExecutor_Execute_TopN_fill_small(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, ShardWidth) @@ -629,10 +633,9 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.SetBit("i", "f", 4, 3*ShardWidth+1) // Execute query. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -641,8 +644,9 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Ensure a TopN() query with a source bitmap can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) @@ -664,10 +668,9 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(other=100), n=3)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, {ID: 10, Count: 2}, {ID: 0, Count: 1}, @@ -678,9 +681,9 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { //Ensure TopN handles Attribute filters func TestExecutor_Execute_TopN_Attr(t *testing.T) { - // - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 10, ShardWidth) @@ -688,10 +691,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -701,9 +703,10 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { //Ensure TopN handles Attribute filters with source bitmap func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { - // - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 10, ShardWidth) @@ -711,10 +714,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -723,9 +725,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure Min() and Max() queries can be executed. func TestExecutor_Execute_MinMax(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -744,22 +746,22 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, x=0) Set(3, x=0) - Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) Set(1, x=1) - Set(`+strconv.Itoa(ShardWidth+2)+`, x=2) + Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) SetValue(col=0, f=20) SetValue(col=1, f=-5) SetValue(col=2, f=-5) SetValue(col=3, f=10) - SetValue(col=`+strconv.Itoa(ShardWidth)+`, f=30) - SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, f=40) - SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, f=50) - SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, f=60) - `), nil, nil); err != nil { + SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30) + SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40) + SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60) + `}); err != nil { t.Fatal(err) } @@ -781,9 +783,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } - if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -807,9 +809,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } - if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -818,9 +820,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -855,33 +857,33 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, x=0) - Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) SetValue(col=0, foo=20) SetValue(col=0, bar=2000) - SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=40) - SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=50) - SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) + SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) + SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40) + SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) SetValue(col=0, other=1000) - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(field=foo)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Row(x=0), field=foo)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -889,9 +891,9 @@ func TestExecutor_Execute_Sum(t *testing.T) { // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -905,7 +907,7 @@ func TestExecutor_Execute_Range(t *testing.T) { } // Set columns. - cc := test.MustParse(` + cc := ` Set(2, f=1, 1999-12-31T00:00) Set(3, f=1, 2000-01-01T00:00) Set(4, f=1, 2000-01-02T00:00) @@ -916,27 +918,27 @@ func TestExecutor_Execute_Range(t *testing.T) { Set(2, f=1, 1999-12-30T00:00) Set(2, f=1, 2002-02-01T00:00) Set(2, f=10, 2001-01-01T00:00) - `) - if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { + ` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) t.Run("Clear", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Clear( 2, f=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f=1)`}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) @@ -944,9 +946,9 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure a Range(bsiGroup) query can be executed. func TestExecutor_Execute_BSIGroupRange(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -989,136 +991,136 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) - Set(`+strconv.Itoa(ShardWidth+1)+`, f=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) SetValue(col=50, foo=20) SetValue(col=50, bar=2000) - SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=10) - SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=20) - SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) + SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) + SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10) + SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) SetValue(col=0, other=1000) SetValue(col=0, edge=100) SetValue(col=1, edge=-100) - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } t.Run("EQ", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != null)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != null)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo != 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != -20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != -20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns()) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) t.Run("LT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo < 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("LTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo <= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("GT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo > 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("GTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo >= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(0 < other < 1000)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(-1 < other < 1000)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("BelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 0)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 0)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("AboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge < 200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge < 200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) } }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge > -200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) } }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -1126,351 +1128,153 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { - t.Skip() // Until test.NewServer() works + c := test.MustRunCluster(t, 2, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + ) + defer c.Close() + hldr0 := test.Holder{Holder: c[0].Server.Holder()} + hldr1 := test.Holder{Holder: c[1].Server.Holder()} - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { - t.Fatal(err) + t.Fatalf("creating index: %v", err) } - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Row(f=10)` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(shards, []uint64{1}) { - t.Fatalf("unexpected shards: %+v", shards) - } - - // Set columns in shard 0 & 2. - r := pilosa.NewRow( - (0*ShardWidth)+1, - (0*ShardWidth)+2, - (2*ShardWidth)+4, - ) - return []interface{}{r}, nil + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field: %v", err) } - // Create local executor data. - // The local node owns shard 1. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr1.MustSetBits("i", "f", 10, ShardWidth+1, ShardWidth+2, (3*ShardWidth)+4) + hldr0.SetBit("i", "f", 10, 1) - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*ShardWidth + 4}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 1, ShardWidth + 2, (3 * ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } -} -// Ensure a remote query can return a count. -func TestExecutor_Execute_Remote_Count(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(10)}, nil - } - - // Create local executor data. The local node owns shard 1. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, (2*ShardWidth)+1) - hldr.SetBit("i", "f", 10, (2*ShardWidth)+2) - - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { - t.Fatal(err) - } else if res[0] != uint64(12) { - t.Fatalf("unexpected n: %d", res[0]) - } -} - -// Ensure a remote query can set columns on multiple nodes. -func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - c.ReplicaN = 2 - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments. - var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != `i` { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Set(_col=2, f=10)` { - t.Fatalf("unexpected query: %s", query.String()) + t.Run("Count", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + t.Fatal(err) + } else if res.Results[0] != uint64(4) { + t.Fatalf("unexpected n: %d", res.Results[0]) } - remoteCalled = true - return []interface{}{nil}, nil - } + }) - // Create local executor data. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - - // Create field. - if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - e := test.NewExecutor(hldr.Holder, c) - cc := test.MustParse("Set(2, f=10)") - if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { - t.Fatal(err) - } - - // Verify that one column is set on both node's holder. - if n := hldr.Row("i", "f", 10).Count(); n != 1 { - t.Fatalf("unexpected local count: %d", n) - } - if !remoteCalled { - t.Fatalf("expected remote execution") - } -} - -// Ensure a remote query can set columns on multiple nodes. -func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - c.ReplicaN = 2 - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments. - var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != `i` { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` { - t.Fatalf("unexpected query: %s", query.String()) - } - remoteCalled = true - return []interface{}{nil}, nil - } - - // Create local executor data. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - - // Create field. - if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if err := f.SetTimeQuantum("Y"); err != nil { - t.Fatal(err) - } - - e := test.NewExecutor(hldr.Holder, c) - cc := test.MustParse(`Set(2, f=10, 2016-12-11T10:09)`) - if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { - t.Fatal(err) - } - - // Verify that one column is set on both node's holder. - if n := hldr.ViewRow("i", "f", "standard_2016", 10).Count(); n != 1 { - t.Fatalf("unexpected local count: %d", n) - } - if !remoteCalled { - t.Fatalf("expected remote execution") - } -} - -// Ensure a remote query can return a top-n query. -func TestExecutor_Execute_Remote_TopN(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - var remoteExecN int - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if !reflect.DeepEqual(shards, []uint64{1, 3}) { - t.Fatalf("unexpected shards: %+v", shards) + t.Run("Remote SetBit", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil { + t.Fatalf("quuerying remote: %v", err) } - // Query should be executed twice. Once to get the top bitmaps for the - // shards and a second time to get the counts for a set of bitmaps. - switch remoteExecN { - case 0: - if query.String() != `TopN(_field="f", n=3)` { - t.Fatalf("unexpected query(0): %s", query.String()) - } - case 1: - if query.String() != `TopN(_field="f", ids=[0,10,30], n=3)` { - t.Fatalf("unexpected query(1): %s", query.String()) - } - default: - t.Fatalf("too many remote exec calls") + if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) { + t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns()) } - remoteExecN++ + }) - // Return pair counts. - return []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, - {ID: 30, Count: 2}, - }}, nil - } - - // Create local executor data on shard 2 & 4. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 30, (2*ShardWidth)+1) - hldr.SetBit("i", "f", 30, (4*ShardWidth)+2) - - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 30, Count: 4}, - {ID: 10, Count: 2}, - }}) { - t.Fatalf("unexpected results: %s", spew.Sdump(res)) - } -} - -// Ensure a remote query can set RowAttrs -func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { - t.Skip("test.NewServer broken") - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` { - t.Fatalf("unexpected query: %s", query.String()) + t.Run("remote with timestamp", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) + if err != nil { + t.Fatalf("creating field: %v", err) } - return []interface{}{}, nil - } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil { + t.Fatalf("quuerying remote: %v", err) + } - // Create local executor data. - // The local node owns shard 1. - hldr := test.MustOpenHolder() - defer hldr.Close() + if !reflect.DeepEqual(hldr1.ViewRow("i", "z", "standard_2010", 5).Columns(), []uint64{1500000}) { + t.Fatalf("unexpected cols from row 7: %v", hldr1.ViewRow("i", "z", "standard_2010", 5).Columns()) + } + }) - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - f := hldr.Field("i", "f") - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, ShardWidth+1) + t.Run("remote topn", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } - e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { - t.Fatal(err) - } else if m, err := f.RowAttrStore().Attrs(10); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"bat": true, "baz": int64(123)}) { - t.Fatalf("unexpected bitmap attr: %#v", m) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` +Set(500001, fn=5) +Set(1500001, fn=5) +Set(2500001, fn=5) +Set(3500001, fn=5) +Set(1500001, fn=3) +Set(1500002, fn=3) +Set(3500003, fn=3) +Set(500001, fn=4) +Set(4500001, fn=4) +`}); err != nil { + t.Fatalf("quuerying remote: %v", err) + } + err := c[0].API.RecalculateCaches(context.Background()) + if err != nil { + t.Fatalf("recalcing caches: %v", err) + } - } + if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `TopN(fn, n=3)`, + }); err != nil { + t.Fatalf("topn querying: %v", err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 5, Count: 4}, + {ID: 3, Count: 3}, + {ID: 4, Count: 2}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + }) + + t.Run("remote setrowattrs", func(t *testing.T) { + if _, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)`, + }); err != nil { + t.Fatalf("setrowattrs querying: %v", err) + } else if attrst, err := hldr0.RowAttrStore("i", "f").Attrs(10); err != nil || !attrst["bat"].(bool) || attrst["baz"].(int64) != 123 { + t.Fatalf("wrong attrs: %v", attrst) + } + }) } // Ensure executor returns an error if too many writes are in a single request. func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustNewCluster(t, 1) + c[0].Config.MaxWritesPerRequest = 3 + err := c.Start() + if err != nil { + t.Fatal(err) + } + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - e.MaxWritesPerRequest = 3 - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Set() Clear() Set() Set()`), nil, nil); err != pilosa.ErrTooManyWrites { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set() Clear() Set() Set()`}); errors.Cause(err) != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } // Ensure SetColumnAttrs doesn't save `field` as an attribute func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateField("f", pilosa.FieldOptions{}) + _, err := index.CreateField("f", pilosa.FieldOptions{}) + if err != nil { + t.Fatalf("creating field: %v", err) + } targetAttrs := map[string]interface{}{ "foo": "bar", } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // SetColumnAttrs call should exclude the field attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("Set(10, f=1)"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(10, f=1)"}) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(10, foo='bar')"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(10, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -1483,11 +1287,11 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { } // SetColumnAttrs call should not break if field is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("Set(20, f=10)"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(20, f=10)"}) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(20, foo='bar')"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(20, foo='bar')"}) if err != nil { t.Fatal(err) } diff --git a/http/client_test.go b/http/client_test.go index ec849ba10..fc8e896b5 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -37,20 +37,15 @@ func init() { } -// modHasher represents a simple, mod-based hashing. -type modHasher struct{} - -func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } - // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { c := test.MustRunCluster(t, 3, []server.CommandOption{ - server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&modHasher{}))}, + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, []server.CommandOption{ - server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&modHasher{}))}, + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, []server.CommandOption{ - server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&modHasher{}))}, + server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, ) defer c.Close() diff --git a/test/cluster.go b/test/cluster.go index 8c1e3791e..d7363e320 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -21,6 +21,11 @@ import ( "github.com/pilosa/pilosa" ) +// modHasher represents a simple, mod-based hashing. +type ModHasher struct{} + +func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } + // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { path, err := ioutil.TempDir("", "pilosa-cluster-") @@ -30,7 +35,7 @@ func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() c.ReplicaN = 1 - c.Hasher = &modHasher{} + c.Hasher = &ModHasher{} c.Path = path c.Topology = pilosa.NewTopology() @@ -56,8 +61,3 @@ func newURI(scheme, host string, port uint16) pilosa.URI { uri.SetPort(port) return *uri } - -// modHasher represents a simple, mod-based hashing. -type modHasher struct{} - -func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } diff --git a/test/holder.go b/test/holder.go index 277c774dd..6cdd57f55 100644 --- a/test/holder.go +++ b/test/holder.go @@ -121,6 +121,15 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { return row } +func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + return f.RowAttrStore() +} + // ViewRow returns a Row for a given field and view. func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})