diff --git a/.circleci/config.yml b/.circleci/config.yml index bd1ca86cd..605990e5b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,12 @@ jobs: <<: *base-test environment: GOARCH: 386 + cluster-tests: + <<: *defaults + steps: + - *fast-checkout + - setup_remote_docker + - run: make clustertests-build prerelease: <<: *base-test steps: @@ -99,6 +105,9 @@ workflows: - test-golang-1.10-386: requires: - build + - cluster-tests: + requires: + - build - prerelease: requires: - linter diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6c77fe0fc..000000000 --- a/.travis.yml +++ /dev/null @@ -1,52 +0,0 @@ -language: go -go: - - "1.10" # Use string, as 1.10==1.1 if interpreted as float. - - master -env: - global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" - matrix: - - GOARCH=386 - - GOARCH=386 ENTERPRISE=1 - - GOARCH=amd64 - - GOARCH=amd64 ENTERPRISE=1 -cache: - directories: - vendor -install: - - make -B install-dep vendor -script: make test -jobs: - include: - - stage: metalinter - install: - - make -B install-dep vendor install-gometalinter - script: make gometalinter - env: - - GOARCH=amd64 - - stage: deploy - script: skip - go: "1.10" - env: - - GOARCH=amd64 - deploy: - - provider: script - script: pip install awscli --user `whoami` && make -B prerelease - skip_cleanup: true - on: - all_branches: true - repo: pilosa/pilosa - tags: false -stages: - - metalinter - - test - - deploy -matrix: - # Excluding or allowing failures on non-primary matrix configurations due to long running times. - fast_finish: true - allow_failures: - - go: master -notifications: - slack: - secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ad9ab13d..3939e927e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,7 @@ If you want to help but you aren't sure where to start, check out our [github la ### Development Environment -- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../master/.travis.yml) for the most up-to-date information. +- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information. - Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests new file mode 100644 index 000000000..c60fda576 --- /dev/null +++ b/Dockerfile-clustertests @@ -0,0 +1,26 @@ +# This Dockerfile is used for cluster testing - it produces a much larger image +# and includes all of Go as well as some utilities. + +FROM golang:1.11 + +LABEL maintainer "dev@pilosa.com" + +COPY . /go/src/github.com/pilosa/pilosa/ + +RUN cd /go/src/github.com/pilosa/pilosa \ + && CGO_ENABLED=0 make install-dep install FLAGS="-a" + +# download pumba for fault injection +ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba +RUN chmod +x /pumba + +RUN cp /go/bin/pilosa /pilosa + +COPY LICENSE /LICENSE +COPY NOTICE /NOTICE + +EXPOSE 10101 +VOLUME /data + +ENTRYPOINT ["bash", "-c"] +CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/Gopkg.lock b/Gopkg.lock index 4a6068840..280f6d056 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -189,6 +189,15 @@ revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" version = "v1.2.0" +[[projects]] + name = "github.com/pilosa/go-pilosa" + packages = [ + ".", + "gopilosa_pbuf" + ] + revision = "4e7807f5ad779407936744057cd17332046b6c3c" + version = "v1.1.0" + [[projects]] name = "github.com/pkg/errors" packages = ["."] @@ -324,6 +333,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8290156ce8b4066c46ab83d743f4c81df0a17e148415bb1ee8409a51ac4c3ba4" + inputs-digest = "be318fa4f2a72e7e849b2faff1ce9300deaaf2d24cf76100f4b538abed86295b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index b403fde23..34f97965e 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,21 @@ release: check-clean $(MAKE) release-build GOOS=linux GOARCH=386 $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1 +# Run cluster integration tests using docker. Requires docker daemon to be +# running. This will catch changes to internal/clustertests/*.go, but if you +# make changes to Pilosa, you'll want to run clustertests-build to rebuild the +# pilosa image. +clustertests: + docker-compose -f internal/clustertests/docker-compose.yml down + docker-compose -f internal/clustertests/docker-compose.yml build client1 + docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 + + +# Like clustertests, but rebuilds all images. +clustertests-build: + docker-compose -f internal/clustertests/docker-compose.yml down + docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build + # Create prerelease builds prerelease: vendor $(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\) diff --git a/README.md b/README.md index 7c1c7cfda..8ccae2b91 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-[![Build Status](https://travis-ci.org/pilosa/pilosa.svg?branch=master)](https://travis-ci.org/pilosa/pilosa) +[![CircleCI](https://circleci.com/gh/pilosa/pilosa/tree/master.svg?style=shield)](https://circleci.com/gh/pilosa/pilosa/tree/master) [![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa) [![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa) [![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE) diff --git a/apimethod_string.go b/apimethod_string.go index 01217092f..cafeab5af 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 166, 182, 191, 205, 213, 229, 237, 257, 270, 284, 301, 314, 322} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/cluster.go b/cluster.go index c3bc67b4d..4b1ab9089 100644 --- a/cluster.go +++ b/cluster.go @@ -65,10 +65,11 @@ type Node struct { ID string `json:"id"` URI URI `json:"uri"` IsCoordinator bool `json:"isCoordinator"` + State string `json:"state"` } func (n Node) String() string { - return fmt.Sprintf("Node: %s", n.ID) + return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6]) } // Nodes represents a list of nodes. @@ -456,7 +457,19 @@ func (c *cluster) unprotectedSetState(state string) { } } +func (c *cluster) setMyNodeState(state string) { + c.mu.Lock() + defer c.mu.Unlock() + c.Node.State = state + for i, n := range c.nodes { + if n.ID == c.Node.ID { + c.nodes[i].State = state + } + } +} + func (c *cluster) setNodeState(state string) error { // nolint: unparam + c.setMyNodeState(state) if c.isCoordinator() { return c.receiveNodeState(c.Node.ID, state) } @@ -486,11 +499,23 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { } c.Topology.mu.Lock() - c.Topology.nodeStates[nodeID] = state + changed := false + if c.Topology.nodeStates[nodeID] != state { + changed = true + c.Topology.nodeStates[nodeID] = state + for i, n := range c.nodes { + if n.ID == nodeID { + c.nodes[i].State = state + } + } + } c.Topology.mu.Unlock() c.logger.Printf("received state %s (%s)", state, nodeID) - return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) + if changed { + return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) + } + return nil } // determineClusterState is unprotected. @@ -932,7 +957,6 @@ func (c *cluster) waitForStarted() error { <-c.joining c.logger.Printf("joining has completed") } - return nil } @@ -1043,8 +1067,9 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error { return nil } // Broadcast cluster status changes to the cluster. - c.logger.Printf("broadcasting ClusterStatus: %s", state) - return c.broadcaster.SendSync(c.unprotectedStatus()) // TODO fix c.Status + status := c.unprotectedStatus() + c.logger.Printf("broadcasting ClusterStatus: %s", status) + return c.broadcaster.SendSync(status) // TODO fix c.Status } @@ -1625,7 +1650,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { switch e.Event { case NodeJoin: - c.logger.Printf("received NodeJoin event: %v", e) + c.logger.Printf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI) // Ignore the event if this is not the coordinator. if !c.isCoordinator() { return nil @@ -1660,6 +1685,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { func (c *cluster) nodeJoin(node *Node) error { c.mu.Lock() defer c.mu.Unlock() + c.logger.Printf("NodeJoin event on coordinator, node: %s, id: %s", node.URI, node.ID) if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsID(node.ID) { @@ -1688,11 +1714,10 @@ func (c *cluster) nodeJoin(node *Node) error { if c.haveTopologyAgreement() && c.allNodesReady() { return c.unprotectedSetStateAndBroadcast(ClusterStateNormal) - } else { - // Send the status to the remote node. This lets the remote node - // know that it can proceed with opening its Holder. - return c.sendTo(node, c.unprotectedStatus()) } + // Send the status to the remote node. This lets the remote node + // know that it can proceed with opening its Holder. + return c.sendTo(node, c.unprotectedStatus()) } // If the cluster already contains the node, just send it the cluster status. @@ -1796,6 +1821,10 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { // Add all nodes from the coordinator. for _, node := range officialNodes { + if node.ID == c.Node.ID && node.State != c.Node.State { + c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State) + go c.setNodeState(c.Node.State) + } if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index a2b529fba..50602d385 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -185,8 +185,8 @@ func TestFragSources(t *testing.T) { "node0": {}, "node1": {}, "node2": { - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -197,11 +197,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*ResizeSource{ "node0": { - {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + {&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)}, }, "node1": { - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -212,11 +212,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*ResizeSource{ "node0": { - {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + {&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, "node1": { - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)}, }, "node2": {}, }, diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 07c75b8da..11e1cf8d2 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -534,6 +534,7 @@ func encodeNode(n *pilosa.Node) *internal.Node { ID: n.ID, URI: encodeURI(n.URI), IsCoordinator: n.IsCoordinator, + State: n.State, } } @@ -789,6 +790,7 @@ func decodeNode(node *internal.Node, m *pilosa.Node) { m.ID = node.ID decodeURI(node.URI, &m.URI) m.IsCoordinator = node.IsCoordinator + m.State = node.State } func decodeURI(i *internal.URI, m *pilosa.URI) { diff --git a/gossip/gossip.go b/gossip/gossip.go index 789ed50b0..ecd663ecf 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "io" "io/ioutil" "log" "net" @@ -49,6 +50,7 @@ type memberSet struct { Logger pilosa.Logger logger *log.Logger + logOutput io.Writer transport *Transport eventReceiver *eventReceiver @@ -156,6 +158,13 @@ func WithLogger(logger *log.Logger) memberSetOption { } } +func WithLogOutput(o io.Writer) memberSetOption { + return func(g *memberSet) error { + g.logOutput = o + return nil + } +} + // NewMemberSet returns a new instance of GossipMemberSet based on options. func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) { host := api.Node().URI.Host @@ -220,7 +229,11 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem conf.Delegate = g conf.SecretKey = gossipKey conf.Events = ger - conf.Logger = g.logger + if g.logOutput != nil { + conf.LogOutput = g.logOutput + } else { + conf.Logger = g.logger + } g.config = &config{ memberlistConfig: conf, diff --git a/internal/clustertests/Dockerfile b/internal/clustertests/Dockerfile new file mode 100644 index 000000000..ecbfa7f21 --- /dev/null +++ b/internal/clustertests/Dockerfile @@ -0,0 +1,3 @@ +FROM ptest + +COPY . /go/src/github.com/pilosa/pilosa/internal/clustertests diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go new file mode 100644 index 000000000..f01835c10 --- /dev/null +++ b/internal/clustertests/cluster_test.go @@ -0,0 +1,81 @@ +package clustertest + +import ( + "context" + "os" + "os/exec" + "testing" + "time" + + "github.com/pilosa/pilosa" + picli "github.com/pilosa/pilosa/http" +) + +func TestClusterStuff(t *testing.T) { + if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" { + t.Skip() + } + cli, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil)) + if err != nil { + t.Fatalf("getting client: %v", err) + } + + t.Run("long pause", func(t *testing.T) { + err := cli.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + err = cli.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + data := make([]pilosa.Bit, 10) + for i := 0; i < 1000; i++ { + data[i%10].RowID = 0 + data[i%10].ColumnID = uint64((i/10)*pilosa.ShardWidth + i%10) + shard := uint64(i / 10) + if i%10 == 9 { + err = cli.Import(context.Background(), "testidx", "testf", shard, data) + if err != nil { + t.Fatalf("importing: %v", err) + } + } + } + + r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) + if err != nil { + t.Fatalf("count querying: %v", err) + } + if r.Results[0].(uint64) != 1000 { + t.Fatalf("count after import is %d", r.Results[0].(uint64)) + } + + pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s") + pcmd.Stdout = os.Stdout + pcmd.Stderr = os.Stderr + t.Log("pausing pilosa3 for 10s") + err = pcmd.Start() + if err != nil { + t.Fatalf("starting pumba command: %v", err) + } + err = pcmd.Wait() + if err != nil { + t.Fatalf("waiting on pumba pause cmd: %v", err) + } + + // TODO change the sleep to wait for status to return to NORMAL - need support in internal client for getting status + t.Log("done with pause, waiting for stability") + time.Sleep(time.Second * 3) + t.Log("done waiting for stability") + + r, err = cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) + if err != nil { + t.Fatalf("count querying: %v", err) + } + if r.Results[0].(uint64) != 1000 { + t.Fatalf("count after import is %d", r.Results[0].(uint64)) + } + }) + +} diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml new file mode 100644 index 000000000..8586eb08f --- /dev/null +++ b/internal/clustertests/docker-compose.yml @@ -0,0 +1,55 @@ +version: '2' +services: + pilosa1: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33455:10101" + environment: + - PILOSA_CLUSTER_COORDINATOR=true + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa1:10101" + pilosa2: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33456:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa2:10101" + pilosa3: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33457:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa3:10101" + client1: + build: + context: . + environment: + - ENABLE_PILOSA_CLUSTER_TESTS=1 + networks: + - pilosanet + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" +networks: + pilosanet: diff --git a/internal/private.pb.go b/internal/private.pb.go index 542f59939..376770e44 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -541,6 +541,7 @@ type Node struct { ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` + State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` } func (m *Node) Reset() { *m = Node{} } @@ -569,6 +570,13 @@ func (m *Node) GetIsCoordinator() bool { return false } +func (m *Node) GetState() string { + if m != nil { + return m.State + } + return "" +} + type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -1750,6 +1758,12 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { } i++ } + if len(m.State) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } return i, nil } @@ -2694,6 +2708,10 @@ func (m *Node) Size() (n int) { if m.IsCoordinator { n += 2 } + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -5245,6 +5263,35 @@ func (m *Node) Unmarshal(dAtA []byte) error { } } m.IsCoordinator = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7417,75 +7464,76 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1113 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, - 0x18, 0x66, 0x0f, 0x76, 0xec, 0xdf, 0x75, 0x9a, 0x6c, 0x69, 0xd9, 0x02, 0x0a, 0x61, 0x54, 0xd1, - 0x50, 0x89, 0x50, 0xb5, 0x37, 0x9c, 0x2a, 0x95, 0xc4, 0xa1, 0x2c, 0x25, 0xa5, 0xcc, 0xa6, 0xb9, - 0xeb, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90, - 0x78, 0x01, 0xc4, 0x93, 0xf0, 0x08, 0x5c, 0xf2, 0x08, 0x28, 0xbc, 0x08, 0x9a, 0x7f, 0x66, 0x76, - 0x37, 0x8e, 0x43, 0xa2, 0xc0, 0xdd, 0xfc, 0xdf, 0x7f, 0x3e, 0xae, 0x0d, 0xfd, 0x49, 0x9e, 0x1c, - 0x32, 0xc9, 0xd7, 0x27, 0xb9, 0x90, 0x22, 0xe8, 0x24, 0x99, 0xe4, 0x79, 0xc6, 0x52, 0xf2, 0x04, - 0xba, 0x51, 0x36, 0xe2, 0xc7, 0xdb, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x29, 0x9f, 0x16, 0xa1, 0xb7, - 0xea, 0xac, 0x75, 0x28, 0xbe, 0x83, 0x0f, 0x60, 0x71, 0x27, 0x67, 0xc3, 0x83, 0xad, 0xe3, 0xa4, - 0x90, 0x3c, 0x1b, 0xf2, 0xd0, 0x47, 0xee, 0x0c, 0x4a, 0x7e, 0x77, 0xe0, 0xda, 0x57, 0x09, 0x4f, - 0x47, 0xdf, 0x4d, 0x64, 0x22, 0xb2, 0x22, 0x78, 0x17, 0xba, 0x9b, 0x6c, 0xb8, 0xcf, 0x77, 0xa6, - 0x13, 0x8e, 0x16, 0xbb, 0xb4, 0x06, 0x2a, 0x6e, 0x9c, 0xbc, 0xd6, 0x16, 0xfb, 0xb4, 0x06, 0x82, - 0x55, 0xe8, 0xed, 0x24, 0x63, 0xfe, 0x7d, 0xc9, 0x32, 0x59, 0x8e, 0xc3, 0x16, 0x6a, 0x37, 0x21, - 0x15, 0x2a, 0x1a, 0xee, 0x20, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0xdb, 0x49, 0x16, 0x76, 0x57, 0x9d, - 0x35, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x38, 0x04, 0x83, 0xb0, 0xe3, 0x2a, 0xc5, 0x5e, 0x9d, 0x22, - 0x21, 0xb0, 0x18, 0x8d, 0x27, 0x22, 0x97, 0x94, 0x17, 0x13, 0x91, 0x15, 0x68, 0x69, 0x2b, 0xcf, - 0x43, 0x07, 0x8d, 0xab, 0x27, 0xf9, 0x11, 0x96, 0x36, 0x52, 0x31, 0x3c, 0x18, 0x30, 0xc9, 0x28, - 0xff, 0xa1, 0xe4, 0x85, 0x0c, 0xde, 0x84, 0x16, 0xd6, 0xce, 0xc8, 0x69, 0x42, 0xa1, 0x58, 0x87, - 0xd0, 0xd5, 0x28, 0x12, 0x0a, 0x45, 0x7d, 0xac, 0x84, 0x4f, 0x35, 0xa1, 0xd0, 0x78, 0x9f, 0xe5, - 0x23, 0xac, 0x80, 0x4f, 0x35, 0xa1, 0x62, 0xdc, 0x4d, 0xf8, 0x91, 0x49, 0x1b, 0xdf, 0x24, 0x82, - 0xe5, 0x86, 0x7f, 0x13, 0xe6, 0x2d, 0x68, 0x53, 0x71, 0x14, 0x0d, 0x8a, 0xd0, 0x59, 0xf5, 0xd6, - 0x7c, 0x6a, 0x28, 0x2c, 0xae, 0x48, 0xcb, 0x71, 0xa6, 0x58, 0x2e, 0xb2, 0x6a, 0x80, 0xdc, 0x86, - 0x16, 0x56, 0x5a, 0x65, 0x59, 0xeb, 0xaa, 0x27, 0xf9, 0xc9, 0x81, 0xee, 0x36, 0x3b, 0xc6, 0x30, - 0x8a, 0xe0, 0x11, 0x74, 0x62, 0xc9, 0xb2, 0x91, 0x0a, 0x50, 0x09, 0xf5, 0x1e, 0xbc, 0xbf, 0x6e, - 0x07, 0x67, 0xbd, 0x12, 0x5b, 0xb7, 0x32, 0x5b, 0x99, 0xcc, 0xa7, 0xb4, 0x52, 0x79, 0xfb, 0x73, - 0xe8, 0x9f, 0x62, 0x29, 0x7f, 0x07, 0x7c, 0x6a, 0xab, 0x7a, 0xc0, 0xa7, 0x2a, 0xff, 0x43, 0x96, - 0x96, 0x1c, 0x6b, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x5d, 0x08, 0x36, 0x73, 0xce, - 0x24, 0x47, 0x27, 0xdb, 0xbc, 0x28, 0xd8, 0x2b, 0x7e, 0x7e, 0xc5, 0x75, 0x15, 0xdd, 0x66, 0x15, - 0xab, 0x3e, 0x78, 0x8d, 0x3e, 0x90, 0x7b, 0x10, 0x0c, 0x78, 0xca, 0x25, 0x37, 0x53, 0xff, 0x2f, - 0x76, 0x49, 0x6c, 0x63, 0xb8, 0x58, 0x36, 0xb8, 0x0b, 0xbe, 0x5a, 0x21, 0x0c, 0xa1, 0xf7, 0xe0, - 0x46, 0x5d, 0xa7, 0x6a, 0xbb, 0x28, 0x0a, 0x90, 0xd4, 0x1a, 0xc5, 0x78, 0x2e, 0x4c, 0x6c, 0xce, - 0x28, 0xdd, 0x33, 0xae, 0x3c, 0x74, 0x75, 0xab, 0x76, 0xd5, 0x5c, 0x3f, 0xe3, 0xed, 0xb1, 0x4d, - 0xf7, 0xaa, 0xde, 0xc8, 0x10, 0xde, 0xd1, 0x16, 0xbe, 0x3c, 0x64, 0x49, 0xca, 0xf6, 0xd2, 0x4b, - 0x76, 0x64, 0x4e, 0xe0, 0x21, 0x2c, 0xa0, 0x6e, 0x34, 0x30, 0x5b, 0x60, 0x49, 0xf2, 0xd2, 0xc8, - 0xab, 0xd1, 0x7f, 0xc6, 0xc6, 0xdc, 0x58, 0xc3, 0x77, 0x95, 0xaf, 0x7b, 0x71, 0xbe, 0xca, 0xb1, - 0x5a, 0x17, 0x75, 0xc2, 0x3c, 0xe5, 0x18, 0x09, 0xf2, 0x10, 0xda, 0xf1, 0x70, 0x9f, 0x8f, 0x59, - 0xf0, 0x21, 0x2c, 0x60, 0x84, 0xbc, 0x30, 0x13, 0x7d, 0x7d, 0xa6, 0x53, 0xd4, 0xf2, 0xc9, 0xc0, - 0x64, 0x36, 0x37, 0xa6, 0xbb, 0xd0, 0x46, 0xef, 0x45, 0xe8, 0xcf, 0x9a, 0x41, 0x9c, 0x1a, 0x36, - 0xd9, 0x02, 0xef, 0x05, 0x8d, 0xd4, 0xa6, 0x62, 0x04, 0xd6, 0x8a, 0xa1, 0x94, 0xed, 0xaf, 0x45, - 0x21, 0x4d, 0x9d, 0xf0, 0xad, 0xb0, 0xe7, 0x22, 0x97, 0x58, 0xa3, 0x3e, 0xc5, 0x37, 0x79, 0x09, - 0xfe, 0x33, 0x31, 0xe2, 0xc1, 0x22, 0xb8, 0xd1, 0xc0, 0xd8, 0x70, 0xa3, 0x41, 0xf0, 0x1e, 0x9a, - 0x37, 0xa5, 0xe9, 0xd7, 0x41, 0xbc, 0xa0, 0x11, 0x45, 0xc7, 0x77, 0xa0, 0x1f, 0x15, 0x9b, 0x42, - 0xe4, 0xa3, 0x24, 0x63, 0x52, 0xe4, 0xe6, 0xb6, 0x9f, 0x06, 0xc9, 0x63, 0x58, 0x52, 0xe6, 0x63, - 0xc9, 0x24, 0xb7, 0x9d, 0xbd, 0x05, 0x6d, 0x85, 0x55, 0xee, 0x0c, 0x85, 0xdb, 0xa6, 0xe4, 0x6c, - 0x6f, 0x91, 0x20, 0xdf, 0x6a, 0x0b, 0x5b, 0x87, 0x3c, 0x93, 0x8d, 0xd9, 0x40, 0x1a, 0x0d, 0xf4, - 0xa9, 0x26, 0x02, 0xa2, 0x53, 0x31, 0x31, 0x2f, 0xd6, 0x31, 0x2b, 0x94, 0x22, 0x8f, 0xfc, 0xe2, - 0x00, 0xd8, 0x80, 0xca, 0xa2, 0x52, 0x71, 0xce, 0x57, 0x09, 0xd6, 0x6c, 0x8f, 0xcd, 0x5e, 0x2c, - 0xd5, 0x52, 0x1a, 0xa7, 0x76, 0x06, 0x3e, 0xae, 0x67, 0x40, 0x37, 0xef, 0xe6, 0xcc, 0x0c, 0x68, - 0xaf, 0xf5, 0x24, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0x3b, 0x0f, 0x1f, 0x55, 0xf3, 0xe0, 0xce, 0x9a, - 0x44, 0xdc, 0x98, 0xb4, 0x53, 0xf1, 0x14, 0x7a, 0x0d, 0x78, 0xae, 0xc5, 0x35, 0xb8, 0x7e, 0x7a, - 0xe3, 0xec, 0x25, 0x9f, 0x85, 0x49, 0x02, 0xfd, 0xcd, 0xb4, 0x2c, 0x24, 0xcf, 0x8d, 0x39, 0x75, - 0xfe, 0x35, 0x50, 0x35, 0xaf, 0x06, 0xe6, 0xf7, 0x2f, 0xb8, 0x03, 0x2d, 0x55, 0x46, 0xbd, 0x38, - 0x67, 0x6b, 0xac, 0x99, 0x64, 0x17, 0x3a, 0x1b, 0x71, 0xf4, 0x24, 0x17, 0xe5, 0x64, 0x6e, 0xd0, - 0xf6, 0xab, 0xec, 0x9e, 0xfd, 0x2a, 0x7b, 0x67, 0xbe, 0xca, 0x7e, 0xf5, 0x55, 0x26, 0x31, 0x2c, - 0xeb, 0xa3, 0xa8, 0xf6, 0xf5, 0x2a, 0xa7, 0xc5, 0x7e, 0x32, 0xbd, 0xc6, 0x27, 0x33, 0x86, 0x65, - 0x7d, 0xb9, 0xfe, 0x4f, 0xa3, 0xbf, 0xb9, 0xb0, 0x4c, 0x79, 0x91, 0xbc, 0xe6, 0x51, 0x56, 0xc8, - 0xbc, 0x1c, 0xaa, 0xeb, 0xa3, 0xf4, 0xbf, 0x11, 0x7b, 0xa6, 0xda, 0x1e, 0xd5, 0xc4, 0x65, 0x26, - 0x3d, 0xb8, 0x0f, 0xbd, 0xd9, 0xed, 0x3c, 0x2b, 0xda, 0x14, 0x09, 0xee, 0xc3, 0x42, 0x2c, 0xca, - 0x7c, 0x58, 0x8d, 0x6f, 0xe3, 0x22, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xc6, 0x6a, 0xb4, 0x2e, - 0x58, 0x8d, 0x47, 0x33, 0xa3, 0x14, 0xb6, 0x51, 0xe1, 0xad, 0x5a, 0xe1, 0x14, 0x9b, 0x9e, 0x96, - 0x26, 0x3f, 0x3b, 0x70, 0xad, 0x19, 0xc2, 0xa5, 0x16, 0xb7, 0xea, 0x88, 0x3b, 0xb7, 0x23, 0xde, - 0xbc, 0x8e, 0xf8, 0x75, 0x47, 0xea, 0xaf, 0x7f, 0xab, 0xf1, 0xf5, 0x27, 0x07, 0x70, 0xfb, 0x4c, - 0x9b, 0x36, 0xc5, 0x78, 0xa2, 0xe6, 0xe1, 0x3f, 0xb4, 0x4b, 0x9d, 0xb4, 0x3c, 0x37, 0x8d, 0xea, - 0x52, 0x4d, 0x90, 0x4f, 0xe1, 0x66, 0xcc, 0x65, 0xa3, 0x49, 0x76, 0xda, 0x56, 0xc1, 0x7b, 0xc6, - 0x8f, 0xce, 0x49, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xbe, 0x98, 0x8c, 0x98, 0xe4, 0x57, 0xd2, 0xde, - 0x80, 0xce, 0x8e, 0x98, 0x88, 0x54, 0xbc, 0x9a, 0x5e, 0xb0, 0xf5, 0x21, 0x2c, 0xe8, 0xfb, 0xad, - 0xcf, 0x48, 0x97, 0x5a, 0x92, 0xdc, 0x50, 0x03, 0x3d, 0x64, 0xe9, 0xb0, 0x4c, 0x55, 0x18, 0xea, - 0x97, 0x61, 0xb1, 0xb1, 0xf4, 0xc7, 0xc9, 0x8a, 0xf3, 0xe7, 0xc9, 0x8a, 0xf3, 0xd7, 0xc9, 0x8a, - 0xf3, 0xeb, 0xdf, 0x2b, 0x6f, 0xec, 0xb5, 0xf1, 0x9f, 0xc3, 0xc3, 0x7f, 0x02, 0x00, 0x00, 0xff, - 0xff, 0xba, 0x1b, 0x62, 0x68, 0x4a, 0x0c, 0x00, 0x00, + // 1121 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5, + 0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51, + 0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c, + 0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7, + 0x20, 0xf1, 0x02, 0x88, 0x27, 0xe1, 0x11, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0x67, 0x66, + 0x76, 0x37, 0x8e, 0x83, 0xa3, 0xc0, 0xdd, 0x9c, 0xdf, 0x99, 0xf9, 0x9d, 0xef, 0xb3, 0x36, 0x74, + 0xc7, 0x59, 0x7c, 0xc2, 0x24, 0xdf, 0x18, 0x67, 0x42, 0x8a, 0xa0, 0x15, 0xa7, 0x92, 0x67, 0x29, + 0x4b, 0xc8, 0x73, 0x68, 0x47, 0xe9, 0x90, 0x9f, 0xed, 0x70, 0xc9, 0x82, 0x00, 0xfc, 0x17, 0x7c, + 0x92, 0x87, 0xde, 0x9a, 0xb3, 0xde, 0xa2, 0x78, 0x0e, 0xde, 0x83, 0xc5, 0xbd, 0x8c, 0x0d, 0x8e, + 0xb7, 0xcf, 0xe2, 0x5c, 0xf2, 0x74, 0xc0, 0x43, 0x1f, 0xb5, 0x53, 0x28, 0xf9, 0xcd, 0x81, 0x5b, + 0x5f, 0xc4, 0x3c, 0x19, 0x7e, 0x33, 0x96, 0xb1, 0x48, 0xf3, 0xe0, 0x6d, 0x68, 0x6f, 0xb1, 0xc1, + 0x11, 0xdf, 0x9b, 0x8c, 0x39, 0x32, 0xb6, 0x69, 0x05, 0x94, 0xda, 0x7e, 0xfc, 0x5a, 0x33, 0x76, + 0x69, 0x05, 0x04, 0x6b, 0xd0, 0xd9, 0x8b, 0x47, 0xfc, 0xdb, 0x82, 0xa5, 0xb2, 0x18, 0x85, 0x0d, + 0x7c, 0x5d, 0x87, 0x94, 0xab, 0x48, 0xdc, 0x42, 0x15, 0x9e, 0x83, 0x25, 0xf0, 0x76, 0xe2, 0x34, + 0x6c, 0xaf, 0x39, 0xeb, 0x1e, 0x55, 0x47, 0x44, 0xd8, 0x59, 0x08, 0x06, 0x61, 0x67, 0x65, 0x88, + 0x9d, 0x2a, 0x44, 0x42, 0x60, 0x31, 0x1a, 0x8d, 0x45, 0x26, 0x29, 0xcf, 0xc7, 0x22, 0xcd, 0x91, + 0x69, 0x3b, 0xcb, 0x42, 0x07, 0xc9, 0xd5, 0x91, 0xfc, 0x00, 0x4b, 0x9b, 0x89, 0x18, 0x1c, 0xf7, + 0x98, 0x64, 0x94, 0x7f, 0x5f, 0xf0, 0x5c, 0x06, 0xff, 0x87, 0x06, 0xe6, 0xce, 0xdc, 0xd3, 0x82, + 0x42, 0x31, 0x0f, 0xa1, 0xab, 0x51, 0x14, 0x14, 0x8a, 0xef, 0x31, 0x13, 0x3e, 0xd5, 0x82, 0x42, + 0xfb, 0x47, 0x2c, 0x1b, 0x62, 0x06, 0x7c, 0xaa, 0x05, 0xe5, 0xe3, 0x41, 0xcc, 0x4f, 0x4d, 0xd8, + 0x78, 0x26, 0x11, 0x2c, 0xd7, 0xec, 0x1b, 0x37, 0x57, 0xa0, 0x49, 0xc5, 0x69, 0xd4, 0xcb, 0x43, + 0x67, 0xcd, 0x5b, 0xf7, 0xa9, 0x91, 0x30, 0xb9, 0x22, 0x29, 0x46, 0xa9, 0x52, 0xb9, 0xa8, 0xaa, + 0x00, 0x72, 0x0f, 0x1a, 0x98, 0x69, 0x15, 0x65, 0xf5, 0x56, 0x1d, 0xc9, 0x8f, 0x0e, 0xb4, 0x77, + 0xd8, 0x19, 0xba, 0x91, 0x07, 0x4f, 0xa1, 0xd5, 0x97, 0x2c, 0x1d, 0x2a, 0x07, 0xd5, 0xa5, 0xce, + 0xe3, 0x77, 0x37, 0x6c, 0xe3, 0x6c, 0x94, 0xd7, 0x36, 0xec, 0x9d, 0xed, 0x54, 0x66, 0x13, 0x5a, + 0x3e, 0x79, 0xf3, 0x53, 0xe8, 0x5e, 0x50, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, + 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, 0x73, 0xe5, 0x53, 0x2d, 0x7c, 0xe2, 0x7e, 0xe4, 0x90, 0x03, + 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x75, 0xc6, 0x75, + 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, + 0x37, 0x5d, 0xff, 0x0f, 0xbc, 0xa4, 0x6f, 0x7d, 0x98, 0x7f, 0x37, 0x78, 0x00, 0xbe, 0x1a, 0x21, + 0x74, 0xa1, 0xf3, 0xf8, 0x4e, 0x95, 0xa7, 0x72, 0xba, 0x28, 0x5e, 0x20, 0x89, 0x25, 0x45, 0x7f, + 0xe6, 0x06, 0x36, 0xa3, 0x95, 0x1e, 0x1a, 0x53, 0x1e, 0x9a, 0x5a, 0xa9, 0x4c, 0xd5, 0xc7, 0xcf, + 0x58, 0x7b, 0x66, 0xc3, 0xbd, 0xa9, 0x35, 0x32, 0x80, 0xb7, 0x34, 0xc3, 0xe7, 0x27, 0x2c, 0x4e, + 0xd8, 0x61, 0x72, 0xcd, 0x8a, 0xcc, 0x70, 0x3c, 0x84, 0x05, 0x7c, 0x1b, 0xf5, 0xcc, 0x14, 0x58, + 0x91, 0x7c, 0x67, 0xee, 0xab, 0xd6, 0xdf, 0x65, 0x23, 0x6e, 0xd8, 0xf0, 0x5c, 0xc6, 0xeb, 0xce, + 0x8f, 0x57, 0x19, 0x56, 0xe3, 0xa2, 0x56, 0x98, 0xa7, 0x0c, 0xa3, 0x40, 0x9e, 0x40, 0xb3, 0x3f, + 0x38, 0xe2, 0x23, 0x16, 0xbc, 0x0f, 0x0b, 0xe8, 0x21, 0xcf, 0x4d, 0x47, 0xdf, 0x9e, 0xaa, 0x14, + 0xb5, 0x7a, 0xd2, 0x33, 0x91, 0xcd, 0xf4, 0xe9, 0x01, 0x34, 0xd1, 0x7a, 0x1e, 0xfa, 0xd3, 0x34, + 0x88, 0x53, 0xa3, 0x26, 0xdb, 0xe0, 0xed, 0xd3, 0x48, 0x4d, 0x2a, 0x7a, 0x60, 0x59, 0x8c, 0xa4, + 0xb8, 0xbf, 0x14, 0xb9, 0x34, 0x79, 0xc2, 0xb3, 0xc2, 0x5e, 0x8a, 0x4c, 0x62, 0x8e, 0xba, 0x14, + 0xcf, 0x24, 0x07, 0x7f, 0x57, 0x0c, 0x79, 0xb0, 0x08, 0x6e, 0xd4, 0x33, 0x1c, 0x6e, 0xd4, 0x0b, + 0xde, 0x41, 0x7a, 0x93, 0x9a, 0x6e, 0xe5, 0xc4, 0x3e, 0x8d, 0x28, 0x1a, 0xbe, 0x0f, 0xdd, 0x28, + 0xdf, 0x12, 0x22, 0x1b, 0xc6, 0x29, 0x93, 0x22, 0x33, 0xbb, 0xfd, 0x22, 0x88, 0x13, 0x24, 0x99, + 0xd4, 0x9b, 0xb8, 0x4d, 0xb5, 0x40, 0x9e, 0xc1, 0x92, 0x32, 0x8a, 0x82, 0xad, 0xf7, 0x0a, 0x34, + 0x15, 0x56, 0x3a, 0x61, 0xa4, 0x8a, 0xc1, 0xad, 0x33, 0x7c, 0xad, 0x19, 0xb6, 0x4f, 0x78, 0x2a, + 0x6b, 0x1d, 0x83, 0x32, 0x12, 0x74, 0xa9, 0x16, 0x02, 0xa2, 0x03, 0x34, 0x91, 0x2c, 0x56, 0x91, + 0x28, 0x94, 0xa2, 0x8e, 0xfc, 0xec, 0x00, 0x58, 0x87, 0x8a, 0xbc, 0x7c, 0xe2, 0x5c, 0xfd, 0x24, + 0x58, 0xb7, 0x95, 0x37, 0xd3, 0xb2, 0x54, 0xdd, 0xd2, 0x38, 0xb5, 0x9d, 0xf1, 0x61, 0xd5, 0x19, + 0xba, 0xa4, 0x77, 0xa7, 0x3a, 0x43, 0x5b, 0xad, 0xfa, 0xe3, 0x25, 0x74, 0x6a, 0xf8, 0xcc, 0x2e, + 0xf9, 0xa0, 0xec, 0x12, 0x77, 0x9a, 0x12, 0x71, 0x43, 0x69, 0x7b, 0xe5, 0x05, 0x74, 0x6a, 0xf0, + 0x4c, 0xc6, 0x75, 0xb8, 0x7d, 0x71, 0x0e, 0xed, 0x7e, 0x9f, 0x86, 0x49, 0x0c, 0xdd, 0xad, 0xa4, + 0xc8, 0x25, 0xcf, 0x0c, 0x9d, 0xfa, 0x28, 0x68, 0xa0, 0x2c, 0x5e, 0x05, 0xcc, 0xae, 0x5f, 0x70, + 0x1f, 0x1a, 0x2a, 0x8d, 0x7a, 0x9c, 0x2e, 0xe7, 0x58, 0x2b, 0xc9, 0x01, 0xb4, 0x36, 0xfb, 0xd1, + 0xf3, 0x4c, 0x14, 0xe3, 0x99, 0x4e, 0xdb, 0x6f, 0xb5, 0x7b, 0xf9, 0x5b, 0xed, 0x5d, 0xfa, 0x56, + 0xfb, 0xe5, 0xb7, 0x9a, 0xf4, 0x61, 0x59, 0xaf, 0x4a, 0x35, 0xc5, 0x37, 0x59, 0x38, 0xf6, 0x43, + 0xea, 0xd5, 0x3e, 0xa4, 0x7d, 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x24, 0xfd, 0xd5, 0x85, 0x65, 0xca, + 0xf3, 0xf8, 0x35, 0x8f, 0xd2, 0x5c, 0x66, 0xc5, 0x40, 0xed, 0x24, 0xf5, 0xfe, 0x2b, 0x71, 0x68, + 0xb2, 0xed, 0x51, 0x2d, 0x5c, 0xa7, 0xd3, 0x83, 0x47, 0xd0, 0x99, 0x9e, 0xd9, 0xcb, 0x57, 0xeb, + 0x57, 0x82, 0x47, 0xb0, 0xd0, 0x17, 0x45, 0x36, 0x28, 0xdb, 0xb7, 0xb6, 0x27, 0xb5, 0x67, 0x5a, + 0x4d, 0xed, 0xb5, 0xda, 0x68, 0x34, 0xe6, 0x8c, 0xc6, 0xd3, 0xa9, 0x56, 0x0a, 0x9b, 0xf8, 0xe0, + 0x8d, 0xea, 0xc1, 0x05, 0x35, 0xbd, 0x78, 0x9b, 0xfc, 0xe4, 0xc0, 0xad, 0xba, 0x0b, 0xd7, 0x1a, + 0xdc, 0xb2, 0x22, 0xee, 0xcc, 0x8a, 0x78, 0xb3, 0x2a, 0xe2, 0x57, 0x15, 0xa9, 0x7e, 0x13, 0x34, + 0x6a, 0xbf, 0x09, 0xc8, 0x31, 0xdc, 0xbb, 0x54, 0xa6, 0x2d, 0x31, 0x1a, 0xab, 0x7e, 0xf8, 0x17, + 0xe5, 0x52, 0x2b, 0x2d, 0xcb, 0x4c, 0xa1, 0xda, 0x54, 0x0b, 0xe4, 0x63, 0xb8, 0xdb, 0xe7, 0xb2, + 0x56, 0x24, 0xdb, 0x6d, 0x6b, 0xe0, 0xed, 0xf2, 0xd3, 0x2b, 0xc2, 0x57, 0x2a, 0xf2, 0x19, 0x84, + 0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xaf, 0x37, 0xa1, 0xb5, 0x27, 0xc6, 0x22, 0x11, 0xaf, 0x26, + 0x73, 0xa6, 0x3e, 0x84, 0x05, 0xbd, 0xbf, 0xf5, 0x1a, 0x69, 0x53, 0x2b, 0x92, 0x3b, 0xaa, 0xa1, + 0x07, 0x2c, 0x19, 0x14, 0x89, 0x72, 0x43, 0xfd, 0x5e, 0xcc, 0x37, 0x97, 0x7e, 0x3f, 0x5f, 0x75, + 0xfe, 0x38, 0x5f, 0x75, 0xfe, 0x3c, 0x5f, 0x75, 0x7e, 0xf9, 0x6b, 0xf5, 0x7f, 0x87, 0x4d, 0xfc, + 0x3f, 0xf1, 0xe4, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x93, 0xc0, 0xc1, 0x60, 0x0c, 0x00, + 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 57b98f62c..2e484b037 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -99,6 +99,7 @@ message Node { string ID = 1; URI URI = 2; bool IsCoordinator = 3; + string State = 4; } message NodeStateMessage { diff --git a/server.go b/server.go index a4cb8fc8b..5e6f61087 100644 --- a/server.go +++ b/server.go @@ -298,6 +298,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { ID: s.nodeID, URI: s.uri, IsCoordinator: s.cluster.Coordinator == s.nodeID, + State: nodeStateDown, } s.cluster.Node = node if s.clusterDisabled { @@ -561,7 +562,10 @@ func (s *Server) receiveMessage(m Message) error { case *RecalculateCaches: s.holder.recalculateCaches() case *NodeEvent: - s.cluster.ReceiveEvent(obj) + err := s.cluster.ReceiveEvent(obj) + if err != nil { + return errors.Wrapf(err, "cluster receiving NodeEvent %v", obj) + } case *NodeStatus: s.handleRemoteStatus(obj) } diff --git a/server/server.go b/server/server.go index 010eb2f8a..6cdc43883 100644 --- a/server/server.go +++ b/server/server.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - +// // Package server contains the `pilosa server` subcommand which runs Pilosa // itself. The purpose of this package is to define an easily tested Command // object which handles interpreting configuration and setting up all the @@ -20,6 +20,7 @@ package server import ( + "bytes" "crypto/tls" "io" "log" @@ -336,7 +337,7 @@ func (m *Command) setupNetworking() error { gossipMemberSet, err := gossip.NewMemberSet( m.Config.Gossip, m.API, - gossip.WithLogger(m.logger.Logger()), + gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}), gossip.WithTransport(m.gossipTransport), ) if err != nil { @@ -407,3 +408,24 @@ func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err erro return ln, nil } + +type filteredWriter struct { + v bool + logOutput io.Writer +} + +// Write forwards the write to logOutput if verbose is true, or it doesn't +// contain [DEBUG] or [INFO]. This implementation isn't technically correct +// since Write could be called with only part of a log line, but I don't think +// that actually happens, so until it becomes a problem, I don't think it's +// worth dealing with the extra complexity. (jaffee) +func (f *filteredWriter) Write(p []byte) (n int, err error) { + if bytes.Contains(p, []byte("[DEBUG]")) || bytes.Contains(p, []byte("[INFO]")) { + if f.v { + return f.logOutput.Write(p) + } + } else { + return f.logOutput.Write(p) + } + return len(p), nil +}