diff --git a/Dockerfile-withgo b/Dockerfile-withgo index 306d16f3e..834634b13 100644 --- a/Dockerfile-withgo +++ b/Dockerfile-withgo @@ -23,4 +23,4 @@ EXPOSE 10101 VOLUME /data ENTRYPOINT ["bash", "-c"] -CMD ["pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] +CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/Makefile b/Makefile index b403fde23..9cf4cffaf 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,22 @@ release: check-clean $(MAKE) release-build GOOS=linux GOARCH=386 $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1 +# Run cluster integration tests using docker. Requires docker daemon to be +# running. This will catch changes to internal/clustertests/*.go, but if you +# make changes to Pilosa, you'll want to run clustertests-build to rebuild the +# pilosa image. +clustertests: + cd internal/clustertests;\ + docker-compose down;\ + docker-compose up; + + +# Like clustertests, but rebuilds all images. +clustertests-build: + cd internal/clustertests;\ + docker-compose down;\ + docker-compose up --build; + # Create prerelease builds prerelease: vendor $(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\) diff --git a/cluster.go b/cluster.go index c3bc67b4d..4b1ab9089 100644 --- a/cluster.go +++ b/cluster.go @@ -65,10 +65,11 @@ type Node struct { ID string `json:"id"` URI URI `json:"uri"` IsCoordinator bool `json:"isCoordinator"` + State string `json:"state"` } func (n Node) String() string { - return fmt.Sprintf("Node: %s", n.ID) + return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6]) } // Nodes represents a list of nodes. @@ -456,7 +457,19 @@ func (c *cluster) unprotectedSetState(state string) { } } +func (c *cluster) setMyNodeState(state string) { + c.mu.Lock() + defer c.mu.Unlock() + c.Node.State = state + for i, n := range c.nodes { + if n.ID == c.Node.ID { + c.nodes[i].State = state + } + } +} + func (c *cluster) setNodeState(state string) error { // nolint: unparam + c.setMyNodeState(state) if c.isCoordinator() { return c.receiveNodeState(c.Node.ID, state) } @@ -486,11 +499,23 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { } c.Topology.mu.Lock() - c.Topology.nodeStates[nodeID] = state + changed := false + if c.Topology.nodeStates[nodeID] != state { + changed = true + c.Topology.nodeStates[nodeID] = state + for i, n := range c.nodes { + if n.ID == nodeID { + c.nodes[i].State = state + } + } + } c.Topology.mu.Unlock() c.logger.Printf("received state %s (%s)", state, nodeID) - return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) + if changed { + return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) + } + return nil } // determineClusterState is unprotected. @@ -932,7 +957,6 @@ func (c *cluster) waitForStarted() error { <-c.joining c.logger.Printf("joining has completed") } - return nil } @@ -1043,8 +1067,9 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error { return nil } // Broadcast cluster status changes to the cluster. - c.logger.Printf("broadcasting ClusterStatus: %s", state) - return c.broadcaster.SendSync(c.unprotectedStatus()) // TODO fix c.Status + status := c.unprotectedStatus() + c.logger.Printf("broadcasting ClusterStatus: %s", status) + return c.broadcaster.SendSync(status) // TODO fix c.Status } @@ -1625,7 +1650,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { switch e.Event { case NodeJoin: - c.logger.Printf("received NodeJoin event: %v", e) + c.logger.Printf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI) // Ignore the event if this is not the coordinator. if !c.isCoordinator() { return nil @@ -1660,6 +1685,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { func (c *cluster) nodeJoin(node *Node) error { c.mu.Lock() defer c.mu.Unlock() + c.logger.Printf("NodeJoin event on coordinator, node: %s, id: %s", node.URI, node.ID) if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsID(node.ID) { @@ -1688,11 +1714,10 @@ func (c *cluster) nodeJoin(node *Node) error { if c.haveTopologyAgreement() && c.allNodesReady() { return c.unprotectedSetStateAndBroadcast(ClusterStateNormal) - } else { - // Send the status to the remote node. This lets the remote node - // know that it can proceed with opening its Holder. - return c.sendTo(node, c.unprotectedStatus()) } + // Send the status to the remote node. This lets the remote node + // know that it can proceed with opening its Holder. + return c.sendTo(node, c.unprotectedStatus()) } // If the cluster already contains the node, just send it the cluster status. @@ -1796,6 +1821,10 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { // Add all nodes from the coordinator. for _, node := range officialNodes { + if node.ID == c.Node.ID && node.State != c.Node.State { + c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State) + go c.setNodeState(c.Node.State) + } if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index a2b529fba..50602d385 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -185,8 +185,8 @@ func TestFragSources(t *testing.T) { "node0": {}, "node1": {}, "node2": { - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -197,11 +197,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*ResizeSource{ "node0": { - {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + {&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)}, }, "node1": { - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -212,11 +212,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*ResizeSource{ "node0": { - {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + {&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, "node1": { - {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)}, }, "node2": {}, }, diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index e8de4ea1a..9826c7ef9 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -506,6 +506,7 @@ func encodeNode(n *pilosa.Node) *internal.Node { ID: n.ID, URI: encodeURI(n.URI), IsCoordinator: n.IsCoordinator, + State: n.State, } } @@ -761,6 +762,7 @@ func decodeNode(node *internal.Node, m *pilosa.Node) { m.ID = node.ID decodeURI(node.URI, &m.URI) m.IsCoordinator = node.IsCoordinator + m.State = node.State } func decodeURI(i *internal.URI, m *pilosa.URI) { diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index 23d27fcba..21c9e35de 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -12,63 +12,77 @@ import ( pi "github.com/pilosa/pilosa" ) -func TestLongPauses(t *testing.T) { - t.Skip() // TODO figure out how to only run in the docker-compose environment +func TestClusterStuff(t *testing.T) { + if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" { + t.Skip() + } cli := getPilosaClient(t) - idx := pilosa.NewIndex("testidx") - err := cli.CreateIndex(idx) - if err != nil { - t.Fatalf("creating index: %v", err) - } - f := idx.Field("testf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) - err = cli.CreateField(f) - if err != nil { - t.Fatalf("creating field: %v", err) - } + t.Run("long pause", func(t *testing.T) { + idx := pilosa.NewIndex("testidx") + err := cli.CreateIndex(idx) + if err != nil { + t.Fatalf("creating index: %v", err) + } + f := idx.Field("testf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + err = cli.CreateField(f) + if err != nil { + t.Fatalf("creating field: %v", err) + } - data := make([]pilosa.Column, 1000) - for i := range data { - data[i].RowID = 0 - data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10) - } + data := make([]pilosa.Column, 1000) + for i := range data { + data[i].RowID = 0 + data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10) + } - err = cli.ImportField(f, &colIterator{cols: data}, pilosa.OptImportBatchSize(1000)) - if err != nil { - t.Fatalf("importing: %v", err) - } + err = cli.ImportField(f, &colIterator{cols: data}, pilosa.OptImportBatchSize(1000)) + if err != nil { + t.Fatalf("importing: %v", err) + } - r, err := cli.Query(idx.Count(f.Row(0))) - if err != nil { - t.Fatalf("count querying: %v", err) - } - if r.Result().Count() != 1000 { - t.Fatalf("count after import is %d", r.Result().Count()) - } + r, err := cli.Query(idx.Count(f.Row(0))) + if err != nil { + t.Fatalf("count querying: %v", err) + } + if r.Result().Count() != 1000 { + t.Fatalf("count after import is %d", r.Result().Count()) + } - pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "30s") - pcmd.Stdout = os.Stdout - pcmd.Stderr = os.Stderr - fmt.Println("pausing pilosa3 for 30s") - err = pcmd.Start() - if err != nil { - t.Fatalf("starting pumba command: %v", err) - } - err = pcmd.Wait() - if err != nil { - t.Fatalf("waiting on pumba pause cmd: %v", err) - } - // TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa - fmt.Println("done with pause, waiting for stability") - time.Sleep(time.Second * 400) - fmt.Println("done waiting") + pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s") + pcmd.Stdout = os.Stdout + pcmd.Stderr = os.Stderr + fmt.Println("pausing pilosa3 for 10s") + err = pcmd.Start() + if err != nil { + t.Fatalf("starting pumba command: %v", err) + } + err = pcmd.Wait() + if err != nil { + t.Fatalf("waiting on pumba pause cmd: %v", err) + } + // TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa + fmt.Println("done with pause, waiting for stability") + time.Sleep(time.Second * 3) + fmt.Println("done waiting") - r, err = cli.Query(idx.Count(f.Row(0))) + r, err = cli.Query(idx.Count(f.Row(0))) + if err != nil { + t.Fatalf("count querying: %v", err) + } else if r.Result().Count() != 1000 { + t.Fatalf("count after import is %d", r.Result().Count()) + } + + fmt.Println("at the bottom") + + }) + + down := exec.Command("/pumba", "stop", "clustertests_pilosa3_1", "clustertests_pilosa2_1", "clustertests_pilosa1_1") + down.Stdout = os.Stdout + down.Stderr = os.Stderr + err := down.Run() if err != nil { - t.Fatalf("count querying: %v", err) - } - if r.Result().Count() != 1000 { - t.Fatalf("count after import is %d", r.Result().Count()) + t.Logf("stopping Pilosa: %v", err) } } diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 752c39177..7196ea0eb 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -41,13 +41,14 @@ services: - "/pilosa server --bind pilosa3:10101" client1: build: - context: ../.. - dockerfile: Dockerfile-withgo + context: . + environment: + - ENABLE_PILOSA_CLUSTER_TESTS=1 networks: - pilosanet volumes: - /var/run/docker.sock:/var/run/docker.sock command: - - "go test -v github.com/pilosa/pilosa/internal/clustertests" + - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" networks: pilosanet: diff --git a/internal/private.pb.go b/internal/private.pb.go index be4719e2f..d5e5d9251 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -540,6 +540,7 @@ type Node struct { ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` + State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` } func (m *Node) Reset() { *m = Node{} } @@ -568,6 +569,13 @@ func (m *Node) GetIsCoordinator() bool { return false } +func (m *Node) GetState() string { + if m != nil { + return m.State + } + return "" +} + type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -1749,6 +1757,12 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { } i++ } + if len(m.State) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } return i, nil } @@ -2675,6 +2689,10 @@ func (m *Node) Size() (n int) { if m.IsCoordinator { n += 2 } + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -5227,6 +5245,35 @@ func (m *Node) Unmarshal(dAtA []byte) error { } } m.IsCoordinator = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7399,75 +7446,76 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1113 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, - 0x18, 0x66, 0x0f, 0x76, 0xec, 0xdf, 0x75, 0x9a, 0x6c, 0x69, 0xd9, 0x02, 0x0a, 0x61, 0x54, 0xd1, - 0x50, 0x89, 0x50, 0xb5, 0x37, 0x9c, 0x2a, 0x95, 0xc4, 0xa1, 0x2c, 0x25, 0xa5, 0xcc, 0xa6, 0xb9, - 0xeb, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90, - 0x78, 0x01, 0xc4, 0x93, 0xf0, 0x08, 0x5c, 0xf2, 0x08, 0x28, 0xbc, 0x08, 0x9a, 0x7f, 0x66, 0x76, - 0x37, 0x8e, 0x43, 0xa2, 0xc0, 0xdd, 0xfc, 0xdf, 0x7f, 0x3e, 0xae, 0x0d, 0xfd, 0x49, 0x9e, 0x1c, - 0x32, 0xc9, 0xd7, 0x27, 0xb9, 0x90, 0x22, 0xe8, 0x24, 0x99, 0xe4, 0x79, 0xc6, 0x52, 0xf2, 0x04, - 0xba, 0x51, 0x36, 0xe2, 0xc7, 0xdb, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x29, 0x9f, 0x16, 0xa1, 0xb7, - 0xea, 0xac, 0x75, 0x28, 0xbe, 0x83, 0x0f, 0x60, 0x71, 0x27, 0x67, 0xc3, 0x83, 0xad, 0xe3, 0xa4, - 0x90, 0x3c, 0x1b, 0xf2, 0xd0, 0x47, 0xee, 0x0c, 0x4a, 0x7e, 0x77, 0xe0, 0xda, 0x57, 0x09, 0x4f, - 0x47, 0xdf, 0x4d, 0x64, 0x22, 0xb2, 0x22, 0x78, 0x17, 0xba, 0x9b, 0x6c, 0xb8, 0xcf, 0x77, 0xa6, - 0x13, 0x8e, 0x16, 0xbb, 0xb4, 0x06, 0x2a, 0x6e, 0x9c, 0xbc, 0xd6, 0x16, 0xfb, 0xb4, 0x06, 0x82, - 0x55, 0xe8, 0xed, 0x24, 0x63, 0xfe, 0x7d, 0xc9, 0x32, 0x59, 0x8e, 0xc3, 0x16, 0x6a, 0x37, 0x21, - 0x15, 0x2a, 0x1a, 0xee, 0x20, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0xdb, 0x49, 0x16, 0x76, 0x57, 0x9d, - 0x35, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x38, 0x04, 0x83, 0xb0, 0xe3, 0x2a, 0xc5, 0x5e, 0x9d, 0x22, - 0x21, 0xb0, 0x18, 0x8d, 0x27, 0x22, 0x97, 0x94, 0x17, 0x13, 0x91, 0x15, 0x68, 0x69, 0x2b, 0xcf, - 0x43, 0x07, 0x8d, 0xab, 0x27, 0xf9, 0x11, 0x96, 0x36, 0x52, 0x31, 0x3c, 0x18, 0x30, 0xc9, 0x28, - 0xff, 0xa1, 0xe4, 0x85, 0x0c, 0xde, 0x84, 0x16, 0xd6, 0xce, 0xc8, 0x69, 0x42, 0xa1, 0x58, 0x87, - 0xd0, 0xd5, 0x28, 0x12, 0x0a, 0x45, 0x7d, 0xac, 0x84, 0x4f, 0x35, 0xa1, 0xd0, 0x78, 0x9f, 0xe5, - 0x23, 0xac, 0x80, 0x4f, 0x35, 0xa1, 0x62, 0xdc, 0x4d, 0xf8, 0x91, 0x49, 0x1b, 0xdf, 0x24, 0x82, - 0xe5, 0x86, 0x7f, 0x13, 0xe6, 0x2d, 0x68, 0x53, 0x71, 0x14, 0x0d, 0x8a, 0xd0, 0x59, 0xf5, 0xd6, - 0x7c, 0x6a, 0x28, 0x2c, 0xae, 0x48, 0xcb, 0x71, 0xa6, 0x58, 0x2e, 0xb2, 0x6a, 0x80, 0xdc, 0x86, - 0x16, 0x56, 0x5a, 0x65, 0x59, 0xeb, 0xaa, 0x27, 0xf9, 0xc9, 0x81, 0xee, 0x36, 0x3b, 0xc6, 0x30, - 0x8a, 0xe0, 0x11, 0x74, 0x62, 0xc9, 0xb2, 0x91, 0x0a, 0x50, 0x09, 0xf5, 0x1e, 0xbc, 0xbf, 0x6e, - 0x07, 0x67, 0xbd, 0x12, 0x5b, 0xb7, 0x32, 0x5b, 0x99, 0xcc, 0xa7, 0xb4, 0x52, 0x79, 0xfb, 0x73, - 0xe8, 0x9f, 0x62, 0x29, 0x7f, 0x07, 0x7c, 0x6a, 0xab, 0x7a, 0xc0, 0xa7, 0x2a, 0xff, 0x43, 0x96, - 0x96, 0x1c, 0x6b, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x5d, 0x08, 0x36, 0x73, 0xce, - 0x24, 0x47, 0x27, 0xdb, 0xbc, 0x28, 0xd8, 0x2b, 0x7e, 0x7e, 0xc5, 0x75, 0x15, 0xdd, 0x66, 0x15, - 0xab, 0x3e, 0x78, 0x8d, 0x3e, 0x90, 0x7b, 0x10, 0x0c, 0x78, 0xca, 0x25, 0x37, 0x53, 0xff, 0x2f, - 0x76, 0x49, 0x6c, 0x63, 0xb8, 0x58, 0x36, 0xb8, 0x0b, 0xbe, 0x5a, 0x21, 0x0c, 0xa1, 0xf7, 0xe0, - 0x46, 0x5d, 0xa7, 0x6a, 0xbb, 0x28, 0x0a, 0x90, 0xd4, 0x1a, 0xc5, 0x78, 0x2e, 0x4c, 0x6c, 0xce, - 0x28, 0xdd, 0x33, 0xae, 0x3c, 0x74, 0x75, 0xab, 0x76, 0xd5, 0x5c, 0x3f, 0xe3, 0xed, 0xb1, 0x4d, - 0xf7, 0xaa, 0xde, 0xc8, 0x10, 0xde, 0xd1, 0x16, 0xbe, 0x3c, 0x64, 0x49, 0xca, 0xf6, 0xd2, 0x4b, - 0x76, 0x64, 0x4e, 0xe0, 0x21, 0x2c, 0xa0, 0x6e, 0x34, 0x30, 0x5b, 0x60, 0x49, 0xf2, 0xd2, 0xc8, - 0xab, 0xd1, 0x7f, 0xc6, 0xc6, 0xdc, 0x58, 0xc3, 0x77, 0x95, 0xaf, 0x7b, 0x71, 0xbe, 0xca, 0xb1, - 0x5a, 0x17, 0x75, 0xc2, 0x3c, 0xe5, 0x18, 0x09, 0xf2, 0x10, 0xda, 0xf1, 0x70, 0x9f, 0x8f, 0x59, - 0xf0, 0x21, 0x2c, 0x60, 0x84, 0xbc, 0x30, 0x13, 0x7d, 0x7d, 0xa6, 0x53, 0xd4, 0xf2, 0xc9, 0xc0, - 0x64, 0x36, 0x37, 0xa6, 0xbb, 0xd0, 0x46, 0xef, 0x45, 0xe8, 0xcf, 0x9a, 0x41, 0x9c, 0x1a, 0x36, - 0xd9, 0x02, 0xef, 0x05, 0x8d, 0xd4, 0xa6, 0x62, 0x04, 0xd6, 0x8a, 0xa1, 0x94, 0xed, 0xaf, 0x45, - 0x21, 0x4d, 0x9d, 0xf0, 0xad, 0xb0, 0xe7, 0x22, 0x97, 0x58, 0xa3, 0x3e, 0xc5, 0x37, 0x79, 0x09, - 0xfe, 0x33, 0x31, 0xe2, 0xc1, 0x22, 0xb8, 0xd1, 0xc0, 0xd8, 0x70, 0xa3, 0x41, 0xf0, 0x1e, 0x9a, - 0x37, 0xa5, 0xe9, 0xd7, 0x41, 0xbc, 0xa0, 0x11, 0x45, 0xc7, 0x77, 0xa0, 0x1f, 0x15, 0x9b, 0x42, - 0xe4, 0xa3, 0x24, 0x63, 0x52, 0xe4, 0xe6, 0xb6, 0x9f, 0x06, 0xc9, 0x63, 0x58, 0x52, 0xe6, 0x63, - 0xc9, 0x24, 0xb7, 0x9d, 0xbd, 0x05, 0x6d, 0x85, 0x55, 0xee, 0x0c, 0x85, 0xdb, 0xa6, 0xe4, 0x6c, - 0x6f, 0x91, 0x20, 0xdf, 0x6a, 0x0b, 0x5b, 0x87, 0x3c, 0x93, 0x8d, 0xd9, 0x40, 0x1a, 0x0d, 0xf4, - 0xa9, 0x26, 0x02, 0xa2, 0x53, 0x31, 0x31, 0x2f, 0xd6, 0x31, 0x2b, 0x94, 0x22, 0x8f, 0xfc, 0xe2, - 0x00, 0xd8, 0x80, 0xca, 0xa2, 0x52, 0x71, 0xce, 0x57, 0x09, 0xd6, 0x6c, 0x8f, 0xcd, 0x5e, 0x2c, - 0xd5, 0x52, 0x1a, 0xa7, 0x76, 0x06, 0x3e, 0xae, 0x67, 0x40, 0x37, 0xef, 0xe6, 0xcc, 0x0c, 0x68, - 0xaf, 0xf5, 0x24, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0x3b, 0x0f, 0x1f, 0x55, 0xf3, 0xe0, 0xce, 0x9a, - 0x44, 0xdc, 0x98, 0xb4, 0x53, 0xf1, 0x14, 0x7a, 0x0d, 0x78, 0xae, 0xc5, 0x35, 0xb8, 0x7e, 0x7a, - 0xe3, 0xec, 0x25, 0x9f, 0x85, 0x49, 0x02, 0xfd, 0xcd, 0xb4, 0x2c, 0x24, 0xcf, 0x8d, 0x39, 0x75, - 0xfe, 0x35, 0x50, 0x35, 0xaf, 0x06, 0xe6, 0xf7, 0x2f, 0xb8, 0x03, 0x2d, 0x55, 0x46, 0xbd, 0x38, - 0x67, 0x6b, 0xac, 0x99, 0x64, 0x17, 0x3a, 0x1b, 0x71, 0xf4, 0x24, 0x17, 0xe5, 0x64, 0x6e, 0xd0, - 0xf6, 0xab, 0xec, 0x9e, 0xfd, 0x2a, 0x7b, 0x67, 0xbe, 0xca, 0x7e, 0xf5, 0x55, 0x26, 0x31, 0x2c, - 0xeb, 0xa3, 0xa8, 0xf6, 0xf5, 0x2a, 0xa7, 0xc5, 0x7e, 0x32, 0xbd, 0xc6, 0x27, 0x33, 0x86, 0x65, - 0x7d, 0xb9, 0xfe, 0x4f, 0xa3, 0xbf, 0xb9, 0xb0, 0x4c, 0x79, 0x91, 0xbc, 0xe6, 0x51, 0x56, 0xc8, - 0xbc, 0x1c, 0xaa, 0xeb, 0xa3, 0xf4, 0xbf, 0x11, 0x7b, 0xa6, 0xda, 0x1e, 0xd5, 0xc4, 0x65, 0x26, - 0x3d, 0xb8, 0x0f, 0xbd, 0xd9, 0xed, 0x3c, 0x2b, 0xda, 0x14, 0x09, 0xee, 0xc3, 0x42, 0x2c, 0xca, - 0x7c, 0x58, 0x8d, 0x6f, 0xe3, 0x22, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xc6, 0x6a, 0xb4, 0x2e, - 0x58, 0x8d, 0x47, 0x33, 0xa3, 0x14, 0xb6, 0x51, 0xe1, 0xad, 0x5a, 0xe1, 0x14, 0x9b, 0x9e, 0x96, - 0x26, 0x3f, 0x3b, 0x70, 0xad, 0x19, 0xc2, 0xa5, 0x16, 0xb7, 0xea, 0x88, 0x3b, 0xb7, 0x23, 0xde, - 0xbc, 0x8e, 0xf8, 0x75, 0x47, 0xea, 0xaf, 0x7f, 0xab, 0xf1, 0xf5, 0x27, 0x07, 0x70, 0xfb, 0x4c, - 0x9b, 0x36, 0xc5, 0x78, 0xa2, 0xe6, 0xe1, 0x3f, 0xb4, 0x4b, 0x9d, 0xb4, 0x3c, 0x37, 0x8d, 0xea, - 0x52, 0x4d, 0x90, 0x4f, 0xe1, 0x66, 0xcc, 0x65, 0xa3, 0x49, 0x76, 0xda, 0x56, 0xc1, 0x7b, 0xc6, - 0x8f, 0xce, 0x49, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xbe, 0x98, 0x8c, 0x98, 0xe4, 0x57, 0xd2, 0xde, - 0x80, 0xce, 0x8e, 0x98, 0x88, 0x54, 0xbc, 0x9a, 0x5e, 0xb0, 0xf5, 0x21, 0x2c, 0xe8, 0xfb, 0xad, - 0xcf, 0x48, 0x97, 0x5a, 0x92, 0xdc, 0x50, 0x03, 0x3d, 0x64, 0xe9, 0xb0, 0x4c, 0x55, 0x18, 0xea, - 0x97, 0x61, 0xb1, 0xb1, 0xf4, 0xc7, 0xc9, 0x8a, 0xf3, 0xe7, 0xc9, 0x8a, 0xf3, 0xd7, 0xc9, 0x8a, - 0xf3, 0xeb, 0xdf, 0x2b, 0x6f, 0xec, 0xb5, 0xf1, 0x9f, 0xc3, 0xc3, 0x7f, 0x02, 0x00, 0x00, 0xff, - 0xff, 0xba, 0x1b, 0x62, 0x68, 0x4a, 0x0c, 0x00, 0x00, + // 1121 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5, + 0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51, + 0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c, + 0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7, + 0x20, 0xf1, 0x02, 0x88, 0x27, 0xe1, 0x11, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0x67, 0x66, + 0x76, 0x37, 0x8e, 0x83, 0xa3, 0xc0, 0xdd, 0x9c, 0xdf, 0x99, 0xf9, 0x9d, 0xef, 0xb3, 0x36, 0x74, + 0xc7, 0x59, 0x7c, 0xc2, 0x24, 0xdf, 0x18, 0x67, 0x42, 0x8a, 0xa0, 0x15, 0xa7, 0x92, 0x67, 0x29, + 0x4b, 0xc8, 0x73, 0x68, 0x47, 0xe9, 0x90, 0x9f, 0xed, 0x70, 0xc9, 0x82, 0x00, 0xfc, 0x17, 0x7c, + 0x92, 0x87, 0xde, 0x9a, 0xb3, 0xde, 0xa2, 0x78, 0x0e, 0xde, 0x83, 0xc5, 0xbd, 0x8c, 0x0d, 0x8e, + 0xb7, 0xcf, 0xe2, 0x5c, 0xf2, 0x74, 0xc0, 0x43, 0x1f, 0xb5, 0x53, 0x28, 0xf9, 0xcd, 0x81, 0x5b, + 0x5f, 0xc4, 0x3c, 0x19, 0x7e, 0x33, 0x96, 0xb1, 0x48, 0xf3, 0xe0, 0x6d, 0x68, 0x6f, 0xb1, 0xc1, + 0x11, 0xdf, 0x9b, 0x8c, 0x39, 0x32, 0xb6, 0x69, 0x05, 0x94, 0xda, 0x7e, 0xfc, 0x5a, 0x33, 0x76, + 0x69, 0x05, 0x04, 0x6b, 0xd0, 0xd9, 0x8b, 0x47, 0xfc, 0xdb, 0x82, 0xa5, 0xb2, 0x18, 0x85, 0x0d, + 0x7c, 0x5d, 0x87, 0x94, 0xab, 0x48, 0xdc, 0x42, 0x15, 0x9e, 0x83, 0x25, 0xf0, 0x76, 0xe2, 0x34, + 0x6c, 0xaf, 0x39, 0xeb, 0x1e, 0x55, 0x47, 0x44, 0xd8, 0x59, 0x08, 0x06, 0x61, 0x67, 0x65, 0x88, + 0x9d, 0x2a, 0x44, 0x42, 0x60, 0x31, 0x1a, 0x8d, 0x45, 0x26, 0x29, 0xcf, 0xc7, 0x22, 0xcd, 0x91, + 0x69, 0x3b, 0xcb, 0x42, 0x07, 0xc9, 0xd5, 0x91, 0xfc, 0x00, 0x4b, 0x9b, 0x89, 0x18, 0x1c, 0xf7, + 0x98, 0x64, 0x94, 0x7f, 0x5f, 0xf0, 0x5c, 0x06, 0xff, 0x87, 0x06, 0xe6, 0xce, 0xdc, 0xd3, 0x82, + 0x42, 0x31, 0x0f, 0xa1, 0xab, 0x51, 0x14, 0x14, 0x8a, 0xef, 0x31, 0x13, 0x3e, 0xd5, 0x82, 0x42, + 0xfb, 0x47, 0x2c, 0x1b, 0x62, 0x06, 0x7c, 0xaa, 0x05, 0xe5, 0xe3, 0x41, 0xcc, 0x4f, 0x4d, 0xd8, + 0x78, 0x26, 0x11, 0x2c, 0xd7, 0xec, 0x1b, 0x37, 0x57, 0xa0, 0x49, 0xc5, 0x69, 0xd4, 0xcb, 0x43, + 0x67, 0xcd, 0x5b, 0xf7, 0xa9, 0x91, 0x30, 0xb9, 0x22, 0x29, 0x46, 0xa9, 0x52, 0xb9, 0xa8, 0xaa, + 0x00, 0x72, 0x0f, 0x1a, 0x98, 0x69, 0x15, 0x65, 0xf5, 0x56, 0x1d, 0xc9, 0x8f, 0x0e, 0xb4, 0x77, + 0xd8, 0x19, 0xba, 0x91, 0x07, 0x4f, 0xa1, 0xd5, 0x97, 0x2c, 0x1d, 0x2a, 0x07, 0xd5, 0xa5, 0xce, + 0xe3, 0x77, 0x37, 0x6c, 0xe3, 0x6c, 0x94, 0xd7, 0x36, 0xec, 0x9d, 0xed, 0x54, 0x66, 0x13, 0x5a, + 0x3e, 0x79, 0xf3, 0x53, 0xe8, 0x5e, 0x50, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, + 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, 0x73, 0xe5, 0x53, 0x2d, 0x7c, 0xe2, 0x7e, 0xe4, 0x90, 0x03, + 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x75, 0xc6, 0x75, + 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, + 0x37, 0x5d, 0xff, 0x0f, 0xbc, 0xa4, 0x6f, 0x7d, 0x98, 0x7f, 0x37, 0x78, 0x00, 0xbe, 0x1a, 0x21, + 0x74, 0xa1, 0xf3, 0xf8, 0x4e, 0x95, 0xa7, 0x72, 0xba, 0x28, 0x5e, 0x20, 0x89, 0x25, 0x45, 0x7f, + 0xe6, 0x06, 0x36, 0xa3, 0x95, 0x1e, 0x1a, 0x53, 0x1e, 0x9a, 0x5a, 0xa9, 0x4c, 0xd5, 0xc7, 0xcf, + 0x58, 0x7b, 0x66, 0xc3, 0xbd, 0xa9, 0x35, 0x32, 0x80, 0xb7, 0x34, 0xc3, 0xe7, 0x27, 0x2c, 0x4e, + 0xd8, 0x61, 0x72, 0xcd, 0x8a, 0xcc, 0x70, 0x3c, 0x84, 0x05, 0x7c, 0x1b, 0xf5, 0xcc, 0x14, 0x58, + 0x91, 0x7c, 0x67, 0xee, 0xab, 0xd6, 0xdf, 0x65, 0x23, 0x6e, 0xd8, 0xf0, 0x5c, 0xc6, 0xeb, 0xce, + 0x8f, 0x57, 0x19, 0x56, 0xe3, 0xa2, 0x56, 0x98, 0xa7, 0x0c, 0xa3, 0x40, 0x9e, 0x40, 0xb3, 0x3f, + 0x38, 0xe2, 0x23, 0x16, 0xbc, 0x0f, 0x0b, 0xe8, 0x21, 0xcf, 0x4d, 0x47, 0xdf, 0x9e, 0xaa, 0x14, + 0xb5, 0x7a, 0xd2, 0x33, 0x91, 0xcd, 0xf4, 0xe9, 0x01, 0x34, 0xd1, 0x7a, 0x1e, 0xfa, 0xd3, 0x34, + 0x88, 0x53, 0xa3, 0x26, 0xdb, 0xe0, 0xed, 0xd3, 0x48, 0x4d, 0x2a, 0x7a, 0x60, 0x59, 0x8c, 0xa4, + 0xb8, 0xbf, 0x14, 0xb9, 0x34, 0x79, 0xc2, 0xb3, 0xc2, 0x5e, 0x8a, 0x4c, 0x62, 0x8e, 0xba, 0x14, + 0xcf, 0x24, 0x07, 0x7f, 0x57, 0x0c, 0x79, 0xb0, 0x08, 0x6e, 0xd4, 0x33, 0x1c, 0x6e, 0xd4, 0x0b, + 0xde, 0x41, 0x7a, 0x93, 0x9a, 0x6e, 0xe5, 0xc4, 0x3e, 0x8d, 0x28, 0x1a, 0xbe, 0x0f, 0xdd, 0x28, + 0xdf, 0x12, 0x22, 0x1b, 0xc6, 0x29, 0x93, 0x22, 0x33, 0xbb, 0xfd, 0x22, 0x88, 0x13, 0x24, 0x99, + 0xd4, 0x9b, 0xb8, 0x4d, 0xb5, 0x40, 0x9e, 0xc1, 0x92, 0x32, 0x8a, 0x82, 0xad, 0xf7, 0x0a, 0x34, + 0x15, 0x56, 0x3a, 0x61, 0xa4, 0x8a, 0xc1, 0xad, 0x33, 0x7c, 0xad, 0x19, 0xb6, 0x4f, 0x78, 0x2a, + 0x6b, 0x1d, 0x83, 0x32, 0x12, 0x74, 0xa9, 0x16, 0x02, 0xa2, 0x03, 0x34, 0x91, 0x2c, 0x56, 0x91, + 0x28, 0x94, 0xa2, 0x8e, 0xfc, 0xec, 0x00, 0x58, 0x87, 0x8a, 0xbc, 0x7c, 0xe2, 0x5c, 0xfd, 0x24, + 0x58, 0xb7, 0x95, 0x37, 0xd3, 0xb2, 0x54, 0xdd, 0xd2, 0x38, 0xb5, 0x9d, 0xf1, 0x61, 0xd5, 0x19, + 0xba, 0xa4, 0x77, 0xa7, 0x3a, 0x43, 0x5b, 0xad, 0xfa, 0xe3, 0x25, 0x74, 0x6a, 0xf8, 0xcc, 0x2e, + 0xf9, 0xa0, 0xec, 0x12, 0x77, 0x9a, 0x12, 0x71, 0x43, 0x69, 0x7b, 0xe5, 0x05, 0x74, 0x6a, 0xf0, + 0x4c, 0xc6, 0x75, 0xb8, 0x7d, 0x71, 0x0e, 0xed, 0x7e, 0x9f, 0x86, 0x49, 0x0c, 0xdd, 0xad, 0xa4, + 0xc8, 0x25, 0xcf, 0x0c, 0x9d, 0xfa, 0x28, 0x68, 0xa0, 0x2c, 0x5e, 0x05, 0xcc, 0xae, 0x5f, 0x70, + 0x1f, 0x1a, 0x2a, 0x8d, 0x7a, 0x9c, 0x2e, 0xe7, 0x58, 0x2b, 0xc9, 0x01, 0xb4, 0x36, 0xfb, 0xd1, + 0xf3, 0x4c, 0x14, 0xe3, 0x99, 0x4e, 0xdb, 0x6f, 0xb5, 0x7b, 0xf9, 0x5b, 0xed, 0x5d, 0xfa, 0x56, + 0xfb, 0xe5, 0xb7, 0x9a, 0xf4, 0x61, 0x59, 0xaf, 0x4a, 0x35, 0xc5, 0x37, 0x59, 0x38, 0xf6, 0x43, + 0xea, 0xd5, 0x3e, 0xa4, 0x7d, 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x24, 0xfd, 0xd5, 0x85, 0x65, 0xca, + 0xf3, 0xf8, 0x35, 0x8f, 0xd2, 0x5c, 0x66, 0xc5, 0x40, 0xed, 0x24, 0xf5, 0xfe, 0x2b, 0x71, 0x68, + 0xb2, 0xed, 0x51, 0x2d, 0x5c, 0xa7, 0xd3, 0x83, 0x47, 0xd0, 0x99, 0x9e, 0xd9, 0xcb, 0x57, 0xeb, + 0x57, 0x82, 0x47, 0xb0, 0xd0, 0x17, 0x45, 0x36, 0x28, 0xdb, 0xb7, 0xb6, 0x27, 0xb5, 0x67, 0x5a, + 0x4d, 0xed, 0xb5, 0xda, 0x68, 0x34, 0xe6, 0x8c, 0xc6, 0xd3, 0xa9, 0x56, 0x0a, 0x9b, 0xf8, 0xe0, + 0x8d, 0xea, 0xc1, 0x05, 0x35, 0xbd, 0x78, 0x9b, 0xfc, 0xe4, 0xc0, 0xad, 0xba, 0x0b, 0xd7, 0x1a, + 0xdc, 0xb2, 0x22, 0xee, 0xcc, 0x8a, 0x78, 0xb3, 0x2a, 0xe2, 0x57, 0x15, 0xa9, 0x7e, 0x13, 0x34, + 0x6a, 0xbf, 0x09, 0xc8, 0x31, 0xdc, 0xbb, 0x54, 0xa6, 0x2d, 0x31, 0x1a, 0xab, 0x7e, 0xf8, 0x17, + 0xe5, 0x52, 0x2b, 0x2d, 0xcb, 0x4c, 0xa1, 0xda, 0x54, 0x0b, 0xe4, 0x63, 0xb8, 0xdb, 0xe7, 0xb2, + 0x56, 0x24, 0xdb, 0x6d, 0x6b, 0xe0, 0xed, 0xf2, 0xd3, 0x2b, 0xc2, 0x57, 0x2a, 0xf2, 0x19, 0x84, + 0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xaf, 0x37, 0xa1, 0xb5, 0x27, 0xc6, 0x22, 0x11, 0xaf, 0x26, + 0x73, 0xa6, 0x3e, 0x84, 0x05, 0xbd, 0xbf, 0xf5, 0x1a, 0x69, 0x53, 0x2b, 0x92, 0x3b, 0xaa, 0xa1, + 0x07, 0x2c, 0x19, 0x14, 0x89, 0x72, 0x43, 0xfd, 0x5e, 0xcc, 0x37, 0x97, 0x7e, 0x3f, 0x5f, 0x75, + 0xfe, 0x38, 0x5f, 0x75, 0xfe, 0x3c, 0x5f, 0x75, 0x7e, 0xf9, 0x6b, 0xf5, 0x7f, 0x87, 0x4d, 0xfc, + 0x3f, 0xf1, 0xe4, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x93, 0xc0, 0xc1, 0x60, 0x0c, 0x00, + 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 57b98f62c..2e484b037 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -99,6 +99,7 @@ message Node { string ID = 1; URI URI = 2; bool IsCoordinator = 3; + string State = 4; } message NodeStateMessage { diff --git a/server.go b/server.go index a4cb8fc8b..5e6f61087 100644 --- a/server.go +++ b/server.go @@ -298,6 +298,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { ID: s.nodeID, URI: s.uri, IsCoordinator: s.cluster.Coordinator == s.nodeID, + State: nodeStateDown, } s.cluster.Node = node if s.clusterDisabled { @@ -561,7 +562,10 @@ func (s *Server) receiveMessage(m Message) error { case *RecalculateCaches: s.holder.recalculateCaches() case *NodeEvent: - s.cluster.ReceiveEvent(obj) + err := s.cluster.ReceiveEvent(obj) + if err != nil { + return errors.Wrapf(err, "cluster receiving NodeEvent %v", obj) + } case *NodeStatus: s.handleRemoteStatus(obj) } diff --git a/server/server.go b/server/server.go index 010eb2f8a..79d25d270 100644 --- a/server/server.go +++ b/server/server.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - +// // Package server contains the `pilosa server` subcommand which runs Pilosa // itself. The purpose of this package is to define an easily tested Command // object which handles interpreting configuration and setting up all the