diff --git a/api.go b/api.go index 9140153a1..8f8a831e4 100644 --- a/api.go +++ b/api.go @@ -34,6 +34,7 @@ import ( "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -682,7 +683,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // ShardNodes returns the node and all replicas which should contain a shard's data. -func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) { +func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*topology.Node, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.ShardNodes") defer span.Finish() @@ -796,7 +797,7 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. -func (api *API) Hosts(ctx context.Context) []*Node { +func (api *API) Hosts(ctx context.Context) []*topology.Node { span, _ := tracing.StartSpanFromContext(ctx, "API.Hosts") defer span.Finish() return api.cluster.Nodes() @@ -809,7 +810,7 @@ func (api *API) HostStates(ctx context.Context) map[string]string { } // Node gets the ID, URI and coordinator status for this particular node. -func (api *API) Node() *Node { +func (api *API) Node() *topology.Node { node := api.server.node() return &node } @@ -1700,7 +1701,7 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I } // SetCoordinator makes a new Node the cluster coordinator. -func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { +func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *topology.Node, err error) { span, _ := tracing.StartSpanFromContext(ctx, "API.SetCoordinator") defer span.Finish() @@ -1733,7 +1734,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // RemoveNode puts the cluster into the "RESIZING" state and begins the job of // removing the given node. -func (api *API) RemoveNode(id string) (*Node, error) { +func (api *API) RemoveNode(id string) (*topology.Node, error) { if err := api.validate(apiRemoveNode); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -1743,7 +1744,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { if !api.cluster.topologyContainsNode(id) { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") } - removeNode = &Node{ + removeNode = &topology.Node{ ID: id, } } diff --git a/broadcast.go b/broadcast.go index 37d2bb39d..f883d421d 100644 --- a/broadcast.go +++ b/broadcast.go @@ -17,6 +17,7 @@ package pilosa import ( "fmt" + "github.com/pilosa/pilosa/v2/topology" "github.com/pkg/errors" ) @@ -30,7 +31,7 @@ type Serializer interface { type broadcaster interface { SendSync(Message) error SendAsync(Message) error - SendTo(*Node, Message) error + SendTo(*topology.Node, Message) error } // Message is the interface implemented by all core pilosa types which can be serialized to messages. @@ -49,7 +50,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil } func (nopBroadcaster) SendAsync(Message) error { return nil } // SendTo is a no-op implementation of Broadcaster SendTo method. -func (nopBroadcaster) SendTo(*Node, Message) error { return nil } +func (nopBroadcaster) SendTo(*topology.Node, Message) error { return nil } // Broadcast message types. const ( diff --git a/client.go b/client.go index 4cd410345..ad53cdf4f 100644 --- a/client.go +++ b/client.go @@ -18,6 +18,9 @@ import ( "context" "io" "time" + + pnet "github.com/pilosa/pilosa/v2/net" + "github.com/pilosa/pilosa/v2/topology" ) // Bit represents the intersection of a row and a column. It can be specified by @@ -51,10 +54,10 @@ type InternalClient interface { MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) - PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error + PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) - Nodes(ctx context.Context) ([]*Node, error) + FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) + Nodes(ctx context.Context) ([]*topology.Node, error) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error @@ -67,69 +70,69 @@ type InternalClient interface { ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) - ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, uri *URI, msg []byte) error - RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) - RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error - ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error + FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) + ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error + RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) + RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) + ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error + ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) FinishTransaction(ctx context.Context, id string) (*Transaction, error) Transactions(ctx context.Context) (map[string]*Transaction, error) GetTransaction(ctx context.Context, id string) (*Transaction, error) - GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) - GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error) + GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) + GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) } //=============== // InternalQueryClient is the internal interface for querying a node. type InternalQueryClient interface { - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) + QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) // Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist. - TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) - TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error) + TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) + TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error) - FindIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) - FindFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) + FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) + FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) - CreateIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) - CreateFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) + CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) + CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) } type nopInternalQueryClient struct{} -func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { +func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) { +func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) { return nil, nil } -func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) { +func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) { return nil, nil } -func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) { +func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) { return nil, nil } -func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) { +func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) { return nil, nil } -func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) { +func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) { return nil, nil } -func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) { +func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) { return nil, nil } @@ -153,17 +156,17 @@ func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, return nil, nil } func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } -func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error { +func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error { return nil } func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { return nil } -func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { +func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) { return nil, nil } -func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) { +func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { return nil, nil } func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { @@ -179,11 +182,11 @@ func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueReq return nil } -func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { +func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { return nil } -func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error { +func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error { return nil } @@ -209,25 +212,25 @@ func (n nopInternalClient) CreateField(ctx context.Context, index, field string) func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { return nil } -func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { +func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { return nil, nil } -func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { +func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { return nil, nil, nil } -func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { +func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error { return nil } -func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) { +func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) { return nil, nil } -func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) { +func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) { return nil, nil } @@ -244,10 +247,10 @@ func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Tran return nil, nil } -func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) { +func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) { return nil, nil } -func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error) { +func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index 3a9976e72..405b65901 100644 --- a/cluster.go +++ b/cluster.go @@ -32,7 +32,9 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/v2/internal" "github.com/pilosa/pilosa/v2/logger" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" @@ -66,138 +68,17 @@ const ( defaultConfirmDownSleep = 1 * time.Second ) -// Node represents a node in the cluster. -type Node struct { - ID string `json:"id"` - URI URI `json:"uri"` - GRPCURI URI `json:"grpc-uri"` - IsCoordinator bool `json:"isCoordinator"` - State string `json:"state"` -} - -func (n *Node) Clone() *Node { - if n == nil { - return nil - } - other := *n - return &other -} - -func (n Node) String() string { - return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID) -} - -// Nodes represents a list of nodes. -type Nodes []*Node - -// Contains returns true if a node exists in the list. -func (a Nodes) Contains(n *Node) bool { - for i := range a { - if a[i] == n { - return true - } - } - return false -} - -// ContainsID returns true if host matches one of the node's id. -func (a Nodes) ContainsID(id string) bool { - for _, n := range a { - if n.ID == id { - return true - } - } - return false -} - -// NodeByID returns the node for an ID. If the ID is not found, -// it returns nil. -func (a Nodes) NodeByID(id string) *Node { - for _, n := range a { - if n.ID == id { - return n - } - } - return nil -} - -// Filter returns a new list of nodes with node removed. -func (a Nodes) Filter(n *Node) []*Node { - other := make([]*Node, 0, len(a)) - for i := range a { - if a[i] != n { - other = append(other, a[i]) - } - } - return other -} - -// FilterID returns a new list of nodes with ID removed. -func (a Nodes) FilterID(id string) []*Node { - other := make([]*Node, 0, len(a)) - for _, node := range a { - if node.ID != id { - other = append(other, node) - } - } - return other -} - -// FilterURI returns a new list of nodes with URI removed. -func (a Nodes) FilterURI(uri URI) []*Node { - other := make([]*Node, 0, len(a)) - for _, node := range a { - if node.URI != uri { - other = append(other, node) - } - } - return other -} - -// IDs returns a list of all node IDs. -func (a Nodes) IDs() []string { - ids := make([]string, len(a)) - for i, n := range a { - ids[i] = n.ID - } - return ids -} - -// URIs returns a list of all uris. -func (a Nodes) URIs() []URI { - uris := make([]URI, len(a)) - for i, n := range a { - uris[i] = n.URI - } - return uris -} - -// Clone returns a shallow copy of nodes. -func (a Nodes) Clone() []*Node { - other := make([]*Node, len(a)) - copy(other, a) - return other -} - -// byID implements sort.Interface for []Node based on -// the ID field. -type byID []*Node - -func (h byID) Len() int { return len(h) } -func (h byID) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h byID) Less(i, j int) bool { return h[i].ID < h[j].ID } - // nodeAction represents a node that is joining or leaving the cluster. type nodeAction struct { - node *Node + node *topology.Node action string } // cluster represents a collection of nodes. type cluster struct { // nolint: maligned id string - Node *Node - nodes []*Node + Node *topology.Node + nodes []*topology.Node // Hashing algorithm used to assign partitions to nodes. Hasher Hasher @@ -303,14 +184,14 @@ func (c *cluster) abortAntiEntropy() { } } -func (c *cluster) coordinatorNode() *Node { +func (c *cluster) coordinatorNode() *topology.Node { c.mu.RLock() defer c.mu.RUnlock() return c.unprotectedCoordinatorNode() } // unprotectedCoordinatorNode returns the coordinator node. -func (c *cluster) unprotectedCoordinatorNode() *Node { +func (c *cluster) unprotectedCoordinatorNode() *topology.Node { return c.unprotectedNodeByID(c.Coordinator) } @@ -329,7 +210,7 @@ func (c *cluster) unprotectedIsCoordinator() bool { // Coordinator. In response to this, the current node // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. -func (c *cluster) setCoordinator(n *Node) error { +func (c *cluster) setCoordinator(n *topology.Node) error { c.mu.Lock() defer c.mu.Unlock() // Verify that the new Coordinator value matches @@ -376,13 +257,13 @@ func (c *cluster) unprotectedSendSync(m Message) error { // changing the corresponding node's IsCoordinator value // to true, and sets all other nodes to false. Returns true if the value // changed. -func (c *cluster) updateCoordinator(n *Node) bool { // nolint: unparam +func (c *cluster) updateCoordinator(n *topology.Node) bool { // nolint: unparam c.mu.Lock() defer c.mu.Unlock() return c.unprotectedUpdateCoordinator(n) } -func (c *cluster) unprotectedUpdateCoordinator(n *Node) bool { +func (c *cluster) unprotectedUpdateCoordinator(n *topology.Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID @@ -400,7 +281,7 @@ func (c *cluster) unprotectedUpdateCoordinator(n *Node) bool { // addNode adds a node to the Cluster and updates and saves the // new topology. unprotected. -func (c *cluster) addNode(node *Node) error { +func (c *cluster) addNode(node *topology.Node) error { // If the node being added is the coordinator, set it for this node. if node.IsCoordinator { c.Coordinator = node.ID @@ -444,7 +325,7 @@ func (c *cluster) removeNode(nodeID string) error { // nodeIDs returns the list of IDs in the cluster. func (c *cluster) nodeIDs() []string { - return Nodes(c.nodes).IDs() + return topology.Nodes(c.nodes).IDs() } func (c *cluster) unprotectedSetID(id string) { @@ -629,14 +510,14 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { } } -func (c *cluster) nodeByID(id string) *Node { +func (c *cluster) nodeByID(id string) *topology.Node { c.mu.RLock() defer c.mu.RUnlock() return c.unprotectedNodeByID(id) } // unprotectedNodeByID returns a node reference by ID. -func (c *cluster) unprotectedNodeByID(id string) *Node { +func (c *cluster) unprotectedNodeByID(id string) *topology.Node { for _, n := range c.nodes { if n.ID == id { return n @@ -668,7 +549,7 @@ func (c *cluster) nodePositionByID(nodeID string) int { // addNodeBasicSorted adds a node to the cluster, sorted by id. Returns a // pointer to the node and true if the node was added. unprotected. -func (c *cluster) addNodeBasicSorted(node *Node) bool { +func (c *cluster) addNodeBasicSorted(node *topology.Node) bool { n := c.unprotectedNodeByID(node.ID) if n != nil { if n.State != node.State || n.IsCoordinator != node.IsCoordinator || n.URI != node.URI { @@ -684,17 +565,17 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool { c.nodes = append(c.nodes, node) // All hosts must be merged in the same order on all nodes in the cluster. - sort.Sort(byID(c.nodes)) + sort.Sort(topology.ByID(c.nodes)) return true } // Nodes returns a copy of the slice of nodes in the cluster. Safe for // concurrent use, result may be modified. -func (c *cluster) Nodes() []*Node { +func (c *cluster) Nodes() []*topology.Node { c.mu.RLock() defer c.mu.RUnlock() - ret := make([]*Node, len(c.nodes)) + ret := make([]*topology.Node, len(c.nodes)) copy(ret, c.nodes) return ret } @@ -851,7 +732,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSour srcCluster := c if action == resizeJobActionAdd && c.ReplicaN > 1 { srcCluster = newCluster() - srcCluster.nodes = Nodes(c.nodes).Clone() + srcCluster.nodes = topology.Nodes(c.nodes).Clone() srcCluster.Hasher = c.Hasher srcCluster.partitionN = c.partitionN srcCluster.ReplicaN = 1 @@ -1041,26 +922,26 @@ func (c *cluster) idPartition(index string, id uint64) int { } // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. -func (c *cluster) ShardNodes(index string, shard uint64) []*Node { +func (c *cluster) ShardNodes(index string, shard uint64) []*topology.Node { c.mu.RLock() defer c.mu.RUnlock() return c.shardNodes(index, shard) } // shardNodes returns a list of nodes that own a shard. unprotected -func (c *cluster) shardNodes(index string, shard uint64) []*Node { +func (c *cluster) shardNodes(index string, shard uint64) []*topology.Node { return c.partitionNodes(c.shardToShardPartition(index, shard)) } // KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use. -func (c *cluster) KeyNodes(index, key string) []*Node { +func (c *cluster) KeyNodes(index, key string) []*topology.Node { c.mu.RLock() defer c.mu.RUnlock() return c.keyNodes(index, key) } // keyNodes returns a list of nodes that own a key. unprotected -func (c *cluster) keyNodes(index, key string) []*Node { +func (c *cluster) keyNodes(index, key string) []*topology.Node { return c.partitionNodes(c.Topology.KeyPartition(index, key)) } @@ -1068,11 +949,11 @@ func (c *cluster) keyNodes(index, key string) []*Node { func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool { c.mu.RLock() defer c.mu.RUnlock() - return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) + return topology.Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) } // partitionNodes returns a list of nodes that own a partition. unprotected. -func (c *cluster) partitionNodes(partitionID int) []*Node { +func (c *cluster) partitionNodes(partitionID int) []*topology.Node { // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. @@ -1114,11 +995,11 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { return nil } // Collect nodes around the ring. - nodes := make([]*Node, 0, replicaN) + nodes := make([]*topology.Node, 0, replicaN) for i := 0; i < replicaN; i++ { if useTopology { maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN] - if node := Nodes(c.nodes).NodeByID(maybeNodeID); node != nil { + if node := topology.Nodes(c.nodes).NodeByID(maybeNodeID); node != nil { nodes = append(nodes, node) } } else { @@ -1129,14 +1010,14 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { return nodes } -func (c *cluster) primaryPartitionNode(partition int) *Node { +func (c *cluster) primaryPartitionNode(partition int) *topology.Node { c.mu.RLock() defer c.mu.RUnlock() return c.unprotectedPrimaryPartitionNode(partition) } // unprotectedPrimaryPartition returns tprimary node of partition. -func (c *cluster) unprotectedPrimaryPartitionNode(partition int) *Node { +func (c *cluster) unprotectedPrimaryPartitionNode(partition int) *topology.Node { if nodes := c.partitionNodes(partition); len(nodes) > 0 { return nodes[0] } @@ -1199,7 +1080,7 @@ func (topo *Topology) GetReplicasForPrimary(primary int) (replicaNodeIDs, nonRep } // containsShards is like OwnsShards, but it includes replicas. -func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 { +func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *topology.Node) []uint64 { var shards []uint64 _ = availableShards.ForEach(func(i uint64) error { p := c.shardToShardPartition(index, i) @@ -1417,7 +1298,7 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error { return c.unprotectedSendSync(status) // TODO fix c.Status } -func (c *cluster) sendTo(node *Node, m Message) error { +func (c *cluster) sendTo(node *topology.Node, m Message) error { if err := c.broadcaster.SendTo(node, m); err != nil { return errors.Wrap(err, "sending") } @@ -1512,7 +1393,7 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (* // toCluster is a clone of Cluster with the new node added/removed for comparison. toCluster := newCluster() - toCluster.nodes = Nodes(c.nodes).Clone() + toCluster.nodes = topology.Nodes(c.nodes).Clone() toCluster.Hasher = c.Hasher toCluster.partitionN = c.partitionN toCluster.ReplicaN = c.ReplicaN @@ -1830,7 +1711,7 @@ type resizeJob struct { } // newResizeJob returns a new instance of resizeJob. -func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob { +func newResizeJob(existingNodes []*topology.Node, node *topology.Node, action string) *resizeJob { // Build a map of uris to track their resize status. // The value for a node will be set to true after that node @@ -1918,7 +1799,7 @@ func (j *resizeJob) distributeResizeInstructions() error { for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. - node := &Node{ + node := &topology.Node{ ID: instr.Node.ID, URI: instr.Node.URI, GRPCURI: instr.Node.GRPCURI, @@ -2147,7 +2028,7 @@ func (c *cluster) considerTopology() error { // band aid to protect against false nodeLeave events from memberlist // the test is the lightest weight endpoint of the node in question /version // TODO provide more robust solution to false nodeLeave events -func (c *cluster) confirmNodeDown(uri URI) bool { +func (c *cluster) confirmNodeDown(uri pnet.URI) bool { u := url.URL{ Scheme: uri.Scheme, Host: uri.HostPort(), @@ -2219,7 +2100,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { } // nodeJoin should only be called by the coordinator. -func (c *cluster) nodeJoin(node *Node) error { +func (c *cluster) nodeJoin(node *topology.Node) error { c.abortAntiEntropy() // Technically there is a race condition here which could // allow the anti-entropy process to re-start (and acquire @@ -2343,7 +2224,7 @@ func (c *cluster) nodeLeave(nodeID string) error { // See if resize job can be generated if _, err := c.unprotectedGenerateResizeJobByAction( nodeAction{ - node: &Node{ID: nodeID}, + node: &topology.Node{ID: nodeID}, action: resizeJobActionRemove}, ); err != nil { return errors.Wrap(err, "generating job") @@ -2364,7 +2245,7 @@ func (c *cluster) nodeLeave(nodeID string) error { if err := c.unprotectedSetStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{node: &Node{ID: nodeID}, action: resizeJobActionRemove} + c.joiningLeavingNodes <- nodeAction{node: &topology.Node{ID: nodeID}, action: resizeJobActionRemove} return nil } @@ -2433,7 +2314,7 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { if node.ID == c.Node.ID { continue } - if Nodes(officialNodes).ContainsID(node.ID) { + if topology.Nodes(officialNodes).ContainsID(node.ID) { continue } nodeIDsToRemove = append(nodeIDsToRemove, node.ID) @@ -2455,7 +2336,7 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { // unprotectedPreviousNode returns the node listed before the current node in c.Nodes. // If there is only one node in the cluster, returns nil. // If the current node is the first node in the list, returns the last node. -func (c *cluster) unprotectedPreviousNode() *Node { +func (c *cluster) unprotectedPreviousNode() *topology.Node { if len(c.nodes) <= 1 { return nil } @@ -2472,13 +2353,13 @@ func (c *cluster) unprotectedPreviousNode() *Node { // PrimaryReplicaNode returns the node listed before the current node in c.Nodes. // This is different than "previous node" as the first node always returns nil. -func (c *cluster) PrimaryReplicaNode() *Node { +func (c *cluster) PrimaryReplicaNode() *topology.Node { c.mu.RLock() defer c.mu.RUnlock() return c.unprotectedPrimaryReplicaNode() } -func (c *cluster) unprotectedPrimaryReplicaNode() *Node { +func (c *cluster) unprotectedPrimaryReplicaNode() *topology.Node { pos := c.nodePositionByID(c.Node.ID) if pos <= 0 { return nil @@ -2492,11 +2373,11 @@ func (c *cluster) setStatic(hosts []string) error { c.Static = true c.Coordinator = c.Node.ID for _, address := range hosts { - uri, err := NewURIFromAddress(address) + uri, err := pnet.NewURIFromAddress(address) if err != nil { return errors.Wrap(err, "getting URI") } - c.nodes = append(c.nodes, &Node{URI: *uri}) + c.nodes = append(c.nodes, &topology.Node{URI: *uri}) } return nil } @@ -2822,7 +2703,7 @@ func (c *cluster) findIndexKeys(ctx context.Context, indexName string, keys ...s // TODO: use local replicas to short-circuit network traffic // Group keys by node. - keysByNode := make(map[*Node][]string) + keysByNode := make(map[*topology.Node][]string) for partitionID, keys := range keysByPartition { // Find the primary node for this partition. primary := c.primaryPartitionNode(partitionID) @@ -2929,7 +2810,7 @@ func (c *cluster) createIndexKeys(ctx context.Context, indexName string, keys .. // Group keys by node. // Delete remote keys from the by-partition map so that it can be used for local translation. - keysByNode := make(map[*Node][]string) + keysByNode := make(map[*topology.Node][]string) for partitionID, keys := range keysByPartition { // Find the primary node for this partition. primary := c.primaryPartitionNode(partitionID) @@ -3088,7 +2969,7 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS type ClusterStatus struct { ClusterID string State string - Nodes []*Node + Nodes []*topology.Node Schema *Schema } @@ -3096,8 +2977,8 @@ type ClusterStatus struct { // during a cluster resize operation. type ResizeInstruction struct { JobID int64 - Node *Node - Coordinator *Node + Node *topology.Node + Coordinator *topology.Node Sources []*ResizeSource TranslationSources []*TranslationResizeSource NodeStatus *NodeStatus @@ -3107,17 +2988,17 @@ type ResizeInstruction struct { // ResizeSource is the source of data for a node acting on a // ResizeInstruction. type ResizeSource struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` - Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` + Node *topology.Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } // TranslationResizeSource is the source of translation data for // a node acting on a ResizeInstruction. type TranslationResizeSource struct { - Node *Node + Node *topology.Node Index string PartitionID int } @@ -3125,7 +3006,7 @@ type TranslationResizeSource struct { // translateResizeNode holds the node/partition pairs used // to create a TranslationResizeSource for each index. type translationResizeNode struct { - node *Node + node *topology.Node partitionID int } @@ -3219,18 +3100,18 @@ type DeleteViewMessage struct { // that the resize instructions performed on a single node have completed. type ResizeInstructionComplete struct { JobID int64 - Node *Node + Node *topology.Node Error string } // SetCoordinatorMessage is an internal message instructing nodes to honor a new coordinator. type SetCoordinatorMessage struct { - New *Node + New *topology.Node } // UpdateCoordinatorMessage is an internal message for reassigning the coordinator. type UpdateCoordinatorMessage struct { - New *Node + New *topology.Node } // NodeStateMessage is an internal message for broadcasting a node's state. @@ -3241,7 +3122,7 @@ type NodeStateMessage struct { // NodeStatus is an internal message representing the contents of a node. type NodeStatus struct { - Node *Node + Node *topology.Node Indexes []*IndexStatus Schema *Schema } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index a9ec930be..77fc173b4 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -33,24 +33,26 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/gorilla/mux" "github.com/pilosa/pilosa/v2/logger" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/testhook" + "github.com/pilosa/pilosa/v2/topology" "github.com/pkg/errors" ) // Ensure that fragCombos creates the correct fragment mapping. func TestFragCombos(t *testing.T) { - uri0, err := NewURIFromAddress("host0") + uri0, err := pnet.NewURIFromAddress("host0") if err != nil { t.Fatal(err) } - uri1, err := NewURIFromAddress("host1") + uri1, err := pnet.NewURIFromAddress("host1") if err != nil { t.Fatal(err) } - node0 := &Node{ID: "node0", URI: *uri0} - node1 := &Node{ID: "node1", URI: *uri1} + node0 := &topology.Node{ID: "node0", URI: *uri0} + node1 := &topology.Node{ID: "node1", URI: *uri1} c := newCluster() c.addNodeBasicSorted(node0) @@ -110,27 +112,27 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index { // Ensure that fragSources creates the correct fragment mapping. func TestFragSources(t *testing.T) { - uri0, err := NewURIFromAddress("host0") + uri0, err := pnet.NewURIFromAddress("host0") if err != nil { t.Fatal(err) } - uri1, err := NewURIFromAddress("host1") + uri1, err := pnet.NewURIFromAddress("host1") if err != nil { t.Fatal(err) } - uri2, err := NewURIFromAddress("host2") + uri2, err := pnet.NewURIFromAddress("host2") if err != nil { t.Fatal(err) } - uri3, err := NewURIFromAddress("host3") + uri3, err := pnet.NewURIFromAddress("host3") if err != nil { t.Fatal(err) } - node0 := &Node{ID: "node0", URI: *uri0} - node1 := &Node{ID: "node1", URI: *uri1} - node2 := &Node{ID: "node2", URI: *uri2} - node3 := &Node{ID: "node3", URI: *uri3} + node0 := &topology.Node{ID: "node0", URI: *uri0} + node1 := &topology.Node{ID: "node1", URI: *uri1} + node2 := &topology.Node{ID: "node2", URI: *uri2} + node3 := &topology.Node{ID: "node3", URI: *uri3} c1 := newCluster() c1.ReplicaN = 1 @@ -224,8 +226,8 @@ func TestFragSources(t *testing.T) { "node0": {}, "node1": {}, "node2": { - {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, - {&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, + {&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -236,11 +238,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*ResizeSource{ "node0": { - {&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)}, + {&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)}, }, "node1": { - {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, - {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, + {&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -251,11 +253,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*ResizeSource{ "node0": { - {&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, - {&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, + {&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)}, + {&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)}, }, "node1": { - {&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)}, + {&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)}, }, "node2": {}, }, @@ -304,37 +306,37 @@ func TestFragSources(t *testing.T) { // Ensure that fragSources creates the correct fragment mapping. func TestResizeJob(t *testing.T) { - uri0, err := NewURIFromAddress("host0") + uri0, err := pnet.NewURIFromAddress("host0") if err != nil { t.Fatal(err) } - uri1, err := NewURIFromAddress("host1") + uri1, err := pnet.NewURIFromAddress("host1") if err != nil { t.Fatal(err) } - uri2, err := NewURIFromAddress("host2") + uri2, err := pnet.NewURIFromAddress("host2") if err != nil { t.Fatal(err) } - node0 := &Node{ID: "node0", URI: *uri0} - node1 := &Node{ID: "node1", URI: *uri1} - node2 := &Node{ID: "node2", URI: *uri2} + node0 := &topology.Node{ID: "node0", URI: *uri0} + node1 := &topology.Node{ID: "node1", URI: *uri1} + node2 := &topology.Node{ID: "node2", URI: *uri2} tests := []struct { - existingNodes []*Node - node *Node + existingNodes []*topology.Node + node *topology.Node action string expectedIDs map[string]bool }{ { - existingNodes: []*Node{node0, node1}, + existingNodes: []*topology.Node{node0, node1}, node: node2, action: resizeJobActionAdd, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, }, { - existingNodes: []*Node{node0, node1, node2}, + existingNodes: []*topology.Node{node0, node1, node2}, node: node2, action: resizeJobActionRemove, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, @@ -355,7 +357,7 @@ func TestResizeJob(t *testing.T) { // Ensure the cluster can fairly distribute partitions across the nodes. func TestCluster_Owners(t *testing.T) { c := cluster{ - nodes: []*Node{ + nodes: []*topology.Node{ {URI: NewTestURIFromHostPort("serverA", 1000)}, {URI: NewTestURIFromHostPort("serverB", 1000)}, {URI: NewTestURIFromHostPort("serverC", 1000)}, @@ -365,12 +367,12 @@ func TestCluster_Owners(t *testing.T) { } // Verify nodes are distributed. - if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.nodes[0], c.nodes[1]}) { + if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*topology.Node{c.nodes[0], c.nodes[1]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } // Verify nodes go around the ring. - if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.nodes[2], c.nodes[0]}) { + if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*topology.Node{c.nodes[2], c.nodes[0]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } } @@ -436,15 +438,15 @@ func TestCluster_Nodes(t *testing.T) { uri2 := NewTestURIFromHostPort("node2", 0) uri3 := NewTestURIFromHostPort("node3", 0) - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - node3 := &Node{ID: "node3", URI: uri3} + node0 := &topology.Node{ID: "node0", URI: uri0} + node1 := &topology.Node{ID: "node1", URI: uri1} + node2 := &topology.Node{ID: "node2", URI: uri2} + node3 := &topology.Node{ID: "node3", URI: uri3} - nodes := []*Node{node0, node1, node2} + nodes := []*topology.Node{node0, node1, node2} t.Run("NodeIDs", func(t *testing.T) { - actual := Nodes(nodes).IDs() + actual := topology.Nodes(nodes).IDs() expected := []string{node0.ID, node1.ID, node2.ID} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) @@ -452,24 +454,24 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Filter", func(t *testing.T) { - actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() - expected := []URI{uri0, uri2} + actual := topology.Nodes(topology.Nodes(nodes).Filter(nodes[1])).URIs() + expected := []pnet.URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("FilterURI", func(t *testing.T) { - actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() - expected := []URI{uri0, uri2} + actual := topology.Nodes(topology.Nodes(nodes).FilterURI(uri1)).URIs() + expected := []pnet.URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("Contains", func(t *testing.T) { - actualTrue := Nodes(nodes).Contains(node1) - actualFalse := Nodes(nodes).Contains(node3) + actualTrue := topology.Nodes(nodes).Contains(node1) + actualFalse := topology.Nodes(nodes).Contains(node3) if !reflect.DeepEqual(actualTrue, true) { t.Errorf("expected: %v, but got: %v", true, actualTrue) } @@ -479,9 +481,9 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Clone", func(t *testing.T) { - clone := Nodes(nodes).Clone() - actual := Nodes(clone).URIs() - expected := []URI{uri0, uri1, uri2} + clone := topology.Nodes(nodes).Clone() + actual := topology.Nodes(clone).URIs() + expected := []pnet.URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -489,9 +491,9 @@ func TestCluster_Nodes(t *testing.T) { } func TestCluster_PreviousNode(t *testing.T) { - node0 := &Node{ID: "node0"} - node1 := &Node{ID: "node1"} - node2 := &Node{ID: "node2"} + node0 := &topology.Node{ID: "node0"} + node1 := &topology.Node{ID: "node1"} + node2 := &topology.Node{ID: "node2"} t.Run("OneNode", func(t *testing.T) { c := newCluster() @@ -547,8 +549,8 @@ func TestCluster_Coordinator(t *testing.T) { uri1 := NewTestURIFromHostPort("node1", 0) uri2 := NewTestURIFromHostPort("node2", 0) - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} + node1 := &topology.Node{ID: "node1", URI: uri1} + node2 := &topology.Node{ID: "node2", URI: uri2} c1 := *newCluster() c1.Node = node1 @@ -574,10 +576,10 @@ func TestCluster_Topology(t *testing.T) { uri2 := NewTestURIFromHostPort("host2", 0) invalid := NewTestURIFromHostPort("invalid", 0) - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} + node0 := &topology.Node{ID: "node0", URI: uri0} + node1 := &topology.Node{ID: "node1", URI: uri1} + node2 := &topology.Node{ID: "node2", URI: uri2} + nodeinvalid := &topology.Node{ID: "nodeinvalid", URI: invalid} t.Run("AddNode", func(t *testing.T) { err := c1.addNode(node1) @@ -984,7 +986,7 @@ func TestCluster_confirmNodeDownUp(t *testing.T) { if err != nil { t.Error("bad test setup") } - uri := URI{} + uri := pnet.URI{} host, port, _ := net.SplitHostPort(u.Host) uri.Scheme = u.Scheme uri.Host = host @@ -1018,7 +1020,7 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) { if err != nil { t.Error("bad test setup") } - uri := URI{} + uri := pnet.URI{} host, port, _ := net.SplitHostPort(u.Host) uri.Scheme = u.Scheme uri.Host = host @@ -1040,7 +1042,7 @@ func TestCluster_confirmNodeDownDown(t *testing.T) { if testing.Short() { t.Skip() } - uri := URI{} + uri := pnet.URI{} uri.Scheme = "http" uri.Host = "DoesntMatter" uri.Port = 6666 @@ -1063,7 +1065,7 @@ func TestCluster_GetNonPrimaryReplicas(t *testing.T) { nNodes := 4 for i := 0; i < nNodes; i++ { nodeID := fmt.Sprintf("node%d", i) - c.nodes = append(c.nodes, &Node{ + c.nodes = append(c.nodes, &topology.Node{ ID: nodeID, URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 719a256f5..5a6b0fad8 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -19,13 +19,17 @@ import ( "compress/gzip" "context" "time" + //"fmt" "fmt" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/http" "io" "io/ioutil" gohttp "net/http" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/http" + pnet "github.com/pilosa/pilosa/v2/net" + //"log" "os" //"path/filepath" @@ -140,15 +144,15 @@ func main() { vv("total elapsed '%v'", time.Since(t0)) } -var globURI *pilosa.URI +var globURI *pnet.URI func init() { var err error - globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101) + globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101) panicOn(err) } // get correct node to go to. -func GetImportRoaringURI(index string, shard uint64) *pilosa.URI { +func GetImportRoaringURI(index string, shard uint64) *pnet.URI { return globURI } diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 9d49d20a5..11bbec314 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -32,6 +32,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/http" + pnet "github.com/pilosa/pilosa/v2/net" ) // slurp: slurp is a load-tester for importing bulk data. @@ -191,7 +192,7 @@ func main() { flag.StringVar(&tarSrcPath, "src", "q2.tar.gz", "data to import") flag.Parse() - uri, err := pilosa.NewURIFromAddress(host) + uri, err := pnet.NewURIFromAddress(host) panicOn(err) globURI = uri @@ -253,9 +254,9 @@ func stopProfile(host, outfile string) { } -var globURI *pilosa.URI +var globURI *pnet.URI // get correct node to go to. -func GetImportRoaringURI(index string, shard uint64) *pilosa.URI { +func GetImportRoaringURI(index string, shard uint64) *pnet.URI { return globURI } diff --git a/disco/disco.go b/disco/disco.go new file mode 100644 index 000000000..007a2e5c5 --- /dev/null +++ b/disco/disco.go @@ -0,0 +1,222 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disco + +import ( + "context" + "fmt" + "io" + + "github.com/pilosa/pilosa/v2/roaring" +) + +var ( + ErrTooManyResults error = fmt.Errorf("too many results") + ErrNoResults error = fmt.Errorf("no results") + ErrKeyDeleted error = fmt.Errorf("key deleted") +) + +type Peer struct { + URL string + ID string +} + +func (p *Peer) String() string { + return fmt.Sprintf(`{"ID": "%s", "URL": "%s"}`, p.ID, p.URL) +} + +type DisCo interface { + io.Closer + + Start(ctx context.Context) (InitialClusterState, error) + IsLeader() bool + ID() string + Leader() *Peer + Peers() []*Peer + DeleteNode(ctx context.Context, id string) error +} + +type ( + InitialClusterState string + ClusterState string +) + +const ( + InitialClusterStateNew InitialClusterState = "new" + InitialClusterStateExisting InitialClusterState = "existing" + + // ClusterState represents the state returned in the /status endpoint. + ClusterStateUnknown ClusterState = "UNKNOWN" + ClusterStateStarting ClusterState = "STARTING" + ClusterStateDegraded ClusterState = "DEGRADED" // cluster is running but we've lost some # of hosts >0 but < replicaN + ClusterStateNormal ClusterState = "NORMAL" + ClusterStateResizing ClusterState = "RESIZING" // cluster is replicating data to other nodes + ClusterStateDown ClusterState = "DOWN" // cluster is unable to serve queries +) + +type NodeState string + +const ( + NodeStateUnknown NodeState = "UNKNOWN" + NodeStateStarting NodeState = "STARTING" + NodeStateStarted NodeState = "STARTED" + NodeStateResizing NodeState = "RESIZING" +) + +type Stator interface { + Started(ctx context.Context) error + ClusterState(context.Context) (ClusterState, error) + NodeState(context.Context, string) (NodeState, error) + NodeStates(context.Context) (map[string]NodeState, error) +} + +// Index is a struct which contains the data encoded for the index as well as +// for each of its fields. +type Index struct { + Data []byte + Fields map[string][]byte +} + +type Schemator interface { + Schema(ctx context.Context) (map[string]*Index, error) + Index(ctx context.Context, name string) ([]byte, error) + CreateIndex(ctx context.Context, name string, val []byte) error + DeleteIndex(ctx context.Context, name string) error + Field(ctx context.Context, index, field string) ([]byte, error) + CreateField(ctx context.Context, index, field string, val []byte) error + DeleteField(ctx context.Context, index, field string) error +} + +type Metadata interface { + Marshal() ([]byte, error) + Unmarshal([]byte) error +} + +type Metadator interface { + Metadata(ctx context.Context, peerID string) ([]byte, error) + SetMetadata(ctx context.Context, metadata []byte) error +} + +// Resizer triggers resizing the node and changes cluster state into RESIZING. +// We can also return some kind of handler from Resize function (e.g. key-value) +type Resizer interface { + Resize(ctx context.Context) (func([]byte) error, error) + DoneResize() error + Watch(ctx context.Context, peerID string, onUpdate func([]byte) error) error +} + +// Sharder is an interface used to maintain the set of availableShards bitmaps +// per field. +type Sharder interface { + Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error) + AddShard(ctx context.Context, index, field string, shard uint64) error + AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error) + RemoveShard(ctx context.Context, index, field string, shard uint64) error +} + +// NopDisCo represents a DisCo that doesn't do anything. +var NopDisCo DisCo = &nopDisCo{ + Closer: nil, +} + +type nopDisCo struct { + io.Closer +} + +// Start is a no-op implementation of the DisCo Start method. +func (n *nopDisCo) Start(ctx context.Context) (InitialClusterState, error) { + return InitialClusterStateNew, nil +} + +// ID is a no-op implementation of the DisCo ID method. +func (n *nopDisCo) ID() string { + return "" +} + +// IsLeader is a no-op implementation of the DisCo IsLeader method. +func (n *nopDisCo) IsLeader() bool { + return false +} + +// Leader is a no-op implementation of the DisCo Leader method. +func (n *nopDisCo) Leader() *Peer { + return nil +} + +// Peers is a no-op implementation of the DisCo Peers method. +func (n *nopDisCo) Peers() []*Peer { + return nil +} + +// DeleteNode a no-op implementation of the DisCo DeleteNode method. +func (n *nopDisCo) DeleteNode(context.Context, string) error { + return nil +} + +// NopStator represents a Stator that doesn't do anything. +var NopStator Stator = &nopStator{} + +type nopStator struct{} + +// ClusterState is a no-op implementation of the Stator ClusterState method. +func (n *nopStator) ClusterState(context.Context) (ClusterState, error) { + return "", nil +} + +func (n *nopStator) Started(ctx context.Context) error { + return nil +} + +func (n *nopStator) NodeState(context.Context, string) (NodeState, error) { + return NodeStateUnknown, nil +} + +func (n *nopStator) NodeStates(context.Context) (map[string]NodeState, error) { + return nil, nil +} + +// NopResizer represents a Resizer that doesn't do anything. +var NopResizer Resizer = &nopResizer{} + +type nopResizer struct{} + +func (*nopResizer) Resize(context.Context) (func([]byte) error, error) { return nil, nil } +func (*nopResizer) DoneResize() error { return nil } +func (*nopResizer) Watch(context.Context, string, func([]byte) error) error { return nil } + +// NopSharder represents a Sharder that doesn't do anything. +var NopSharder Sharder = &nopSharder{} + +type nopSharder struct{} + +// Shards is a no-op implementation of the Sharder Shards method. +func (n *nopSharder) Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error) { + return nil, nil +} + +// AddShard is a no-op implementation of the Sharder AddShard method. +func (n *nopSharder) AddShard(ctx context.Context, index, field string, shard uint64) error { + return nil +} + +// AddShards is a no-op implementation of the Sharder AddShards method. +func (n *nopSharder) AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error) { + return nil, nil +} + +// RemoveShard is a no-op implementation of the Sharder RemoveShard method. +func (n *nopSharder) RemoveShard(ctx context.Context, index, field string, shard uint64) error { + return nil +} diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 10a2f05a2..1444247e3 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -22,8 +22,10 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/internal" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/topology" "github.com/pkg/errors" ) @@ -184,7 +186,7 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } s.decodeNodeStatus(msg, mt) return nil - case *pilosa.Node: + case *topology.Node: msg := &internal.Node{} err := proto.Unmarshal(buf, msg) if err != nil { @@ -361,7 +363,7 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeNodeEventMessage(mt) case *pilosa.NodeStatus: return s.encodeNodeStatus(mt) - case *pilosa.Node: + case *topology.Node: return s.encodeNode(mt) case *pilosa.QueryRequest: return s.encodeQueryRequest(mt) @@ -679,7 +681,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOp } // s.encodeNodes converts a slice of Nodes into its internal representation. -func (s Serializer) encodeNodes(a []*pilosa.Node) []*internal.Node { +func (s Serializer) encodeNodes(a []*topology.Node) []*internal.Node { other := make([]*internal.Node, len(a)) for i := range a { other[i] = s.encodeNode(a[i]) @@ -688,7 +690,7 @@ func (s Serializer) encodeNodes(a []*pilosa.Node) []*internal.Node { } // s.encodeNode converts a Node into its internal representation. -func (s Serializer) encodeNode(n *pilosa.Node) *internal.Node { +func (s Serializer) encodeNode(n *topology.Node) *internal.Node { return &internal.Node{ ID: n.ID, URI: s.encodeURI(n.URI), @@ -698,7 +700,7 @@ func (s Serializer) encodeNode(n *pilosa.Node) *internal.Node { } } -func (s Serializer) encodeURI(u pilosa.URI) *internal.URI { +func (s Serializer) encodeURI(u pnet.URI) *internal.URI { return &internal.URI{ Scheme: u.Scheme, Host: u.Host, @@ -948,9 +950,9 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *inter func (s Serializer) decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { m.JobID = ri.JobID - m.Node = &pilosa.Node{} + m.Node = &topology.Node{} s.decodeNode(ri.Node, m.Node) - m.Coordinator = &pilosa.Node{} + m.Coordinator = &topology.Node{} s.decodeNode(ri.Coordinator, m.Coordinator) m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) s.decodeResizeSources(ri.Sources, m.Sources) @@ -970,7 +972,7 @@ func (s Serializer) decodeResizeSources(srcs []*internal.ResizeSource, m []*pilo } func (s Serializer) decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) { - m.Node = &pilosa.Node{} + m.Node = &topology.Node{} s.decodeNode(rs.Node, m.Node) m.Index = rs.Index m.Field = rs.Field @@ -986,7 +988,7 @@ func (s Serializer) decodeTranslationResizeSources(srcs []*internal.TranslationR } func (s Serializer) decodeTranslationResizeSource(rs *internal.TranslationResizeSource, m *pilosa.TranslationResizeSource) { - m.Node = &pilosa.Node{} + m.Node = &topology.Node{} s.decodeNode(rs.Node, m.Node) m.Index = rs.Index m.PartitionID = int(rs.PartitionID) @@ -1050,9 +1052,9 @@ func (s Serializer) decodeDecimal(d *internal.Decimal, m *pql.Decimal) { m.Scale = d.Scale } -func (s Serializer) decodeNodes(a []*internal.Node, m []*pilosa.Node) { +func (s Serializer) decodeNodes(a []*internal.Node, m []*topology.Node) { for i := range a { - m[i] = &pilosa.Node{} + m[i] = &topology.Node{} s.decodeNode(a[i], m[i]) } } @@ -1060,13 +1062,13 @@ func (s Serializer) decodeNodes(a []*internal.Node, m []*pilosa.Node) { func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) { m.State = cs.State m.ClusterID = cs.ClusterID - m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) + m.Nodes = make([]*topology.Node, len(cs.Nodes)) s.decodeNodes(cs.Nodes, m.Nodes) m.Schema = &pilosa.Schema{} s.decodeSchema(cs.Schema, m.Schema) } -func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { +func (s Serializer) decodeNode(node *internal.Node, m *topology.Node) { m.ID = node.ID s.decodeURI(node.URI, &m.URI) s.decodeURI(node.GRPCURI, &m.GRPCURI) @@ -1074,7 +1076,7 @@ func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { m.State = node.State } -func (s Serializer) decodeURI(i *internal.URI, m *pilosa.URI) { +func (s Serializer) decodeURI(i *internal.URI, m *pnet.URI) { m.Scheme = i.Scheme m.Host = i.Host m.Port = uint16(i.Port) @@ -1137,18 +1139,18 @@ func (s Serializer) decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *p func (s Serializer) decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) { m.JobID = pb.JobID - m.Node = &pilosa.Node{} + m.Node = &topology.Node{} s.decodeNode(pb.Node, m.Node) m.Error = pb.Error } func (s Serializer) decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) { - m.New = &pilosa.Node{} + m.New = &topology.Node{} s.decodeNode(pb.New, m.New) } func (s Serializer) decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) { - m.New = &pilosa.Node{} + m.New = &topology.Node{} s.decodeNode(pb.New, m.New) } @@ -1159,12 +1161,12 @@ func (s Serializer) decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pil func (s Serializer) decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) { m.Event = pilosa.NodeEventType(pb.Event) - m.Node = &pilosa.Node{} + m.Node = &topology.Node{} s.decodeNode(pb.Node, m.Node) } func (s Serializer) decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { - m.Node = &pilosa.Node{} + m.Node = &topology.Node{} m.Indexes = s.decodeIndexStatuses(pb.Indexes) m.Schema = &pilosa.Schema{} s.decodeSchema(pb.Schema, m.Schema) diff --git a/etcd/cache.go b/etcd/cache.go new file mode 100644 index 000000000..633b4cf8e --- /dev/null +++ b/etcd/cache.go @@ -0,0 +1,148 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcd + +import ( + "context" + "sync" + "time" + + "github.com/pilosa/pilosa/v2/disco" +) + +// EtcdWithCache is a wrapper around the Etcd type which will return a +// cached value when the number of requests come in below a configured +// frequency. It also breaks the cache after a configured TTL. +type EtcdWithCache struct { + *Etcd + + peerMetadataMu sync.RWMutex + peerMetadata map[string][]byte + + stateMu sync.Mutex + + nodeStates map[string]nodeState + nodeStateTTL int // seconds + nodeStateFrequency int // max requests per second allowed before using the cache + + clusterStateVal disco.ClusterState + clusterStateTTL int // seconds + clusterStateFrequency int // max requests per second allowed before using the cache + clusterStateLastRequest time.Time + clusterStateLastCache time.Time +} + +type nodeState struct { + val disco.NodeState + lastRequest time.Time + lastCache time.Time +} + +// NewEtcdWithCache returns a new instance of Cache. +func NewEtcdWithCache(opt Options, replicas int) *EtcdWithCache { + return &EtcdWithCache{ + Etcd: NewEtcd(opt, replicas), + + nodeStateTTL: 6, + nodeStateFrequency: 1, + clusterStateTTL: 6, + clusterStateFrequency: 1, + + peerMetadata: make(map[string][]byte), + nodeStates: make(map[string]nodeState), + } +} + +// Metadata is a cache wrapper around the Metadator.Metadata method. +func (c *EtcdWithCache) Metadata(ctx context.Context, peerID string) ([]byte, error) { + c.peerMetadataMu.RLock() + v, ok := c.peerMetadata[peerID] + c.peerMetadataMu.RUnlock() + if ok { + return v, nil + } + v, err := c.Etcd.Metadata(ctx, peerID) + if err == nil { + c.peerMetadataMu.Lock() + c.peerMetadata[peerID] = v + c.peerMetadataMu.Unlock() + } + return v, err +} + +// ClusterState is a cache wrapper around the Stator.ClusterState method. +func (c *EtcdWithCache) ClusterState(ctx context.Context) (disco.ClusterState, error) { + c.stateMu.Lock() + defer c.stateMu.Unlock() + + now := time.Now() + if now.Sub(c.clusterStateLastCache) > (time.Duration(c.clusterStateTTL)*time.Second) || + now.Sub(c.clusterStateLastRequest) > (time.Second/time.Duration(c.clusterStateFrequency)) { + v, err := c.Etcd.ClusterState(ctx) + if err == nil { + // In order to avoid NodeState() returning a cached value after + // cluster state has changed, we reset the node state caches to + // ensure that the next call to NodeState() returns the latest + // value. And we only need to do this if the cluster state value has + // actually changed. + if c.clusterStateVal != v { + for k, ns := range c.nodeStates { + ns.lastCache = time.Time{} + c.nodeStates[k] = ns + } + } + + c.clusterStateVal = v + c.clusterStateLastCache = now + c.clusterStateLastRequest = now + } + return v, err + } + c.clusterStateLastRequest = now + return c.clusterStateVal, nil +} + +// NodeState is a cache wrapper around the Stator.NodeState method. +func (c *EtcdWithCache) NodeState(ctx context.Context, peerID string) (disco.NodeState, error) { + c.stateMu.Lock() + defer c.stateMu.Unlock() + + ns := c.nodeStates[peerID] + + now := time.Now() + if now.Sub(ns.lastCache) > (time.Duration(c.nodeStateTTL)*time.Second) || + now.Sub(ns.lastRequest) > (time.Second/time.Duration(c.nodeStateFrequency)) { + v, err := c.Etcd.NodeState(ctx, peerID) + if err == nil { + // In order to avoid ClusterState() returning a cached value after a + // node state has changed, we reset the cluster state cache to + // ensure that the next call to ClusterState() returns the latest + // value. And we only need to do this if the node state value has + // actually changed. + if ns.val != v { + c.clusterStateLastCache = time.Time{} + } + + ns.val = v + ns.lastCache = now + ns.lastRequest = now + c.nodeStates[peerID] = ns + } + return v, err + } + ns.lastRequest = now + c.nodeStates[peerID] = ns + return ns.val, nil +} diff --git a/etcd/embed.go b/etcd/embed.go new file mode 100644 index 000000000..410368ec0 --- /dev/null +++ b/etcd/embed.go @@ -0,0 +1,1003 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcd + +import ( + "bytes" + "context" + "fmt" + "log" + "path" + "strings" + "time" + + "github.com/pilosa/pilosa/v2/disco" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/clientv3/clientv3util" + "go.etcd.io/etcd/clientv3/concurrency" + "go.etcd.io/etcd/embed" + "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/pkg/types" +) + +type Options struct { + Name string `toml:"name"` + Dir string `toml:"dir"` + LClientURL string `toml:"listen-client-addr"` + AClientURL string `toml:"advertise-client-addr"` + LPeerURL string `toml:"listen-peer-addr"` + APeerURL string `toml:"advertise-peer-addr"` + InitCluster string `toml:"initial-cluster"` + ClusterURL string `toml:"cluster-url"` + ClusterName string `toml:"cluster-name"` + HeartbeatTTL int64 `toml:"heartbeat-ttl"` +} + +var ( + _ disco.DisCo = &Etcd{} + _ disco.Schemator = &Etcd{} + _ disco.Stator = &Etcd{} + _ disco.Metadator = &Etcd{} + _ disco.Resizer = &Etcd{} + _ disco.Sharder = &Etcd{} + + ErrIndexExists = errors.New("index already exists") + ErrFieldExists = errors.New("field already exists") +) + +const ( + heartbeatPrefix = "/heartbeat/" + schemaPrefix = "/schema/" + resizePrefix = "/resize/" + metadataPrefix = "/metadata/" + shardPrefix = "/shard/" + lockPrefix = "/lock/" +) + +type leaseMetadata struct { + started bool +} + +type Etcd struct { + options Options + replicas int + + heartbeatID clientv3.LeaseID + heartbeatCancel context.CancelFunc + + resizeCancel context.CancelFunc + + lm leaseMetadata + + e *embed.Etcd +} + +func NewEtcd(opt Options, replicas int) *Etcd { + e := &Etcd{ + options: opt, + replicas: replicas, + } + return e +} + +// Close implements io.Closer +func (e *Etcd) Close() error { + if e.e != nil { + if e.resizeCancel != nil { + e.resizeCancel() + } + if e.heartbeatCancel != nil { + e.heartbeatCancel() + } + e.e.Server.Stop() + e.e.Close() + <-e.e.Server.StopNotify() + } + + return nil +} + +func parseOptions(opt Options) *embed.Config { + cfg := embed.NewConfig() + cfg.Debug = true + cfg.Name = opt.Name + cfg.Dir = opt.Dir + cfg.InitialClusterToken = opt.ClusterName + cfg.LCUrls = types.MustNewURLs([]string{opt.LClientURL}) + cfg.ACUrls = types.MustNewURLs([]string{opt.AClientURL}) + cfg.LPUrls = types.MustNewURLs([]string{opt.LPeerURL}) + cfg.APUrls = types.MustNewURLs([]string{opt.APeerURL}) + + if opt.InitCluster != "" { + cfg.InitialCluster = opt.InitCluster + cfg.ClusterState = embed.ClusterStateFlagNew + } else { + cfg.InitialCluster = cfg.Name + "=" + opt.APeerURL + } + + if opt.ClusterURL != "" { + cfg.ClusterState = embed.ClusterStateFlagExisting + + cli, err := clientv3.NewFromURL(opt.ClusterURL) + if err != nil { + panic(err) + } + defer cli.Close() + + log.Println("Cluster Members:") + mIDs, mNames, mURLs := memberList(cli) + for i, id := range mIDs { + log.Printf("\tid: %d, name: %s, url: %s\n", id, mNames[i], mURLs[i]) + cfg.InitialCluster += "," + mNames[i] + "=" + mURLs[i] + } + + log.Println("Joining Cluster:") + id, name := memberAdd(cli, opt.APeerURL) + log.Printf("\tid: %d, name: %s\n", id, name) + } + + return cfg +} + +// Start starts etcd and hearbeat +func (e *Etcd) Start(ctx context.Context) (disco.InitialClusterState, error) { + opts := parseOptions(e.options) + state := disco.InitialClusterState(opts.ClusterState) + + etcd, err := embed.StartEtcd(opts) + if err != nil { + return state, errors.Wrap(err, "starting etcd") + } + e.e = etcd + + select { + case <-ctx.Done(): + e.e.Server.Stop() + return state, ctx.Err() + + case err := <-e.e.Err(): + return state, err + + case <-e.e.Server.ReadyNotify(): + return state, e.startHeartbeat() + } +} + +func (e *Etcd) startHeartbeat() error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "startHeartbeat: creates a new client") + } + defer cli.Close() + + heartbeatID, heartbeatFunc, err := e.leaseKeepAlive(e.options.HeartbeatTTL) + if err != nil { + return errors.Wrap(err, "startHeartbeat: creates a new hearbeat") + } + + ctx, heartbeatCancel := context.WithCancel(context.Background()) + key, value := heartbeatPrefix+e.e.Server.ID().String(), disco.ClusterStateStarting + if e.e.Config().ClusterState == embed.ClusterStateFlagExisting { + value = disco.ClusterStateResizing + } + + if _, err := cli.Put(ctx, key, string(value), clientv3.WithLease(heartbeatID)); err != nil { + heartbeatCancel() + return errors.Wrapf(err, "startHeartbeat: puts a key-value (%s, %s) with lease (%v)", key, value, heartbeatID) + } + + e.heartbeatID, e.heartbeatCancel = heartbeatID, heartbeatCancel + go heartbeatFunc(ctx, time.Second) + + return nil +} + +func (e *Etcd) NodeState(ctx context.Context, peerID string) (disco.NodeState, error) { + cli, err := e.client() + if err != nil { + return disco.NodeStateUnknown, errors.Wrap(err, "NodeState: creates a new client") + } + defer cli.Close() + + return e.nodeState(ctx, cli, peerID) +} + +func (e *Etcd) nodeState(ctx context.Context, cli *clientv3.Client, peerID string) (disco.NodeState, error) { + resp, err := cli.Get(ctx, path.Join(resizePrefix, peerID), clientv3.WithCountOnly()) + if err != nil { + return disco.NodeStateUnknown, err + } + if resp.Count > 0 { + return disco.NodeStateResizing, nil + } + + resp, err = cli.Get(ctx, path.Join(heartbeatPrefix, peerID)) + if err != nil { + return disco.NodeStateUnknown, err + } + + if len(resp.Kvs) > 1 { + return disco.NodeStateUnknown, disco.ErrTooManyResults + } + + if len(resp.Kvs) == 0 { + return disco.NodeStateUnknown, disco.ErrNoResults + } + + return disco.NodeState(resp.Kvs[0].Value), nil +} + +func (e *Etcd) NodeStates(ctx context.Context) (map[string]disco.NodeState, error) { + out := make(map[string]disco.NodeState) + + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "NodeStates") + } + defer cli.Close() + + members := e.e.Server.Cluster().Members() + for _, member := range members { + s, err := e.nodeState(ctx, cli, member.ID.String()) + if err != nil { + log.Println("NodeStates get node state", member.ID.String(), err.Error()) + } + + out[member.ID.String()] = s + } + + return out, nil +} + +func (e *Etcd) Started(ctx context.Context) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "Started") + } + defer cli.Close() + + key, value := heartbeatPrefix+e.e.Server.ID().String(), disco.NodeStateStarted + if _, err = cli.Put(ctx, key, string(value), clientv3.WithLease(e.heartbeatID)); err == nil { + e.lm.started = true + } + return err +} + +func (e *Etcd) ID() string { + return e.e.Server.ID().String() +} + +func (e *Etcd) Peers() []*disco.Peer { + var peers []*disco.Peer + for _, member := range e.e.Server.Cluster().Members() { + peers = append(peers, &disco.Peer{ID: member.ID.String(), URL: member.PickPeerURL()}) + } + return peers +} + +func (e *Etcd) IsLeader() bool { + return e.e.Server.Leader() == e.e.Server.ID() +} + +func (e *Etcd) Leader() *disco.Peer { + id := e.e.Server.Leader() + m := e.e.Server.Cluster().Member(id) + return &disco.Peer{ID: id.String(), URL: m.PickPeerURL()} +} + +func (e *Etcd) ClusterState(ctx context.Context) (disco.ClusterState, error) { + if e.e == nil { + return disco.ClusterStateUnknown, nil + } + + cli, err := e.client() + if err != nil { + return disco.ClusterStateUnknown, errors.WithMessage(err, "ClusterState: creates a new client") + } + defer cli.Close() + + var ( + heartbeats int = 0 + resize bool + starting bool + ) + members := e.e.Server.Cluster().Members() + for _, m := range members { + ns, err := e.nodeState(ctx, cli, m.ID.String()) + if err != nil { + log.Println("ClusterState get node state", err.Error()) + continue + } + + heartbeats++ + + if ns == disco.NodeStateStarting { + starting = true + } + + if ns == disco.NodeStateResizing { + resize = true + } + } + + if resize { + return disco.ClusterStateResizing, nil + } + + if starting { + return disco.ClusterStateStarting, nil + } + + if heartbeats < len(members) { + if len(members)-heartbeats >= e.replicas { + return disco.ClusterStateDown, nil + } + + return disco.ClusterStateDegraded, nil + } + + return disco.ClusterStateNormal, nil +} + +func (e *Etcd) Resize(ctx context.Context) (func([]byte) error, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "Resize: creates a new client") + } + defer cli.Close() + + resizeID, resizeFunc, err := e.leaseKeepAlive(e.options.HeartbeatTTL) + if err != nil { + return nil, errors.Wrap(err, "Resize: creates a new hearbeat") + } + + ctx, resizeCancel := context.WithCancel(ctx) + // Check if key exists - maybe we are still resizing + key := path.Join(resizePrefix, e.e.Server.ID().String()) + txnResp, err := cli.Txn(ctx). + If(clientv3util.KeyMissing(key)). + Then(clientv3.OpPut(key, "", clientv3.WithLease(resizeID))). + Commit() + if err != nil { + resizeCancel() + return nil, errors.Wrapf(err, "Resize: txn puts key (%s) with lease (%v)", key, resizeID) + } + + if !txnResp.Succeeded { + resizeCancel() + return nil, errors.Errorf("Resize: key (%s) exists - maybe node (%s) is resizing", key, e.ID()) + } + + e.resizeCancel = resizeCancel + go resizeFunc(ctx, time.Second) + + return func(value []byte) error { + log.Println("Update progress:", key, string(value)) + return e.putKey(ctx, key, string(value), clientv3.WithLease(resizeID)) + }, nil +} + +func (e *Etcd) DoneResize() error { + if e.resizeCancel != nil { + e.resizeCancel() + } + return nil +} + +func (e *Etcd) Watch(ctx context.Context, peerID string, onUpdate func([]byte) error) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "Watch: creates a new client") + } + defer cli.Close() + + key := path.Join(resizePrefix, peerID) + for resp := range cli.Watch(ctx, key) { + if err := resp.Err(); err != nil { + return errors.Wrapf(err, "Watch: key (%s) response", key) + } + + for _, ev := range resp.Events { + switch ev.Type { + case mvccpb.PUT: + if onUpdate != nil && ev.Kv.Value != nil { + if err := onUpdate(ev.Kv.Value); err != nil { + return err + } + } + + case mvccpb.DELETE: + // nothing to watch - key was deleted + return errors.WithMessagef(disco.ErrKeyDeleted, "Watch key %s", key) + } + } + } + + return nil +} + +func (e *Etcd) DeleteNode(ctx context.Context, nodeID string) error { + id, err := types.IDFromString(nodeID) + if err != nil { + return err + } + + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "DeleteNode: creates a new client") + } + defer cli.Close() + + _, err = cli.MemberRemove(ctx, uint64(id)) + if err != nil { + return errors.Wrap(err, "DeleteNode: removes an existing member from the cluster") + } + + return nil +} + +func (e *Etcd) Schema(ctx context.Context) (map[string]*disco.Index, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "Schema: creating client") + } + defer cli.Close() + + keys, vals, err := e.getKey(ctx, cli, schemaPrefix) + if err != nil { + return nil, err + } + + m := make(map[string]*disco.Index) + for i, k := range keys { + tokens := strings.Split(strings.Trim(k, "/"), "/") + // token[0] contains the schemaPrefix + index := tokens[1] + if _, ok := m[index]; !ok { + m[index] = &disco.Index{ + Data: vals[i], + Fields: make(map[string][]byte), + } + } + flds := m[index].Fields + + if len(tokens) > 2 { + field := tokens[2] + flds[field] = vals[i] + } + } + + return m, nil +} + +func (e *Etcd) Metadata(ctx context.Context, peerID string) ([]byte, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "Metadata") + } + defer cli.Close() + + resp, err := cli.Get(ctx, path.Join(metadataPrefix, peerID)) + if err != nil { + return nil, err + } + + if len(resp.Kvs) > 1 { + return nil, disco.ErrTooManyResults + } + + if len(resp.Kvs) == 0 { + return nil, disco.ErrNoResults + } + + return resp.Kvs[0].Value, nil +} + +func (e *Etcd) SetMetadata(ctx context.Context, metadata []byte) error { + err := e.putKey(ctx, path.Join(metadataPrefix, + e.e.Server.ID().String()), + string(metadata), + ) + if err != nil { + return errors.Wrap(err, "SetMetadata") + } + + return nil +} + +func (e *Etcd) CreateIndex(ctx context.Context, name string, val []byte) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "CreateIndex: creating client") + } + defer cli.Close() + + key := schemaPrefix + name + + // Set up Op to write index value as bytes. + op := clientv3.OpPut(key, "") + op.WithValueBytes(val) + + // Check for key existence, and execute Op within a transaction. + resp, err := cli.KV.Txn(ctx). + If(clientv3util.KeyMissing(key)). + Then(op). + Commit() + if err != nil { + return errors.Wrap(err, "executing transaction") + } + + if !resp.Succeeded { + return ErrIndexExists + } + + return nil +} + +func (e *Etcd) Index(ctx context.Context, name string) ([]byte, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "Index: creating client") + } + defer cli.Close() + + return e.getKeyBytes(ctx, cli, schemaPrefix+name) +} + +func (e *Etcd) DeleteIndex(ctx context.Context, name string) error { + // Delete any fields below the index path. + if err := e.delKey(ctx, schemaPrefix+name+"/", true); err != nil { + return errors.Wrap(err, "deleting index fields") + } + // Delete the index. + return e.delKey(ctx, schemaPrefix+name, false) +} + +func (e *Etcd) Field(ctx context.Context, indexName string, name string) ([]byte, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "GetField: creating client") + } + defer cli.Close() + + key := schemaPrefix + indexName + "/" + name + return e.getKeyBytes(ctx, cli, key) +} + +func (e *Etcd) CreateField(ctx context.Context, indexName string, name string, val []byte) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "CreateIndex: creating client") + } + defer cli.Close() + + key := schemaPrefix + indexName + "/" + name + + // Set up Op to write field value as bytes. + op := clientv3.OpPut(key, "") + op.WithValueBytes(val) + + // Check for key existence, and execute Op within a transaction. + resp, err := cli.KV.Txn(ctx). + If(clientv3util.KeyMissing(key)). + Then(op). + Commit() + if err != nil { + return errors.Wrap(err, "executing transaction") + } + + if !resp.Succeeded { + return ErrFieldExists + } + + return nil +} + +func (e *Etcd) DeleteField(ctx context.Context, indexname string, name string) error { + return e.delKey(ctx, schemaPrefix+indexname+"/"+name, false) +} + +func (e *Etcd) putKey(ctx context.Context, key, val string, opts ...clientv3.OpOption) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "putKey: creates a new client") + } + defer cli.Close() + + if _, err := cli.KV.Put(ctx, key, val, opts...); err != nil { + return errors.Wrapf(err, "putKey: Put(%s, %s)", key, val) + } + + return nil +} + +func (e *Etcd) getKeyBytes(ctx context.Context, cli *clientv3.Client, key string) ([]byte, error) { + // Get the current value for the key. + resp, err := cli.Get(ctx, key) + if err != nil { + return nil, err + } + + // TODO: consider returning a "key does not exist" error instead of (nil, nil) + if len(resp.Kvs) == 0 { + return nil, nil + } + + return resp.Kvs[0].Value, nil +} + +func (e *Etcd) getKey(ctx context.Context, cli *clientv3.Client, key string) ([]string, [][]byte, error) { + resp, err := cli.KV.Txn(ctx). + If(clientv3.Compare(clientv3.Version(key), ">", -1)). + Then(clientv3.OpGet(key, clientv3.WithPrefix())). + Commit() + if err != nil { + return nil, nil, err + } + + if !resp.Succeeded { + return nil, nil, fmt.Errorf("key %s does not exist", key) + } + + var ( + keys []string + values [][]byte + ) + + for _, r := range resp.Responses { + for _, kv := range r.GetResponseRange().Kvs { + keys = append(keys, string(kv.Key)) + values = append(values, kv.Value) + } + } + + return keys, values, nil +} + +func (e *Etcd) delKey(ctx context.Context, key string, withPrefix bool) error { + cli, err := clientv3.NewFromURLs(e.e.Server.Cluster().ClientURLs()) + if err != nil { + return errors.Wrap(err, "delKey") + } + defer cli.Close() + + var opts []clientv3.OpOption + if withPrefix { + opts = append(opts, clientv3.WithPrefix()) + } + + _, err = cli.KV.Txn(ctx). + If(clientv3.Compare(clientv3.Version(key), ">", -1)). + Then(clientv3.OpDelete(key, opts...)). + Commit() + + return err +} + +func (e *Etcd) leaseKeepAlive(ttl int64) (clientv3.LeaseID, func(context.Context, time.Duration), error) { + cli, err := e.client() + if err != nil { + return 0, nil, errors.Wrap(err, "leaseKeepAlive: creates a new client") + } + defer cli.Close() + + leaseResp, err := cli.Grant(context.TODO(), ttl) + if err != nil { + return 0, nil, errors.Wrapf(err, "leaseKeepAlive: creates a new lease (TTL: %d)", ttl) + } + + keepaliveFunc := func(ctx context.Context, tick time.Duration) { + ticker := time.NewTicker(tick) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Printf("leaseKeepAlive: %v\n", ctx.Err()) + return + + case <-ticker.C: + if cli, err := e.client(); err != nil { + log.Printf("leaseKeepAlive: creates a new client: %v\n", err) + } else { + if _, err = cli.KeepAliveOnce(ctx, leaseResp.ID); err != nil { + log.Printf("leaseKeepAlive: renews the lease (ID: %v): %v\n", leaseResp.ID, err) + } + cli.Close() + } + } + } + } + + return leaseResp.ID, keepaliveFunc, nil +} + +func (e *Etcd) client() (*clientv3.Client, error) { + urls := e.e.Server.Cluster().ClientURLs() + cli, err := clientv3.NewFromURLs(urls) + if err != nil { + return nil, errors.Wrapf(err, "creates a new etcd client from URLs (%v)", urls) + } + return cli, nil +} + +func memberList(cli *clientv3.Client) (ids []uint64, names []string, urls []string) { + ml, err := cli.MemberList(context.TODO()) + if err != nil { + panic(err) + } + n := len(ml.Members) + ids = make([]uint64, n) + names = make([]string, n) + urls = make([]string, n) + + for i, m := range ml.Members { + ids[i], names[i], urls[i] = m.ID, m.Name, m.PeerURLs[0] + } + return +} + +func memberAdd(cli *clientv3.Client, peerURL string) (id uint64, name string) { + ma, err := cli.MemberAdd(context.TODO(), []string{peerURL}) + if err != nil { + return 0, "" + } + + return ma.Member.ID, ma.Member.Name +} + +// Shards implements the Sharder interface. +func (e *Etcd) Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "Shards: creating client") + } + defer cli.Close() + + return e.shards(ctx, cli, index, field) +} + +func (e *Etcd) shards(ctx context.Context, cli *clientv3.Client, index, field string) (*roaring.Bitmap, error) { + key := path.Join(shardPrefix, index, field) + + // Get the current shards for the field. + resp, err := cli.Get(ctx, key) + if err != nil { + return nil, err + } + + bm := roaring.NewBitmap() + + if len(resp.Kvs) == 0 { + return bm, nil + } + + bytes := resp.Kvs[0].Value + if err = bm.UnmarshalBinary(bytes); err != nil { + return nil, errors.Wrap(err, "unmarshalling shards") + } + + return bm, nil +} + +// AddShards implements the Sharder interface. +func (e *Etcd) AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error) { + cli, err := e.client() + if err != nil { + return nil, errors.Wrap(err, "AddShards: creating client") + } + defer cli.Close() + + key := path.Join(shardPrefix, index, field) + + // This tended to add more overhead than it saved. + // // Read shards outside of a lock just to check if shard is already included. + // // If shard is already included, no-op. + // if currentShards, err := e.shards(ctx, cli, index, field); err != nil { + // return nil, errors.Wrap(err, "reading shards") + // } else if currentShards.Count() == currentShards.Union(shards).Count() { + // return currentShards, nil + // } + + // Create a session to acquire a lock. + sess, _ := concurrency.NewSession(cli) + defer sess.Close() + + muKey := path.Join(lockPrefix, index, field) + mu := concurrency.NewMutex(sess, muKey) + + // Acquire lock (or wait to have it). + if err := mu.Lock(ctx); err != nil { + return nil, errors.Wrap(err, "acquiring lock") + } + + // Read shards within lock. + globalShards, err := e.shards(ctx, cli, index, field) + if err != nil { + return nil, errors.Wrap(err, "reading shards") + } + + // Union shard into shards. + globalShards.UnionInPlace(shards) + + // Write shards to etcd. + var buf bytes.Buffer + if _, err := globalShards.WriteTo(&buf); err != nil { + return nil, errors.Wrap(err, "writing shards to bytes buffer") + } + + op := clientv3.OpPut(key, "") + op.WithValueBytes(buf.Bytes()) + + if _, err := cli.Do(ctx, op); err != nil { + return nil, errors.Wrap(err, "doing op") + } + + // Release lock. + if err := mu.Unlock(ctx); err != nil { + return nil, errors.Wrap(err, "releasing lock") + } + + return globalShards, nil +} + +// AddShard implements the Sharder interface. +func (e *Etcd) AddShard(ctx context.Context, index, field string, shard uint64) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "AddShard: creating client") + } + defer cli.Close() + + key := path.Join(shardPrefix, index, field) + + // Read shards outside of a lock just to check if shard is already included. + // If shard is already included, no-op. + if shards, err := e.shards(ctx, cli, index, field); err != nil { + return errors.Wrap(err, "reading shards") + } else if shards.Contains(shard) { + return nil + } + + // According to the previous read, shard is not yet included in shards. So + // we will acquire a distributed lock, read shards again (in case it has + // been updated since we last read it), add shard to shards, and finally + // write shards to etcd. + + // Create a session to acquire a lock. + sess, _ := concurrency.NewSession(cli) + defer sess.Close() + + muKey := path.Join(lockPrefix, index, field) + mu := concurrency.NewMutex(sess, muKey) + + // Acquire lock (or wait to have it). + if err := mu.Lock(ctx); err != nil { + return errors.Wrap(err, "acquiring lock") + } + + // Read shards again (within lock). + shards, err := e.shards(ctx, cli, index, field) + if err != nil { + return errors.Wrap(err, "reading shards") + } + + if shards.Contains(shard) { + return nil + } + + // Union shard into shards. + shards.UnionInPlace(roaring.NewBitmap(shard)) + + // Write shards to etcd. + var buf bytes.Buffer + if _, err := shards.WriteTo(&buf); err != nil { + return errors.Wrap(err, "writing shards to bytes buffer") + } + + op := clientv3.OpPut(key, "") + op.WithValueBytes(buf.Bytes()) + + if _, err := cli.Do(ctx, op); err != nil { + return errors.Wrap(err, "doing op") + } + + // Release lock. + if err := mu.Unlock(ctx); err != nil { + return errors.Wrap(err, "releasing lock") + } + + return nil +} + +// RemoveShard implements the Sharder interface. +func (e *Etcd) RemoveShard(ctx context.Context, index, field string, shard uint64) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "RemoveShard: creating client") + } + defer cli.Close() + + key := path.Join(shardPrefix, index, field) + + // Read shards outside of a lock just to check if shard is already excluded. + // If shard is already excluded, no-op. + if shards, err := e.shards(ctx, cli, index, field); err != nil { + return errors.Wrap(err, "reading shards") + } else if !shards.Contains(shard) { + return nil + } + + // According to the previous read, shard is included in shards. So + // we will acquire a distributed lock, read shards again (in case it has + // been updated since we last read it), remove shard from shards, and finally + // write shards to etcd. + + // Create a session to acquire a lock. + sess, _ := concurrency.NewSession(cli) + defer sess.Close() + + muKey := path.Join(lockPrefix, index, field) + mu := concurrency.NewMutex(sess, muKey) + + // Acquire lock (or wait to have it). + if err := mu.Lock(ctx); err != nil { + return errors.Wrap(err, "acquiring lock") + } + + // Read shards again (within lock). + shards, err := e.shards(ctx, cli, index, field) + if err != nil { + return errors.Wrap(err, "reading shards") + } + + if !shards.Contains(shard) { + return nil + } + + // Remove shard from shards. + if _, err := shards.RemoveN(shard); err != nil { + return errors.Wrap(err, "removing shard") + } + + // If this is removing the last bit from the shards bitmap, then instead of + // writing an empty bitmap, just delete the key. + if shards.Count() == 0 { + _, err := cli.Delete(ctx, key) + return err + } + + // Write shards to etcd. + var buf bytes.Buffer + if _, err := shards.WriteTo(&buf); err != nil { + return errors.Wrap(err, "writing shards to bytes buffer") + } + + op := clientv3.OpPut(key, "") + op.WithValueBytes(buf.Bytes()) + + if _, err := cli.Do(ctx, op); err != nil { + return errors.Wrap(err, "doing op") + } + + // Release lock. + if err := mu.Unlock(ctx); err != nil { + return errors.Wrap(err, "releasing lock") + } + + return nil +} diff --git a/event.go b/event.go index b27bd1bf6..39e688f07 100644 --- a/event.go +++ b/event.go @@ -14,6 +14,8 @@ package pilosa +import "github.com/pilosa/pilosa/v2/topology" + // NodeEventType are the types of node events. type NodeEventType int @@ -27,5 +29,5 @@ const ( // NodeEvent is a single event related to node activity in the cluster. type NodeEvent struct { Event NodeEventType - Node *Node + Node *topology.Node } diff --git a/executor.go b/executor.go index 6536c17f6..76eadf161 100644 --- a/executor.go +++ b/executor.go @@ -30,6 +30,7 @@ import ( "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/testhook" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" ) @@ -51,7 +52,7 @@ type executor struct { Holder *Holder // Local hostname & cluster configuration. - Node *Node + Node *topology.Node Cluster *cluster // Client used for remote requests. @@ -5102,10 +5103,10 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, qcx *Qcx, index strin } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.nodes).FilterID(e.Node.ID) + nodes := topology.Nodes(e.Cluster.nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { - go func(node *Node) { + go func(node *topology.Node) { _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) resp <- err }(node) @@ -5214,10 +5215,10 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, qcx *Qcx, index s } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.nodes).FilterID(e.Node.ID) + nodes := topology.Nodes(e.Cluster.nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { - go func(node *Node) { + go func(node *topology.Node) { _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil) resp <- err }(node) @@ -5266,10 +5267,10 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, qcx *Qcx, index st } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.nodes).FilterID(e.Node.ID) + nodes := topology.Nodes(e.Cluster.nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { - go func(node *Node) { + go func(node *topology.Node) { _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) resp <- err }(node) @@ -5286,7 +5287,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, qcx *Qcx, index st } // remoteExec executes a PQL query remotely for a set of shards on a node. -func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer +func (e *executor) remoteExec(ctx context.Context, node *topology.Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec") defer span.Finish() @@ -5308,13 +5309,13 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q * // shardsByNode returns a mapping of nodes to shards. // Returns errShardUnavailable if a shard cannot be allocated to a node. -func (e *executor) shardsByNode(nodes []*Node, index string, shards []uint64) (map[*Node][]uint64, error) { - m := make(map[*Node][]uint64) +func (e *executor) shardsByNode(nodes []*topology.Node, index string, shards []uint64) (map[*topology.Node][]uint64, error) { + m := make(map[*topology.Node][]uint64) loop: for _, shard := range shards { for _, node := range e.Cluster.ShardNodes(index, shard) { - if Nodes(nodes).Contains(node) { + if topology.Nodes(nodes).Contains(node) { m[node] = append(m[node], shard) continue loop } @@ -5342,11 +5343,11 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, // // However, if this request is being sent from the coordinator then all // processing should be done locally so we start with just the local node. - var nodes []*Node + var nodes []*topology.Node if !opt.Remote { - nodes = Nodes(e.Cluster.nodes).Clone() + nodes = topology.Nodes(e.Cluster.nodes).Clone() } else { - nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} + nodes = []*topology.Node{e.Cluster.nodeByID(e.Node.ID)} } // Start mapping across all primary owners. @@ -5367,7 +5368,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, if resp.err != nil { // Filter out unavailable nodes. - nodes = Nodes(nodes).Filter(resp.node) + nodes = topology.Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. if err := e.mapper(ctx, cancel, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { @@ -5434,7 +5435,7 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row { return newRows } -func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { +func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch chan mapResponse, nodes []*topology.Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper") defer span.Finish() done := ctx.Done() @@ -5447,7 +5448,7 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha // Execute each node in a separate goroutine. for n, nodeShards := range m { - go func(n *Node, nodeShards []uint64) { + go func(n *topology.Node, nodeShards []uint64) { resp := mapResponse{node: n, shards: nodeShards} // Send local shards to mapper, otherwise remote exec. @@ -6831,7 +6832,7 @@ type mapFunc func(ctx context.Context, shard uint64) (_ interface{}, err error) type reduceFunc func(ctx context.Context, prev, v interface{}) interface{} type mapResponse struct { - node *Node + node *topology.Node shards []uint64 result interface{} diff --git a/fragment.go b/fragment.go index aa93dcc9a..20cdcf380 100644 --- a/fragment.go +++ b/fragment.go @@ -42,11 +42,13 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/v2/internal" "github.com/pilosa/pilosa/v2/logger" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/testhook" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" ) @@ -3528,7 +3530,7 @@ func (h *blockHasher) WriteValue(v uint64) { type fragmentSyncer struct { Fragment *fragment - Node *Node + Node *topology.Node Cluster *cluster // FieldType helps determine which method of syncing to use. @@ -3720,7 +3722,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { f := s.Fragment // Read pairs from each remote block. - var uris []*URI + var uris []*pnet.URI var pairSets []pairSet for _, node := range s.Cluster.shardNodes(f.index(), f.shard) { if s.Node.ID == node.ID { diff --git a/go.mod b/go.mod index f0d161c95..ce37e4910 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.2 + github.com/google/uuid v1.1.4 // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 @@ -46,18 +47,19 @@ require ( github.com/uber/jaeger-lib v2.2.0+incompatible // indirect github.com/zeebo/blake3 v0.0.4 go.etcd.io/bbolt v1.3.5 + go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect - golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 + sigs.k8s.io/yaml v1.2.0 // indirect vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index b9d5e44d7..6af49c800 100644 --- a/go.sum +++ b/go.sum @@ -43,25 +43,39 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -88,6 +102,8 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -114,10 +130,14 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0= +github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -127,11 +147,19 @@ github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6Ylu github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -165,8 +193,12 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -175,6 +207,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -190,6 +223,8 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= @@ -204,7 +239,11 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= @@ -213,6 +252,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= @@ -267,11 +307,13 @@ github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21 github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -280,10 +322,12 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -301,6 +345,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= @@ -308,6 +354,8 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQG github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= @@ -317,13 +365,20 @@ github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU= github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b h1:5makfKENOTVu2bNoHzSqwwz+g70ivWLSnExzd33/2bI= +go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -371,6 +426,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -397,6 +453,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -411,7 +468,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -463,6 +522,7 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -477,6 +537,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -501,5 +562,8 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible h1:GWnLrAdetgJM0Co5bwwczO49iFZBSInpyGAT77BP9Y0= vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/gossip/gossip.go b/gossip/gossip.go index d7b6377e7..e4f9110ef 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -31,8 +31,10 @@ import ( "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/logger" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/toml" + "github.com/pilosa/pilosa/v2/topology" "github.com/pkg/errors" ) @@ -79,21 +81,21 @@ func (g *memberSet) Open() (err error) { RetransmitMult: 3, } - var uris = make([]*pilosa.URI, len(g.config.gossipSeeds)) + var uris = make([]*pnet.URI, len(g.config.gossipSeeds)) for i, addr := range g.config.gossipSeeds { - uris[i], err = pilosa.NewURIFromAddress(addr) + uris[i], err = pnet.NewURIFromAddress(addr) if err != nil { return fmt.Errorf("new uri from address: %s", err) } } - var nodes = make([]*pilosa.Node, len(uris)) + var nodes = make([]*topology.Node, len(uris)) for i, uri := range uris { - nodes[i] = &pilosa.Node{URI: *uri} + nodes[i] = &topology.Node{URI: *uri} } g.mu.RLock() - err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings()) + err = g.joinWithRetry(pnet.URIs(topology.Nodes(nodes).URIs()).HostPortStrings()) g.mu.RUnlock() if err != nil { return errors.Wrap(err, "joinWithRetry") @@ -447,7 +449,7 @@ func (g *eventReceiver) listen() { } // Get the node from the event.Node meta data. - var n pilosa.Node + var n topology.Node if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil { panic("failed to unmarshal event node meta into node") } @@ -470,7 +472,7 @@ func (g *eventReceiver) listen() { type Transport struct { //memberlist.Transport net *memberlist.NetTransport - URI *pilosa.URI + URI *pnet.URI } // NewTransport returns a NetTransport based on the given host and port. @@ -490,7 +492,7 @@ func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) return nil, fmt.Errorf("new transport: %s", err) } - uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort())) + uri, err := pnet.NewURIFromHostPort(host, uint16(net.GetAutoBindPort())) if err != nil { return nil, fmt.Errorf("new uri from host port: %s", err) } diff --git a/holder.go b/holder.go index 256871058..f5145ba31 100644 --- a/holder.go +++ b/holder.go @@ -36,6 +36,7 @@ import ( "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/testhook" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" @@ -1304,7 +1305,7 @@ type holderSyncer struct { Holder *Holder - Node *Node + Node *topology.Node Cluster *cluster // Translation sync handling. @@ -1416,7 +1417,7 @@ func (s *holderSyncer) syncIndex(index string) error { s.Stats.CountWithCustomTags(MetricColumnAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag}) // Sync with every other host. - for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) { + for _, node := range topology.Nodes(s.Cluster.nodes).FilterID(s.Node.ID) { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. m, err := s.Cluster.InternalClient.ColumnAttrDiff(ctx, &node.URI, index, blks) @@ -1463,7 +1464,7 @@ func (s *holderSyncer) syncField(index, name string) error { s.Stats.CountWithCustomTags(MetricRowAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag, fieldTag}) // Sync with every other host. - for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) { + for _, node := range topology.Nodes(s.Cluster.nodes).FilterID(s.Node.ID) { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. m, err := s.Cluster.InternalClient.RowAttrDiff(ctx, &node.URI, index, name, blks) @@ -1669,8 +1670,8 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error { } for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ { partitionNodes := s.Cluster.partitionNodes(partitionID) - isPrimary := partitionNodes[0].ID == node.ID // remote is primary? - isReplica := Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica? + isPrimary := partitionNodes[0].ID == node.ID // remote is primary? + isReplica := topology.Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica? if !isPrimary || !isReplica { continue } @@ -1797,7 +1798,7 @@ func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) { // holderCleaner removes fragments and data files that are no longer used. type holderCleaner struct { - Node *Node + Node *topology.Node Holder *Holder Cluster *cluster diff --git a/http/client.go b/http/client.go index 4f37d7a36..7eb5d025f 100644 --- a/http/client.go +++ b/http/client.go @@ -30,13 +30,15 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/encoding/proto" + pnet "github.com/pilosa/pilosa/v2/net" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" ) // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { - defaultURI *pilosa.URI + defaultURI *pnet.URI serializer pilosa.Serializer // The client to use for HTTP communication. @@ -49,7 +51,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, return nil, pilosa.ErrHostRequired } - uri, err := pilosa.NewURIFromAddress(host) + uri, err := pnet.NewURIFromAddress(host) if err != nil { return nil, errors.Wrap(err, "getting URI") } @@ -58,7 +60,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, return client, nil } -func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { +func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client) *InternalClient { return &InternalClient{ defaultURI: defaultURI, serializer: proto.Serializer{}, @@ -133,7 +135,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error return rsp.Indexes, nil } -func (c *InternalClient) PostSchema(ctx context.Context, uri *pilosa.URI, s *pilosa.Schema, remote bool) error { +func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilosa.Schema, remote bool) error { u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote)) buf, err := json.Marshal(s) if err != nil { @@ -207,7 +209,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo } // FragmentNodes returns a list of nodes that own a shard. -func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) { +func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentNodes") defer span.Finish() @@ -231,7 +233,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard } defer resp.Body.Close() - var a []*pilosa.Node + var a []*topology.Node if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { return nil, fmt.Errorf("json decode: %s", err) } @@ -239,7 +241,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard } // Nodes returns a list of all nodes. -func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) { +func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Nodes") defer span.Finish() @@ -262,7 +264,7 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) { } defer resp.Body.Close() - var a []*pilosa.Node + var a []*topology.Node if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { return nil, fmt.Errorf("json decode: %s", err) } @@ -277,7 +279,7 @@ func (c *InternalClient) Query(ctx context.Context, index string, queryRequest * } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "QueryNode") defer span.Finish() @@ -368,7 +370,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard return nil } -func getCoordinatorNode(nodes []*pilosa.Node) *pilosa.Node { +func getCoordinatorNode(nodes []*topology.Node) *topology.Node { for _, node := range nodes { if node.IsCoordinator { return node @@ -482,7 +484,7 @@ func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, } // importNode sends a pre-marshaled import request to a node. -func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error { +func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.importNode") defer span.Finish() @@ -664,7 +666,7 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui // ImportRoaring does fast import of raw bits in roaring format (pilosa or // official format, see API.ImportRoaring). -func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error { +func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring") defer span.Finish() @@ -718,7 +720,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind } // ImportColumnAttrs does bulk import of column attrs -func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pilosa.URI, index string, req *pilosa.ImportColumnAttrsRequest) error { +func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *pilosa.ImportColumnAttrsRequest) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring") defer span.Finish() @@ -802,7 +804,7 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sha } // exportNode copies a CSV export from a node to w. -func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error { +func (c *InternalClient) exportNodeCSV(ctx context.Context, node *topology.Node, index, field string, shard uint64, w io.Writer) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.exportNodeCSV") defer span.Finish() @@ -840,11 +842,11 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i // RetrieveShardFromURI returns a ReadCloser which contains the data of the // specified shard from the specified node. Caller *must* close the returned // ReadCloser or risk leaking goroutines/tcp connections. -func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { +func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI") defer span.Finish() - node := &pilosa.Node{ + node := &topology.Node{ URI: uri, } @@ -961,7 +963,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) { +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentBlocks") defer span.Finish() @@ -1005,7 +1007,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in } // BlockData returns row/column id pairs for a block. -func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.BlockData") defer span.Finish() @@ -1054,7 +1056,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, } // ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ColumnAttrDiff") defer span.Finish() @@ -1094,7 +1096,7 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in } // RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RowAttrDiff") defer span.Finish() @@ -1137,7 +1139,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index } // SendMessage posts a message synchronously. -func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error { +func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.SendMessage") defer span.Finish() @@ -1163,7 +1165,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [ // TranslateKeysNode function is mainly called to translate keys from coordinator node. // If coordinator node returns 404 error the function wraps it with pilosa.ErrTranslatingKeyNotFound. -func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string, writable bool) ([]uint64, error) { +func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode") defer span.Finish() @@ -1218,7 +1220,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, } // TranslateIDsNode sends an id translation request to a specific node. -func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) { +func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) { span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode") defer span.Finish() @@ -1269,7 +1271,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, } // GetNodeUsage retrieves the size-on-disk information for the specified node. -func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pilosa.URI) (map[string]pilosa.NodeUsage, error) { +func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]pilosa.NodeUsage, error) { u := uri.Path("/ui/usage?remote=true") req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -1300,7 +1302,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pilosa.URI) (map } // GetPastQueries retrieves the query history log for the specified node. -func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pilosa.URI) ([]pilosa.PastQueryStatus, error) { +func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]pilosa.PastQueryStatus, error) { u := uri.Path("/query-history?remote=true") req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -1330,7 +1332,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pilosa.URI) ([ return queries, nil } -func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pilosa.URI, index string, keys ...string) (transMap map[string]uint64, err error) { +func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (transMap map[string]uint64, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FindIndexKeysNode") defer span.Finish() @@ -1379,7 +1381,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pilosa.URI, return transMap, nil } -func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pilosa.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) { +func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FindFieldKeysNode") defer span.Finish() @@ -1427,7 +1429,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pilosa.URI, return transMap, nil } -func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pilosa.URI, index string, keys ...string) (transMap map[string]uint64, err error) { +func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (transMap map[string]uint64, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndexKeysNode") defer span.Finish() @@ -1476,7 +1478,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pilosa.UR return transMap, nil } -func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pilosa.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) { +func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldKeysNode") defer span.Finish() @@ -1922,7 +1924,7 @@ func pos(rowID, columnID uint64) uint64 { return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth) } -func uriPathToURL(uri *pilosa.URI, path string) url.URL { +func uriPathToURL(uri *pnet.URI, path string) url.URL { return url.URL{ Scheme: uri.Scheme, Host: uri.HostPort(), @@ -1930,7 +1932,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL { } } -func nodePathToURL(node *pilosa.Node, path string) url.URL { +func nodePathToURL(node *topology.Node, path string) url.URL { return url.URL{ Scheme: node.URI.Scheme, Host: node.URI.HostPort(), @@ -1941,11 +1943,11 @@ func nodePathToURL(node *pilosa.Node, path string) url.URL { // RetrieveTranslatePartitionFromURI returns a ReadCloser which contains the data of the // specified translate partition from the specified node. Caller *must* close the returned // ReadCloser or risk leaking goroutines/tcp connections. -func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pilosa.URI) (io.ReadCloser, error) { +func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveTranslatePartitionFromURI") defer span.Finish() - node := &pilosa.Node{ + node := &topology.Node{ URI: uri, } @@ -1974,7 +1976,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, return resp.Body, nil } -func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pilosa.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error { +func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys") defer span.Finish() @@ -2006,7 +2008,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pilosa.URI, i return nil } -func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pilosa.URI, index, field string, remote bool, rddbdata io.Reader) error { +func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportFieldKeys") defer span.Finish() diff --git a/http/handler.go b/http/handler.go index bc2fbb1ab..efc993f45 100644 --- a/http/handler.go +++ b/http/handler.go @@ -44,6 +44,7 @@ import ( "github.com/pilosa/pilosa/v2/encoding/proto" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/topology" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -793,10 +794,10 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - LocalID string `json:"localID"` - ClusterName string `json:"clusterName"` + State string `json:"state"` + Nodes []*topology.Node `json:"nodes"` + LocalID string `json:"localID"` + ClusterName string `json:"clusterName"` } func hash(s string) string { @@ -2053,8 +2054,8 @@ type setCoordinatorRequest struct { } type setCoordinatorResponse struct { - Old *pilosa.Node `json:"old"` - New *pilosa.Node `json:"new"` + Old *topology.Node `json:"old"` + New *topology.Node `json:"new"` } // handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. @@ -2095,7 +2096,7 @@ type removeNodeRequest struct { } type removeNodeResponse struct { - Remove *pilosa.Node `json:"remove"` + Remove *topology.Node `json:"remove"` } // handlePostClusterResizeAbort handles POST /cluster/resize/abort request. diff --git a/uri.go b/net/uri.go similarity index 94% rename from uri.go rename to net/uri.go index b1030f8ce..d83f3b456 100644 --- a/uri.go +++ b/net/uri.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package net import ( "encoding/json" @@ -54,6 +54,11 @@ func (u *URI) URL() url.URL { return url.URL{Scheme: u.Scheme, Host: net.JoinHostPort(u.Host, strconv.Itoa(int(u.Port)))} } +// DefaultURI creates and returns the default URI. +func DefaultURI() *URI { + return defaultURI() +} + // defaultURI creates and returns the default URI. func defaultURI() *URI { return &URI{ @@ -79,7 +84,7 @@ func (u URIs) HostPortStrings() []string { // NewURIFromHostPort returns a URI with specified host and port. func NewURIFromHostPort(host string, port uint16) (*URI, error) { uri := defaultURI() - err := uri.setHost(host) + err := uri.SetHost(host) if err != nil { return nil, errors.Wrap(err, "setting uri host") } @@ -92,8 +97,8 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// setScheme sets the scheme of this URI. -func (u *URI) setScheme(scheme string) error { +// SetScheme sets the scheme of this URI. +func (u *URI) SetScheme(scheme string) error { m := schemeRegexp.FindStringSubmatch(scheme) if m == nil { return errors.New("invalid scheme") @@ -102,8 +107,8 @@ func (u *URI) setScheme(scheme string) error { return nil } -// setHost sets the host of this URI. -func (u *URI) setHost(host string) error { +// SetHost sets the host of this URI. +func (u *URI) SetHost(host string) error { m := hostRegexp.FindStringSubmatch(host) if m == nil { return errors.New("invalid host") diff --git a/uri_internal_test.go b/net/uri_internal_test.go similarity index 96% rename from uri_internal_test.go rename to net/uri_internal_test.go index cb59c75c6..3cedc30ed 100644 --- a/uri_internal_test.go +++ b/net/uri_internal_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package net import "testing" @@ -79,7 +79,7 @@ func TestURIPath(t *testing.T) { func TestSetScheme(t *testing.T) { uri := defaultURI() target := "fun" - err := uri.setScheme(target) + err := uri.SetScheme(target) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func TestSetScheme(t *testing.T) { func TestSetHost(t *testing.T) { uri := defaultURI() target := "10.20.30.40" - err := uri.setHost(target) + err := uri.SetHost(target) if err != nil { t.Fatal(err) } @@ -111,7 +111,7 @@ func TestSetPort(t *testing.T) { func TestSetInvalidScheme(t *testing.T) { uri := defaultURI() - err := uri.setScheme("?invalid") + err := uri.SetScheme("?invalid") if err == nil { t.Fatalf("Should have failed") } @@ -119,7 +119,7 @@ func TestSetInvalidScheme(t *testing.T) { func TestSetInvalidHost(t *testing.T) { uri := defaultURI() - err := uri.setHost("index?.pilosa.com") + err := uri.SetHost("index?.pilosa.com") if err == nil { t.Fatalf("Should have failed") } diff --git a/pilosa.go b/pilosa.go index a05228087..edb0240d1 100644 --- a/pilosa.go +++ b/pilosa.go @@ -19,6 +19,7 @@ import ( "regexp" "time" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pkg/errors" ) @@ -208,9 +209,9 @@ func timestamp() int64 { // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. -func AddressWithDefaults(addr string) (*URI, error) { +func AddressWithDefaults(addr string) (*pnet.URI, error) { if addr == "" { - return defaultURI(), nil + return pnet.DefaultURI(), nil } - return NewURIFromAddress(addr) + return pnet.NewURIFromAddress(addr) } diff --git a/server.go b/server.go index 2fb74e560..fb37f3a0d 100644 --- a/server.go +++ b/server.go @@ -30,9 +30,11 @@ import ( uuid "github.com/satori/go.uuid" "github.com/pilosa/pilosa/v2/logger" + pnet "github.com/pilosa/pilosa/v2/net" rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/topology" "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -68,8 +70,8 @@ type Server struct { // nolint: maligned snapshotQueue SnapshotQueue nodeID string - uri URI - grpcURI URI + uri pnet.URI + grpcURI pnet.URI antiEntropyInterval time.Duration metricInterval time.Duration diagnosticInterval time.Duration @@ -248,7 +250,7 @@ func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption { // OptServerURI is a functional option on Server // used to set the server URI. -func OptServerURI(uri *URI) ServerOption { +func OptServerURI(uri *pnet.URI) ServerOption { return func(s *Server) error { s.uri = *uri return nil @@ -257,7 +259,7 @@ func OptServerURI(uri *URI) ServerOption { // OptServerGRPCURI is a functional option on Server // used to set the server gRPC URI. -func OptServerGRPCURI(uri *URI) ServerOption { +func OptServerGRPCURI(uri *pnet.URI) ServerOption { return func(s *Server) error { s.grpcURI = *uri return nil @@ -459,7 +461,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { } // Set Cluster Node. - node := &Node{ + node := &topology.Node{ ID: s.nodeID, URI: s.uri, GRPCURI: s.grpcURI, @@ -499,7 +501,7 @@ func (s *Server) InternalClient() InternalClient { return s.defaultClient } -func (s *Server) GRPCURI() URI { +func (s *Server) GRPCURI() pnet.URI { return s.grpcURI } @@ -905,7 +907,7 @@ func (s *Server) SendAsync(m Message) error { } // SendTo represents an implementation of Broadcaster. -func (s *Server) SendTo(to *Node, m Message) error { +func (s *Server) SendTo(to *topology.Node, m Message) error { msg, err := s.serializer.Marshal(m) if err != nil { return fmt.Errorf("marshaling message: %v", err) @@ -916,7 +918,7 @@ func (s *Server) SendTo(to *Node, m Message) error { // node returns the pilosa.node object. It is used by membership protocols to // get this node's name(ID), location(URI), and coordinator status. -func (s *Server) node() Node { +func (s *Server) node() topology.Node { return *s.cluster.Node } diff --git a/server/server.go b/server/server.go index 291c986e1..5fab8ae3c 100644 --- a/server/server.go +++ b/server/server.go @@ -46,6 +46,7 @@ import ( "github.com/pilosa/pilosa/v2/gossip" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/logger" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/prometheus" "github.com/pilosa/pilosa/v2/statik" "github.com/pilosa/pilosa/v2/stats" @@ -88,7 +89,7 @@ type Command struct { grpcLn net.Listener API *pilosa.API ln net.Listener - listenURI *pilosa.URI + listenURI *pnet.URI tlsConfig *tls.Config closeTimeout time.Duration pgserver *PostgresServer @@ -309,7 +310,7 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "processing bind address") } - grpcURI, err := pilosa.NewURIFromAddress(m.Config.BindGRPC) + grpcURI, err := pnet.NewURIFromAddress(m.Config.BindGRPC) if err != nil { return errors.Wrap(err, "processing bind grpc address") } @@ -368,7 +369,7 @@ func (m *Command) SetupServer() error { } // Get grpc advertise address as uri. - advertiseGRPCURI, err := pilosa.NewURIFromAddress(m.Config.AdvertiseGRPC) + advertiseGRPCURI, err := pnet.NewURIFromAddress(m.Config.AdvertiseGRPC) if err != nil { return errors.Wrap(err, "processing grpc advertise address") } @@ -595,7 +596,7 @@ func newStatsClient(name string, host string) (stats.StatsClient, error) { } // getListener gets a net.Listener based on the config. -func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { +func getListener(uri pnet.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS if uri.Scheme == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) diff --git a/topology/hasher.go b/topology/hasher.go new file mode 100644 index 000000000..a5c3f5964 --- /dev/null +++ b/topology/hasher.go @@ -0,0 +1,41 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +// Hasher represents an interface to hash integers into buckets. +type Hasher interface { + // Hashes the key into a number between [0,N). + Hash(key uint64, n int) int + Name() string +} + +// Jmphasher represents an implementation of jmphash. Implements Hasher. +type Jmphasher struct{} + +// Hash returns the integer hash for the given key. +func (h *Jmphasher) Hash(key uint64, n int) int { + b, j := int64(-1), int64(0) + for j < int64(n) { + b = j + key = key*uint64(2862933555777941757) + 1 + j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) + } + return int(b) +} + +// Name returns the name of this hash. +func (h *Jmphasher) Name() string { + return "jump-hash" +} diff --git a/topology/node.go b/topology/node.go new file mode 100644 index 000000000..dd46f6207 --- /dev/null +++ b/topology/node.go @@ -0,0 +1,142 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "fmt" + + "github.com/pilosa/pilosa/v2/net" +) + +// Node represents a node in the cluster. +type Node struct { + ID string `json:"id"` + URI net.URI `json:"uri"` + GRPCURI net.URI `json:"grpc-uri"` + IsCoordinator bool `json:"isCoordinator"` + State string `json:"state"` +} + +func (n *Node) Clone() *Node { + if n == nil { + return nil + } + other := *n + return &other +} + +func (n Node) String() string { + return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID) +} + +// Nodes represents a list of nodes. +type Nodes []*Node + +// Contains returns true if a node exists in the list. +func (a Nodes) Contains(n *Node) bool { + for i := range a { + if a[i] == n { + return true + } + } + return false +} + +// ContainsID returns true if host matches one of the node's id. +func (a Nodes) ContainsID(id string) bool { + for _, n := range a { + if n.ID == id { + return true + } + } + return false +} + +// NodeByID returns the node for an ID. If the ID is not found, +// it returns nil. +func (a Nodes) NodeByID(id string) *Node { + for _, n := range a { + if n.ID == id { + return n + } + } + return nil +} + +// Filter returns a new list of nodes with node removed. +func (a Nodes) Filter(n *Node) []*Node { + other := make([]*Node, 0, len(a)) + for i := range a { + if a[i] != n { + other = append(other, a[i]) + } + } + return other +} + +// FilterID returns a new list of nodes with ID removed. +func (a Nodes) FilterID(id string) []*Node { + other := make([]*Node, 0, len(a)) + for _, node := range a { + if node.ID != id { + other = append(other, node) + } + } + return other +} + +// FilterURI returns a new list of nodes with URI removed. +func (a Nodes) FilterURI(uri net.URI) []*Node { + other := make([]*Node, 0, len(a)) + for _, node := range a { + if node.URI != uri { + other = append(other, node) + } + } + return other +} + +// IDs returns a list of all node IDs. +func (a Nodes) IDs() []string { + ids := make([]string, len(a)) + for i, n := range a { + ids[i] = n.ID + } + return ids +} + +// URIs returns a list of all uris. +func (a Nodes) URIs() []net.URI { + uris := make([]net.URI, len(a)) + for i, n := range a { + uris[i] = n.URI + } + return uris +} + +// Clone returns a shallow copy of nodes. +func (a Nodes) Clone() []*Node { + other := make([]*Node, len(a)) + copy(other, a) + return other +} + +// ByID implements sort.Interface for []Node based on +// the ID field. +type ByID []*Node + +func (h ByID) Len() int { return len(h) } +func (h ByID) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h ByID) Less(i, j int) bool { return h[i].ID < h[j].ID } diff --git a/topology/noder.go b/topology/noder.go new file mode 100644 index 000000000..d6dff517a --- /dev/null +++ b/topology/noder.go @@ -0,0 +1,74 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "sort" +) + +// Noder is an interface which abstracts the Node slice so that the list of +// nodes in a cluster can be maintained outside of the cluster struct. +type Noder interface { + Nodes() []*Node // Remember: this has to be sorted correctly!! + SetNodes([]*Node) + AppendNode(*Node) + RemoveNode(nodeID string) bool +} + +// localNoder is a simple implementation of the Noder interface +// which maintains an instance of the `nodes` slice. +type localNoder struct { + nodes []*Node +} + +// NewLocalNoder is a helper function for wrapping an existing slice of Nodes +// with something which implements Noder. +func NewLocalNoder(nodes []*Node) *localNoder { + return &localNoder{ + nodes: nodes, + } +} + +// Nodes implements the Noder interface. +func (n *localNoder) Nodes() []*Node { + return n.nodes +} + +// SetNodes implements the Noder interface. +func (n *localNoder) SetNodes(nodes []*Node) { + n.nodes = nodes +} + +// AppendNode implements the Noder interface. +func (n *localNoder) AppendNode(node *Node) { + n.nodes = append(n.nodes, node) + + // All hosts must be merged in the same order on all nodes in the cluster. + sort.Sort(ByID(n.nodes)) +} + +// RemoveNode implements the Noder interface. +func (n *localNoder) RemoveNode(nodeID string) bool { + i := NodePositionByID(n.nodes, nodeID) + if i < 0 { + return false + } + + copy(n.nodes[i:], n.nodes[i+1:]) + n.nodes[len(n.nodes)-1] = nil + n.nodes = n.nodes[:len(n.nodes)-1] + + return true +} diff --git a/topology/snapshot.go b/topology/snapshot.go new file mode 100644 index 000000000..2eaa7b0b9 --- /dev/null +++ b/topology/snapshot.go @@ -0,0 +1,272 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "encoding/binary" + "hash/fnv" + + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/shardwidth" +) + +const ( + // DefaultPartitionN is the default number of partitions in a cluster. + DefaultPartitionN = 256 + + // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. + // shardWidthExponent = 20 // set in shardwidthNN.go files + ShardWidth = 1 << shardwidth.Exponent +) + +// ClusterSnapshot is a static representation of a cluster and its nodes. It is +// used to calculate things like partition location and data distribution. +type ClusterSnapshot struct { + Nodes []*Node + + // Hashing algorithm used to assign partitions to nodes. + Hasher Hasher + + // The number of partitions in the cluster. + PartitionN int + + // The number of replicas a partition has. + ReplicaN int +} + +// NewClusterSnapshot returns a new instance of ClusterSnapshot. +func NewClusterSnapshot(noder Noder, hasher Hasher, replicas int) *ClusterSnapshot { + nodes := noder.Nodes() + + // Make sure replica count doesn't exceed the number of nodes. + nodeN := len(nodes) + if replicas > nodeN { + replicas = nodeN + } else if replicas == 0 { + replicas = 1 + } + + return &ClusterSnapshot{ + Nodes: nodes, + Hasher: hasher, + PartitionN: DefaultPartitionN, + ReplicaN: replicas, + } +} + +////////////////////////////////////////////////////////////////////////////// + +// shardToShardPartition returns the shard-partition that the given shard +// belongs to. NOTE: This is DIFFERENT from the key-partition. +func (c *ClusterSnapshot) shardToShardPartition(index string, shard uint64) int { + return dedupShardToShardPartition(index, shard, c.PartitionN) +} + +// dedupShardToShardParition would ideally be called `shardToShardPartition`, but since +// we can't put this into it's own package yet (see the TODO below about import loops), +// that name conflicts with a function that already exists in the `pilosa` package. +func dedupShardToShardPartition(index string, shard uint64, partitionN int) int { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], shard) + + // Hash the bytes and mod by partition count. + h := fnv.New64a() + _, _ = h.Write([]byte(index)) + _, _ = h.Write(buf[:]) + return int(h.Sum64() % uint64(partitionN)) +} + +// keyToKeyPartition returns the key-partition that the given key belongs to. +// NOTE: The key-partition is DIFFERENT from the shard-partition. +func (c *ClusterSnapshot) keyToKeyPartition(index, key string) int { + // Hash the bytes and mod by partition count. + h := fnv.New64a() + _, _ = h.Write([]byte(index)) + _, _ = h.Write([]byte(key)) + return int(h.Sum64() % uint64(c.PartitionN)) +} + +// ShardNodes returns a list of nodes that own a shard. +func (c *ClusterSnapshot) ShardNodes(index string, shard uint64) []*Node { + return c.PartitionNodes(c.shardToShardPartition(index, shard)) +} + +// KeyNodes returns a list of nodes that own a key. +func (c *ClusterSnapshot) KeyNodes(index, key string) []*Node { + return c.PartitionNodes(c.keyToKeyPartition(index, key)) +} + +// PartitionNodes returns a list of nodes that own the given partition. +func (c *ClusterSnapshot) PartitionNodes(partitionID int) []*Node { + // Determine primary owner node. + nodeIndex := c.PrimaryNodeIndex(partitionID) + if nodeIndex < 0 { + // no nodes anyway + return nil + } + // Collect nodes around the ring. + nodes := make([]*Node, 0, c.ReplicaN) + for i := 0; i < c.ReplicaN; i++ { + nodes = append(nodes, c.Nodes[(nodeIndex+i)%len(c.Nodes)]) + } + + return nodes +} + +// PrimaryFieldTranslationNode is the primary node responsible for translating +// field keys. The primary could be any node in the cluster, but we arbitrarily +// define it to be the node responsible for partition 0. +func (c *ClusterSnapshot) PrimaryFieldTranslationNode() *Node { + return c.PrimaryPartitionNode(0) +} + +// IsPrimaryFieldTranslationNode returns true if nodeID represents the primary +// node responsible for field translation. +func (c *ClusterSnapshot) IsPrimaryFieldTranslationNode(nodeID string) bool { + return c.PrimaryFieldTranslationNode().ID == nodeID +} + +// PrimaryPartitionNode returns the primary node of the given partition. +func (c *ClusterSnapshot) PrimaryPartitionNode(partition int) *Node { + if nodes := c.PartitionNodes(partition); len(nodes) > 0 { + return nodes[0] + } + return nil +} + +// IsPrimary returns true if the given node is the primary for the given +// partition. +func (c *ClusterSnapshot) IsPrimary(nodeID string, partition int) bool { + primary := c.PrimaryNodeIndex(partition) + return nodeID == c.Nodes[primary].ID +} + +// PrimaryNodeIndex returns the index (position in the cluster) of the primary +// node for the given partition. +func (c *ClusterSnapshot) PrimaryNodeIndex(partition int) int { + return c.Hasher.Hash(uint64(partition), len(c.Nodes)) +} + +// NonPrimaryReplicas returns the list of node IDs which are replicas for the +// given partition. +func (c *ClusterSnapshot) NonPrimaryReplicas(partition int) (nonPrimaryReplicas []string) { + primary := c.PrimaryNodeIndex(partition) + nodeN := len(c.Nodes) + + // Collect nodes around the ring. + for i := 1; i < nodeN; i++ { + node := c.Nodes[(primary+i)%nodeN] + if i < c.ReplicaN { + nonPrimaryReplicas = append(nonPrimaryReplicas, node.ID) + } + } + return +} + +// ReplicasForPrimary returns the map replicaNodeIDs[nodeID] which will have a +// true value for the primary nodeID, and false for others. +func (c *ClusterSnapshot) ReplicasForPrimary(primary int) (replicaNodeIDs, nonReplicas map[string]bool) { + if primary < 0 { + // no nodes anyway + return + } + replicaNodeIDs = make(map[string]bool) + nonReplicas = make(map[string]bool) + + nodeN := len(c.Nodes) + + // Collect nodes around the ring. + for i := 0; i < nodeN; i++ { + node := c.Nodes[(primary+i)%nodeN] + if i < c.ReplicaN { + // mark true if primary + replicaNodeIDs[node.ID] = (i == 0) + } else { + nonReplicas[node.ID] = false + } + } + return +} + +// ContainsShards is like OwnsShards, but it includes replicas. +func (c *ClusterSnapshot) ContainsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 { + var shards []uint64 + _ = availableShards.ForEach(func(i uint64) error { + p := c.shardToShardPartition(index, i) + // Determine the nodes for partition. + nodes := c.PartitionNodes(p) + for _, n := range nodes { + if n.ID == node.ID { + shards = append(shards, i) + } + } + return nil + }) + return shards +} + +// TODO: update this comment +// The boltdb key translation stores are partitioned, designated by partitionIDs. These +// are shared between replicas, and one node is the primary for +// replication. So with 4 nodes and 3-way replication, each node has 3/4 of +// the translation stores on it. +func (c *ClusterSnapshot) PrimaryForColKeyTranslation(index, key string) (primary int) { + partitionID := c.keyToKeyPartition(index, key) + return c.PrimaryNodeIndex(partitionID) +} + +// TODO: update this comment +// should match cluster.go:1033 cluster.ownsShard(nodeID, index, shard) +// return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) +func (c *ClusterSnapshot) PrimaryForShardReplication(index string, shard uint64) int { + n := len(c.Nodes) + if n == 0 { + return -1 + } + partition := uint64(dedupShardToShardPartition(index, shard, c.PartitionN)) + nodeIndex := c.Hasher.Hash(partition, n) + return nodeIndex +} + +// PrimaryReplicaNode returns the node listed before the current node in Nodes(). +// This is different than "previous node" as the first node always returns nil. +func (c *ClusterSnapshot) PrimaryReplicaNode(nodeID string) *Node { + pos := c.nodePositionByID(nodeID) + if pos <= 0 { + return nil + } + return c.Nodes[pos-1] +} + +// nodePositionByID returns the position of the node in slice c.Nodes. +func (c *ClusterSnapshot) nodePositionByID(nodeID string) int { + return NodePositionByID(c.Nodes, nodeID) +} + +// NodePositionByID returns the position of the node in slice nodes. +// TODO: this is exported because it's used in noder.go. Because that's the same +// package, it doesn't need to be exported, but ideally we could put this +// snapshot code into its own package. I tried to do that (by putting it into a +// package called `topology`), but that created an import loop. So what we +// really need to do is do a better job of creating sub-packages under pilosa +// (for things like `Noder` and `Nodes`). +func NodePositionByID(nodes []*Node, nodeID string) int { + for i, n := range nodes { + if n.ID == nodeID { + return i + } + } + return -1 +} diff --git a/utils_internal_test.go b/utils_internal_test.go index d59f6aabc..4b5820351 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -24,8 +24,10 @@ import ( "time" "github.com/gogo/protobuf/proto" + pnet "github.com/pilosa/pilosa/v2/net" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/testhook" + "github.com/pilosa/pilosa/v2/topology" "github.com/pkg/errors" ) @@ -73,7 +75,7 @@ func NewTestCluster(tb testing.TB, n int) *cluster { c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) for i := 0; i < n; i++ { - c.nodes = append(c.nodes, &Node{ + c.nodes = append(c.nodes, &topology.Node{ ID: fmt.Sprintf("node%d", i), URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) @@ -87,17 +89,17 @@ func NewTestCluster(tb testing.TB, n int) *cluster { } // NewTestURI is a test URI creator that intentionally swallows errors. -func NewTestURI(scheme, host string, port uint16) URI { - uri := defaultURI() - _ = uri.setScheme(scheme) - _ = uri.setHost(host) +func NewTestURI(scheme, host string, port uint16) pnet.URI { + uri := pnet.DefaultURI() + _ = uri.SetScheme(scheme) + _ = uri.SetHost(host) uri.SetPort(port) return *uri } -func NewTestURIFromHostPort(host string, port uint16) URI { - uri := defaultURI() - _ = uri.setHost(host) +func NewTestURIFromHostPort(host string, port uint16) pnet.URI { + uri := pnet.DefaultURI() + _ = uri.SetHost(host) uri.SetPort(port) return *uri } @@ -127,7 +129,7 @@ type ClusterCluster struct { } type commonClusterSettings struct { - Nodes []*Node + Nodes []*topology.Node } func (t *ClusterCluster) CreateIndex(name string) error { @@ -257,7 +259,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) id := fmt.Sprintf("node%d", i) uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)) - node := &Node{ + node := &topology.Node{ ID: id, URI: uri, } @@ -406,7 +408,7 @@ func (bcast) SendAsync(Message) error { } // SendTo is a test implementation of Broadcaster SendTo method. -func (b bcast) SendTo(to *Node, m Message) error { +func (b bcast) SendTo(to *topology.Node, m Message) error { switch obj := m.(type) { case *ResizeInstruction: err := b.t.FollowResizeInstruction(obj) @@ -551,7 +553,7 @@ func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN for i := 0; i < nNodes; i++ { nodeID := fmt.Sprintf("node%d", i) - c.nodes = append(c.nodes, &Node{ + c.nodes = append(c.nodes, &topology.Node{ ID: nodeID, URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), })