From 4ef8fff9b6dbce49f951bf4c3e75481c3b2b7327 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 4 Jul 2018 07:29:16 -0500 Subject: [PATCH 01/18] WIP, broken. refactoring to isolate intneral structs and define core structs --- broadcast.go | 33 ++++-- cluster.go | 306 ++++++++++++++++++++++++++++++++++++++------------- server.go | 6 +- 3 files changed, 259 insertions(+), 86 deletions(-) diff --git a/broadcast.go b/broadcast.go index 19e927c18..3f6b960e4 100644 --- a/broadcast.go +++ b/broadcast.go @@ -25,11 +25,15 @@ import ( // broadcaster is an interface for broadcasting messages. type broadcaster interface { - SendSync(pb proto.Message) error - SendAsync(pb proto.Message) error - SendTo(to *Node, pb proto.Message) error + SendSync(Message) error + SendAsync(Message) error + SendTo(*Node, Message) error } +// Message is the interface implemented by all core pilosa types which can be serialized to messages. +// TODO add at least a single "isMessage()" method. +type Message interface{} + func init() { NopBroadcaster = &nopBroadcaster{} } @@ -40,13 +44,13 @@ var NopBroadcaster broadcaster type nopBroadcaster struct{} // SendSync A no-op implementation of Broadcaster SendSync method. -func (n nopBroadcaster) SendSync(pb proto.Message) error { return nil } +func (nopBroadcaster) SendSync(Message) error { return nil } // SendAsync A no-op implementation of Broadcaster SendAsync method. -func (n nopBroadcaster) SendAsync(pb proto.Message) error { return nil } +func (nopBroadcaster) SendAsync(Message) error { return nil } // SendTo is a no-op implementation of Broadcaster SendTo method. -func (c nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil } +func (nopBroadcaster) SendTo(*Node, Message) error { return nil } // Broadcast message types. const ( @@ -69,7 +73,8 @@ const ( ) // MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(m proto.Message) ([]byte, error) { +func MarshalMessage(pm Message) ([]byte, error) { + m := encode(pm) var typ uint8 switch obj := m.(type) { case *internal.CreateShardMessage: @@ -114,8 +119,20 @@ func MarshalMessage(m proto.Message) ([]byte, error) { return append([]byte{typ}, buf...), nil } +func encode(m Message) proto.Message { + var pm proto.Message + switch mt := m.(type) { + case *CreateShardMessage: + return encodeCreateShardMessage(mt) + case *CreateIndexMessage: + return encodeCreateIndexMessage(mt) + // TODO, the rest + } + return nil +} + // UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (proto.Message, error) { +func UnmarshalMessage(buf []byte) (Message, error) { typ, buf := buf[0], buf[1:] var m proto.Message switch typ { diff --git a/cluster.go b/cluster.go index acf06dae0..fcfb08d73 100644 --- a/cluster.go +++ b/cluster.go @@ -69,52 +69,6 @@ func (n Node) String() string { return fmt.Sprintf("Node: %s", n.ID) } -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*Node) []*internal.Node { - other := make([]*internal.Node, len(a)) - for i := range a { - other[i] = EncodeNode(a[i]) - } - return other -} - -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *Node) *internal.Node { - return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), - IsCoordinator: n.IsCoordinator, - } -} - -// DecodeNodes converts a proto message into a slice of Nodes. -func DecodeNodes(a []*internal.Node) []*Node { - if len(a) == 0 { - return nil - } - other := make([]*Node, len(a)) - for i := range a { - other[i] = DecodeNode(a[i]) - } - return other -} - -// DecodeNode converts a proto message into a Node. -func DecodeNode(node *internal.Node) *Node { - return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), - IsCoordinator: node.IsCoordinator, - } -} - -func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ - Event: NodeEventType(ne.Event), - Node: DecodeNode(ne.Node), - } -} - // Nodes represents a list of nodes. type Nodes []*Node @@ -1176,7 +1130,7 @@ func (c *cluster) completeCurrentJob(state string) error { } // followResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) error { +func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { c.logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. @@ -1553,32 +1507,6 @@ func (c *cluster) saveTopology() error { return nil } -func encodeTopology(topology *Topology) *internal.Topology { - if topology == nil { - return nil - } - return &internal.Topology{ - ClusterID: topology.ClusterID, - NodeIDs: topology.NodeIDs, - } -} - -func decodeTopology(topology *internal.Topology) (*Topology, error) { - if topology == nil { - return nil, nil - } - - t := NewTopology() - t.ClusterID = topology.ClusterID - t.NodeIDs = topology.NodeIDs - sort.Slice(t.NodeIDs, - func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] - }) - - return t, nil -} - func (c *cluster) considerTopology() error { // Create ClusterID if one does not already exist. if c.id == "" { @@ -1752,7 +1680,7 @@ func (c *cluster) nodeLeave(node *Node) error { return nil } -func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() c.logger.Printf("merge cluster status: %v", cs) @@ -1764,7 +1692,7 @@ func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { // Set ClusterID. c.setID(cs.ClusterID) - officialNodes := DecodeNodes(cs.Nodes) + officialNodes := cs.Nodes // Add all nodes from the coordinator. for _, node := range officialNodes { @@ -1813,3 +1741,231 @@ func (c *cluster) setStatic(hosts []string) error { } return nil } + +type ClusterStatus struct { + ClusterID string + State string + Nodes []*Node +} + +type ResizeInstruction struct { + JobID int64 + Node *Node + Coordinator *Node + Sources []*ResizeSource + Schema *Schema + ClusterStatus *ClusterStatus +} + +func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction { + return &ResizeInstruction{ + JobID: ri.JobID, + Node: DecodeNode(ri.Node), + Coordinator: DecodeNode(ri.Coordinator), + Sources: decodeResizeSources(ri.Sources), + Schema: decodeSchema(ri.Schema), + ClusterStatus: decodeClusterStatus(ri.ClusterStatus), + } +} + +type ResizeSource struct { + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` +} + +func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource { + new := make([]*ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, decodeResizeSource(src)) + } + return new +} + +func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { + return &ResizeSource{ + Node: DecodeNode(rs.Node), + Index: rs.Index, + Field: rs.Field, + View: rs.View, + Shard: rs.Shard, + } +} + +// Schema is a schema +type Schema struct { + Indexes []*IndexInfo +} + +func decodeSchema(s *internal.Schema) *Schema { + return &Schema{ + Indexes: decodeIndexes(s.Indexes), + } +} + +func decodeIndexes(idxs []*internal.Index) []*IndexInfo { + new := make([]*IndexInfo, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, decodeIndex(idx)) + } + return new +} + +func decodeIndex(idx *internal.Index) *IndexInfo { + return &IndexInfo{ + Name: idx.Name, + Fields: decodeFields(idx.Fields), + } +} + +func decodeFields(fs []*internal.Field) []*FieldInfo { + new := make([]*FieldInfo, 0, len(fs)) + for _, f := range fs { + new = append(new, decodeField(f)) + } + return new +} + +func decodeField(f *internal.Field) *FieldInfo { + fi := &FieldInfo{ + Name: f.Name, + Options: *decodeFieldOptions(f.Meta), + Views: make([]*viewInfo, 0, len(f.Views)), + } + for _, viewname := range f.Views { + fi.Views = append(fi.Views, &viewInfo{Name: viewname}) + } + return fi +} + +// EncodeNodes converts a slice of Nodes into its internal representation. +func EncodeNodes(a []*Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = EncodeNode(a[i]) + } + return other +} + +// EncodeNode converts a Node into its internal representation. +func EncodeNode(n *Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: n.URI.Encode(), + IsCoordinator: n.IsCoordinator, + } +} + +// DecodeNodes converts a proto message into a slice of Nodes. +func DecodeNodes(a []*internal.Node) []*Node { + if len(a) == 0 { + return nil + } + other := make([]*Node, len(a)) + for i := range a { + other[i] = DecodeNode(a[i]) + } + return other +} + +func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus { + return &ClusterStatus{ + State: cs.State, + ClusterID: cs.ClusterID, + Nodes: DecodeNodes(cs.Nodes), + } +} + +// DecodeNode converts a proto message into a Node. +func DecodeNode(node *internal.Node) *Node { + return &Node{ + ID: node.ID, + URI: decodeURI(node.URI), + IsCoordinator: node.IsCoordinator, + } +} + +func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { + return &nodeEvent{ + Event: NodeEventType(ne.Event), + Node: DecodeNode(ne.Node), + } +} + +func encodeTopology(topology *Topology) *internal.Topology { + if topology == nil { + return nil + } + return &internal.Topology{ + ClusterID: topology.ClusterID, + NodeIDs: topology.NodeIDs, + } +} + +func decodeTopology(topology *internal.Topology) (*Topology, error) { + if topology == nil { + return nil, nil + } + + t := NewTopology() + t.ClusterID = topology.ClusterID + t.NodeIDs = topology.NodeIDs + sort.Slice(t.NodeIDs, + func(i, j int) bool { + return t.NodeIDs[i] < t.NodeIDs[j] + }) + + return t, nil +} + +type CreateShardMessage struct { + Index string + Shard uint64 +} + +func encodeCreateShardMessage(m *CreateShardMessage) *internal.CreateShardMessage { + return &internal.CreateShardMessage{ + Index: m.Index, + Shard: m.Shard, + } +} + +func decodeCreateShardMessage(pb *internal.CreateShardMessage) *CreateShardMessage { + return &CreateShardMessage{ + Index: pb.Index, + Shard: pb.Shard, + } +} + +type CreateIndexMessage struct { + Index string + Meta *IndexOptions +} + +func encodeCreateIndexMessage(m *CreateIndexMessage) *internal.CreateIndexMessage { + return &internal.CreateIndexMessage{ + Index: m.Index, + Meta: encodeIndexMeta(m.Meta), + } +} + +func decodeCreateIndexMessage(pb *internal.CreateIndexMessage) *CreateIndexMessage { + return &CreateIndexMessage{ + Index: pb.Index, + Meta: decodeIndexMeta(pb.Meta), + } +} + +func encodeIndexMeta(m *IndexOptions) *internal.IndexMeta { + return &internal.IndexMeta{ + Keys: m.Keys, + } +} + +func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions { + return &IndexOptions{ + Keys: pb.Keys, + } +} diff --git a/server.go b/server.go index 56195b427..dffeb94a0 100644 --- a/server.go +++ b/server.go @@ -483,12 +483,12 @@ func (s *Server) receiveMessage(pb proto.Message) error { return err } case *internal.ClusterStatus: - err := s.cluster.mergeClusterStatus(obj) + err := s.cluster.mergeClusterStatus(decodeClusterStatus(obj)) if err != nil { return err } case *internal.ResizeInstruction: - err := s.cluster.followResizeInstruction(obj) + err := s.cluster.followResizeInstruction(decodeResizeInstruction(obj)) if err != nil { return err } @@ -518,7 +518,7 @@ func (s *Server) receiveMessage(pb proto.Message) error { } // SendSync represents an implementation of Broadcaster. -func (s *Server) SendSync(pb proto.Message) error { +func (s *Server) SendSync(m Message) error { var eg errgroup.Group for _, node := range s.cluster.Nodes { node := node From 6417f468bb70a292d97bdcf6d7897d353440d5e0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 4 Jul 2018 16:50:20 -0500 Subject: [PATCH 02/18] wip implement more... still quite broken --- api.go | 2 +- broadcast.go | 73 ++++++++- cluster.go | 357 +++++++++++++++++++++++++++++++++++++---- holder.go | 7 +- server.go | 68 ++++---- utils_internal_test.go | 27 ++-- 6 files changed, 447 insertions(+), 87 deletions(-) diff --git a/api.go b/api.go index abf2a84ea..65ff90870 100644 --- a/api.go +++ b/api.go @@ -517,7 +517,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { } // Forward the error message. - if err := api.server.receiveMessage(pb); err != nil { + if err := api.server.receiveMessage(decode(pb)); err != nil { return errors.Wrap(err, "receiving message") } return nil diff --git a/broadcast.go b/broadcast.go index 3f6b960e4..0a33d4ca7 100644 --- a/broadcast.go +++ b/broadcast.go @@ -73,8 +73,7 @@ const ( ) // MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(pm Message) ([]byte, error) { - m := encode(pm) +func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { case *internal.CreateShardMessage: @@ -120,19 +119,45 @@ func MarshalMessage(pm Message) ([]byte, error) { } func encode(m Message) proto.Message { - var pm proto.Message switch mt := m.(type) { case *CreateShardMessage: return encodeCreateShardMessage(mt) case *CreateIndexMessage: return encodeCreateIndexMessage(mt) - // TODO, the rest + case *DeleteIndexMessage: + return encodeDeleteIndexMessage(mt) + case *CreateFieldMessage: + return encodeCreateFieldMessage(mt) + case *DeleteFieldMessage: + return encodeDeleteFieldMessage(mt) + case *CreateViewMessage: + return encodeCreateViewMessage(mt) + case *DeleteViewMessage: + return encodeDeleteViewMessage(mt) + case *ClusterStatus: + return encodeClusterStatus(mt) + case *ResizeInstruction: + return encodeResizeInstruction(mt) + case *ResizeInstructionComplete: + return encodeResizeInstructionComplete(mt) + case *SetCoordinatorMessage: + return encodeSetCoordinatorMessage(mt) + case *UpdateCoordinatorMessage: + return encodeUpdateCoordinatorMessage(mt) + case *NodeStateMessage: + return encodeNodeStateMessage(mt) + case *RecalculateCaches: + return encodeRecalculateCaches(mt) + case *nodeEvent: + return encodeNodeEventMessage(mt) + case *NodeStatus: + return encodeNodeStatus(mt) } return nil } // UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (Message, error) { +func UnmarshalMessage(buf []byte) (proto.Message, error) { typ, buf := buf[0], buf[1:] var m proto.Message switch typ { @@ -177,3 +202,41 @@ func UnmarshalMessage(buf []byte) (Message, error) { } return m, nil } + +func decode(m proto.Message) Message { + switch mt := m.(type) { + case *internal.CreateShardMessage: + return decodeCreateShardMessage(mt) + case *internal.CreateIndexMessage: + return decodeCreateIndexMessage(mt) + case *internal.DeleteIndexMessage: + return decodeDeleteIndexMessage(mt) + case *internal.CreateFieldMessage: + return decodeCreateFieldMessage(mt) + case *internal.DeleteFieldMessage: + return decodeDeleteFieldMessage(mt) + case *internal.CreateViewMessage: + return decodeCreateViewMessage(mt) + case *internal.DeleteViewMessage: + return decodeDeleteViewMessage(mt) + case *internal.ClusterStatus: + return decodeClusterStatus(mt) + case *internal.ResizeInstruction: + return decodeResizeInstruction(mt) + case *internal.ResizeInstructionComplete: + return decodeResizeInstructionComplete(mt) + case *internal.SetCoordinatorMessage: + return decodeSetCoordinatorMessage(mt) + case *internal.UpdateCoordinatorMessage: + return decodeUpdateCoordinatorMessage(mt) + case *internal.NodeStateMessage: + return decodeNodeStateMessage(mt) + case *internal.RecalculateCaches: + return decodeRecalculateCaches(mt) + case *internal.NodeEventMessage: + return decodeNodeEventMessage(mt) + case *internal.NodeStatus: + return decodeNodeStatus(mt) + } + return nil +} diff --git a/cluster.go b/cluster.go index fcfb08d73..e97ba46e4 100644 --- a/cluster.go +++ b/cluster.go @@ -268,8 +268,8 @@ func (c *cluster) setCoordinator(n *Node) error { c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.broadcaster.SendSync( - &internal.UpdateCoordinatorMessage{ - New: EncodeNode(n), + &UpdateCoordinatorMessage{ + New: n, }) if err != nil { return fmt.Errorf("problem sending UpdateCoordinator message: %v", err) @@ -423,7 +423,7 @@ func (c *cluster) setNodeState(state string) error { } // Send node state to coordinator. - ns := &internal.NodeStateMessage{ + ns := &NodeStateMessage{ NodeID: c.Node.ID, State: state, } @@ -460,12 +460,12 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { return nil } -// Status returns the internal ClusterStatus representation. -func (c *cluster) Status() *internal.ClusterStatus { - return &internal.ClusterStatus{ +// Status returns the the cluster's status including what nodes it contains, it's ID, and current state. +func (c *cluster) Status() *ClusterStatus { + return &ClusterStatus{ ClusterID: c.id, State: c.state, - Nodes: EncodeNodes(c.Nodes), + Nodes: c.Nodes, } } @@ -640,8 +640,8 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error) // fragSources returns a list of ResizeSources - for each node in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.ResizeSource, error) { - m := make(map[string][]*internal.ResizeSource) +func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) { + m := make(map[string][]*ResizeSource) // Determine if a node is being added or removed. action, diffNodeID, err := c.diff(to) @@ -700,7 +700,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R // Get the ResizeSource for each diff. for nodeID, diff := range diffs { - m[nodeID] = []*internal.ResizeSource{} + m[nodeID] = []*ResizeSource{} for _, frag := range diff { // If there is no valid source node ID for a fragment, // it likely means that the replica factor was not @@ -711,8 +711,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)") } - src := &internal.ResizeSource{ - Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)), + src := &ResizeSource{ + Node: c.unprotectedNodeByID(srcNodeID), Index: idx.Name(), Field: frag.field, View: frag.view, @@ -856,9 +856,9 @@ func (c *cluster) waitForStarted() error { // TODO: Because the normal code path already sends a NodeJoin event (via // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. - msg := &internal.NodeEventMessage{ - Event: uint32(NodeJoin), - Node: EncodeNode(c.Node), + msg := &nodeEvent{ + Event: NodeJoin, + Node: c.Node, } if err := c.broadcaster.SendSync(msg); err != nil { return fmt.Errorf("sending restart NodeJoin: %v", err) @@ -968,8 +968,8 @@ func (c *cluster) setStateAndBroadcast(state string) error { return c.broadcaster.SendSync(c.Status()) } -func (c *cluster) sendTo(node *Node, msg proto.Message) error { - if err := c.broadcaster.SendTo(node, msg); err != nil { +func (c *cluster) sendTo(node *Node, m Message) error { + if err := c.broadcaster.SendTo(node, m); err != nil { return errors.Wrap(err, "sending") } return nil @@ -1075,7 +1075,7 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, } // multiIndex is a map of sources initialized with all the nodes in toCluster. - multiIndex := make(map[string][]*internal.ResizeSource) + multiIndex := make(map[string][]*ResizeSource) for _, n := range toCluster.Nodes { multiIndex[n.ID] = nil @@ -1099,12 +1099,12 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, j.IDs[id] = true continue } - instr := &internal.ResizeInstruction{ + instr := &ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.unprotectedNodeByID(id)), - Coordinator: EncodeNode(c.coordinatorNode()), + Node: toCluster.unprotectedNodeByID(id), + Coordinator: c.coordinatorNode(), Sources: sources, - Schema: c.holder.encodeSchema(), // Include the schema to ensure it's in sync on the receiving node. + Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), } j.Instructions = append(j.Instructions, instr) @@ -1148,7 +1148,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { <-c.holder.opened // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -1167,7 +1167,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { for _, src := range instr.Sources { c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - srcURI := decodeURI(src.Node.URI) + srcURI := src.Node.URI // Retrieve field. f := c.holder.Field(src.Index, src.Field) @@ -1219,14 +1219,14 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { complete.Error = err.Error() } - if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { + if err := c.sendTo(instr.Coordinator, complete); err != nil { c.logger.Printf("sending resizeInstructionComplete error: err=%s", err) } }() return nil } -func (c *cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error { j := c.job(complete.JobID) @@ -1263,7 +1263,7 @@ func (c *cluster) job(id int64) *resizeJob { type resizeJob struct { ID int64 IDs map[string]bool - Instructions []*internal.ResizeInstruction + Instructions []*ResizeInstruction Broadcaster broadcaster action string @@ -1366,7 +1366,7 @@ func (j *resizeJob) distributeResizeInstructions() error { // a dummy node object to use in the SendTo() method. node := &Node{ ID: instr.Node.ID, - URI: decodeURI(instr.Node.URI), + URI: instr.Node.URI, } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -1757,7 +1757,7 @@ type ResizeInstruction struct { ClusterStatus *ClusterStatus } -func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction { +func decodeResizeInstruction(ri *internal.ResizeInstruction) *ResizeInstruction { return &ResizeInstruction{ JobID: ri.JobID, Node: DecodeNode(ri.Node), @@ -1768,6 +1768,17 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction { } } +func encodeResizeInstruction(m *ResizeInstruction) *internal.ResizeInstruction { + return &internal.ResizeInstruction{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Coordinator: EncodeNode(m.Coordinator), + Sources: encodeResizeSources(m.Sources), + Schema: encodeSchema(m.Schema), + ClusterStatus: encodeClusterStatus(m.ClusterStatus), + } +} + type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -1784,6 +1795,14 @@ func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource { return new } +func encodeResizeSources(srcs []*ResizeSource) []*internal.ResizeSource { + new := make([]*internal.ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, encodeResizeSource(src)) + } + return new +} + func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { return &ResizeSource{ Node: DecodeNode(rs.Node), @@ -1794,6 +1813,16 @@ func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { } } +func encodeResizeSource(m *ResizeSource) *internal.ResizeSource { + return &internal.ResizeSource{ + Node: EncodeNode(m.Node), + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + } +} + // Schema is a schema type Schema struct { Indexes []*IndexInfo @@ -1805,6 +1834,12 @@ func decodeSchema(s *internal.Schema) *Schema { } } +func encodeSchema(m *Schema) *internal.Schema { + return &internal.Schema{ + Indexes: encodeIndexInfos(m.Indexes), + } +} + func decodeIndexes(idxs []*internal.Index) []*IndexInfo { new := make([]*IndexInfo, 0, len(idxs)) for _, idx := range idxs { @@ -1813,6 +1848,14 @@ func decodeIndexes(idxs []*internal.Index) []*IndexInfo { return new } +func encodeIndexInfos(idxs []*IndexInfo) []*internal.Index { + new := make([]*internal.Index, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, encodeIndexInfo(idx)) + } + return new +} + func decodeIndex(idx *internal.Index) *IndexInfo { return &IndexInfo{ Name: idx.Name, @@ -1820,6 +1863,13 @@ func decodeIndex(idx *internal.Index) *IndexInfo { } } +func encodeIndexInfo(idx *IndexInfo) *internal.Index { + return &internal.Index{ + Name: idx.Name, + Fields: encodeFieldInfos(idx.Fields), + } +} + func decodeFields(fs []*internal.Field) []*FieldInfo { new := make([]*FieldInfo, 0, len(fs)) for _, f := range fs { @@ -1828,6 +1878,14 @@ func decodeFields(fs []*internal.Field) []*FieldInfo { return new } +func encodeFieldInfos(fs []*FieldInfo) []*internal.Field { + new := make([]*internal.Field, 0, len(fs)) + for _, f := range fs { + new = append(new, encodeFieldInfo(f)) + } + return new +} + func decodeField(f *internal.Field) *FieldInfo { fi := &FieldInfo{ Name: f.Name, @@ -1840,6 +1898,19 @@ func decodeField(f *internal.Field) *FieldInfo { return fi } +func encodeFieldInfo(f *FieldInfo) *internal.Field { + ifield := &internal.Field{ + Name: f.Name, + Meta: encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), + } + + for _, viewinfo := range f.Views { + ifield.Views = append(ifield.Views, viewinfo.Name) + } + return ifield +} + // EncodeNodes converts a slice of Nodes into its internal representation. func EncodeNodes(a []*Node) []*internal.Node { other := make([]*internal.Node, len(a)) @@ -1878,6 +1949,14 @@ func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus { } } +func encodeClusterStatus(m *ClusterStatus) *internal.ClusterStatus { + return &internal.ClusterStatus{ + State: m.State, + ClusterID: m.ClusterID, + Nodes: EncodeNodes(m.Nodes), + } +} + // DecodeNode converts a proto message into a Node. func DecodeNode(node *internal.Node) *Node { return &Node{ @@ -1969,3 +2048,223 @@ func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions { Keys: pb.Keys, } } + +type DeleteIndexMessage struct { + Index string +} + +func encodeDeleteIndexMessage(m *DeleteIndexMessage) *internal.DeleteIndexMessage { + return &internal.DeleteIndexMessage{ + Index: m.Index, + } +} + +func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage) *DeleteIndexMessage { + return &DeleteIndexMessage{ + Index: pb.Index, + } +} + +type CreateFieldMessage struct { + Index string + Field string + Meta *FieldOptions +} + +func encodeCreateFieldMessage(m *CreateFieldMessage) *internal.CreateFieldMessage { + return &internal.CreateFieldMessage{ + Index: m.Index, + Field: m.Field, + Meta: encodeFieldOptions(m.Meta), + } +} + +func decodeCreateFieldMessage(pb *internal.CreateFieldMessage) *CreateFieldMessage { + return &CreateFieldMessage{ + Index: pb.Index, + Field: pb.Field, + Meta: decodeFieldOptions(pb.Meta), + } +} + +type DeleteFieldMessage struct { + Index string + Field string +} + +func encodeDeleteFieldMessage(m *DeleteFieldMessage) *internal.DeleteFieldMessage { + return &internal.DeleteFieldMessage{ + Index: m.Index, + Field: m.Field, + } +} + +func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage) *DeleteFieldMessage { + return &DeleteFieldMessage{ + Index: pb.Index, + Field: pb.Field, + } +} + +type CreateViewMessage struct { + Index string + Field string + View string +} + +func encodeCreateViewMessage(m *CreateViewMessage) *internal.CreateViewMessage { + return &internal.CreateViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func decodeCreateViewMessage(pb *internal.CreateViewMessage) *CreateViewMessage { + return &CreateViewMessage{ + Index: pb.Index, + Field: pb.Field, + View: pb.View, + } +} + +type DeleteViewMessage struct { + Index string + Field string + View string +} + +func encodeDeleteViewMessage(m *DeleteViewMessage) *internal.DeleteViewMessage { + return &internal.DeleteViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func decodeDeleteViewMessage(pb *internal.DeleteViewMessage) *DeleteViewMessage { + return &DeleteViewMessage{ + Index: pb.Index, + Field: pb.Field, + View: pb.View, + } +} + +type ResizeInstructionComplete struct { + JobID int64 + Node *Node + Error string +} + +func encodeResizeInstructionComplete(m *ResizeInstructionComplete) *internal.ResizeInstructionComplete { + return &internal.ResizeInstructionComplete{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Error: m.Error, + } +} + +func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete) *ResizeInstructionComplete { + return &ResizeInstructionComplete{ + JobID: pb.JobID, + Node: DecodeNode(pb.Node), + Error: pb.Error, + } +} + +type SetCoordinatorMessage struct { + New *Node +} + +func encodeSetCoordinatorMessage(m *SetCoordinatorMessage) *internal.SetCoordinatorMessage { + return &internal.SetCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage) *SetCoordinatorMessage { + return &SetCoordinatorMessage{ + New: DecodeNode(pb.New), + } +} + +type UpdateCoordinatorMessage struct { + New *Node +} + +func encodeUpdateCoordinatorMessage(m *UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { + return &internal.UpdateCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage) *UpdateCoordinatorMessage { + return &UpdateCoordinatorMessage{ + New: DecodeNode(pb.New), + } +} + +type NodeStateMessage struct { + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` +} + +func encodeNodeStateMessage(m *NodeStateMessage) *internal.NodeStateMessage { + return &internal.NodeStateMessage{ + NodeID: m.NodeID, + State: m.State, + } +} + +func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage { + return &NodeStateMessage{ + NodeID: pb.NodeID, + State: pb.State, + } +} + +func encodeNodeEventMessage(m *nodeEvent) *internal.NodeEventMessage { + return &internal.NodeEventMessage{ + Event: uint32(m.Event), + Node: EncodeNode(m.Node), + } +} + +func decodeNodeEventMessage(pb *internal.NodeEventMessage) *nodeEvent { + return &nodeEvent{ + Event: NodeEventType(pb.Event), + Node: DecodeNode(pb.Node), + } +} + +type NodeStatus struct { + Node *Node + MaxShards map[string]uint64 + Schema *Schema +} + +func encodeNodeStatus(m *NodeStatus) *internal.NodeStatus { + return &internal.NodeStatus{ + Node: EncodeNode(m.Node), + MaxShards: &internal.MaxShards{Standard: m.MaxShards}, + Schema: encodeSchema(m.Schema), + } +} + +func decodeNodeStatus(pb *internal.NodeStatus) *NodeStatus { + return &NodeStatus{ + Node: DecodeNode(pb.Node), + MaxShards: pb.MaxShards.Standard, + Schema: decodeSchema(pb.Schema), + } +} + +type RecalculateCaches struct{} + +func decodeRecalculateCaches(pb *internal.RecalculateCaches) *RecalculateCaches { + return &RecalculateCaches{} +} + +func encodeRecalculateCaches(*RecalculateCaches) *internal.RecalculateCaches { + return &internal.RecalculateCaches{} +} diff --git a/holder.go b/holder.go index e599d21f2..83ffd6967 100644 --- a/holder.go +++ b/holder.go @@ -230,7 +230,7 @@ func (h *Holder) Schema() []*IndexInfo { } // applySchema applies an internal Schema to Holder. -func (h *Holder) applySchema(schema *internal.Schema) error { +func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. for _, index := range schema.Indexes { opt := IndexOptions{} @@ -240,14 +240,13 @@ func (h *Holder) applySchema(schema *internal.Schema) error { } // Create fields that don't exist. for _, f := range index.Fields { - opt := decodeFieldOptions(f.Meta) - field, err := idx.CreateFieldIfNotExists(f.Name, *opt) + field, err := idx.CreateFieldIfNotExists(f.Name, f.Options) if err != nil { return errors.Wrap(err, "creating field") } // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v) + _, err := field.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/server.go b/server.go index dffeb94a0..7199b7178 100644 --- a/server.go +++ b/server.go @@ -27,8 +27,6 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -431,40 +429,40 @@ func (s *Server) monitorAntiEntropy() { } // receiveMessage represents an implementation of BroadcastHandler. -func (s *Server) receiveMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateShardMessage: +func (s *Server) receiveMessage(m Message) error { + switch obj := m.(type) { + case *CreateShardMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } idx.setRemoteMaxShard(obj.Shard) - case *internal.CreateIndexMessage: + case *CreateIndexMessage: opt := IndexOptions{} _, err := s.holder.CreateIndex(obj.Index, opt) if err != nil { return err } - case *internal.DeleteIndexMessage: + case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err } - case *internal.CreateFieldMessage: + case *CreateFieldMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := decodeFieldOptions(obj.Meta) + opt := obj.Meta _, err := idx.CreateField(obj.Field, *opt) if err != nil { return err } - case *internal.DeleteFieldMessage: + case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { return err } - case *internal.CreateViewMessage: + case *CreateViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) @@ -473,7 +471,7 @@ func (s *Server) receiveMessage(pb proto.Message) error { if err != nil { return err } - case *internal.DeleteViewMessage: + case *DeleteViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) @@ -482,36 +480,36 @@ func (s *Server) receiveMessage(pb proto.Message) error { if err != nil { return err } - case *internal.ClusterStatus: - err := s.cluster.mergeClusterStatus(decodeClusterStatus(obj)) + case *ClusterStatus: + err := s.cluster.mergeClusterStatus(obj) if err != nil { return err } - case *internal.ResizeInstruction: - err := s.cluster.followResizeInstruction(decodeResizeInstruction(obj)) + case *ResizeInstruction: + err := s.cluster.followResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: err := s.cluster.markResizeInstructionComplete(obj) if err != nil { return err } - case *internal.SetCoordinatorMessage: - s.cluster.setCoordinator(DecodeNode(obj.New)) - case *internal.UpdateCoordinatorMessage: - s.cluster.updateCoordinator(DecodeNode(obj.New)) - case *internal.NodeStateMessage: + case *SetCoordinatorMessage: + s.cluster.setCoordinator(obj.New) + case *UpdateCoordinatorMessage: + s.cluster.updateCoordinator(obj.New) + case *NodeStateMessage: err := s.cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } - case *internal.RecalculateCaches: + case *RecalculateCaches: s.holder.RecalculateCaches() - case *internal.NodeEventMessage: - s.cluster.ReceiveEvent(DecodeNodeEvent(obj)) - case *internal.NodeStatus: - s.handleRemoteStatus(pb) + case *nodeEvent: + s.cluster.ReceiveEvent(obj) + case *NodeStatus: + s.handleRemoteStatus(obj) } return nil @@ -519,6 +517,7 @@ func (s *Server) receiveMessage(pb proto.Message) error { // SendSync represents an implementation of Broadcaster. func (s *Server) SendSync(m Message) error { + pb := encode(m) var eg errgroup.Group for _, node := range s.cluster.Nodes { node := node @@ -537,12 +536,13 @@ func (s *Server) SendSync(m Message) error { } // SendAsync represents an implementation of Broadcaster. -func (s *Server) SendAsync(pb proto.Message) error { +func (s *Server) SendAsync(m Message) error { return ErrNotImplemented } // SendTo represents an implementation of Broadcaster. -func (s *Server) SendTo(to *Node, pb proto.Message) error { +func (s *Server) SendTo(to *Node, m Message) error { + pb := encode(m) s.logger.Printf("SendTo: %s", to.URI) return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) } @@ -554,7 +554,7 @@ func (s *Server) node() Node { } // handleRemoteStatus receives incoming NodeStatus from remote nodes. -func (s *Server) handleRemoteStatus(pb proto.Message) { +func (s *Server) handleRemoteStatus(pb Message) { // Ignore NodeStatus messages until the cluster is in a Normal state. if s.cluster.State() != ClusterStateNormal { return @@ -564,16 +564,16 @@ func (s *Server) handleRemoteStatus(pb proto.Message) { // Make sure the holder has opened. <-s.holder.opened - err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + err := s.mergeRemoteStatus(pb.(*NodeStatus)) if err != nil { s.logger.Printf("merge remote status: %s", err) } }() } -func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { +func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { // Ignore status updates from self. - if s.nodeID == DecodeNode(ns.Node).ID { + if s.nodeID == ns.Node.ID { return nil } @@ -584,7 +584,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // Sync maxShards. oldmaxshards := s.holder.maxShards() - for index, newMax := range ns.MaxShards.Standard { + for index, newMax := range ns.MaxShards { localIndex := s.holder.Index(index) // if we don't know about an index locally, log an error because // indexes should be created and synced prior to shard creation diff --git a/utils_internal_test.go b/utils_internal_test.go index ca7bd1fa7..f7c11d6a7 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -24,7 +24,6 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -304,9 +303,9 @@ func (t *ClusterCluster) Close() error { } // SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *ClusterCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: +func (t *ClusterCluster) SendSync(m Message) error { + switch obj := m.(type) { + case *ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { c.mergeClusterStatus(obj) @@ -322,19 +321,19 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { } // SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *ClusterCluster) SendAsync(pb proto.Message) error { +func (t *ClusterCluster) SendAsync(Message) error { return nil } // SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: +func (t *ClusterCluster) SendTo(to *Node, m Message) error { + switch obj := m.(type) { + case *ResizeInstruction: err := t.FollowResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: coord := t.clusterByID(to.ID) go coord.markResizeInstructionComplete(obj) } @@ -342,10 +341,10 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { } // FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error { // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -356,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi // figure out which node it was meant for, then call the operation on that cluster // basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI) - instrNode := DecodeNode(instr.Node) + instrNode := instr.Node destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. @@ -365,7 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi } for _, src := range instr.Sources { - srcNode := DecodeNode(src.Node) + srcNode := src.Node srcCluster := t.clusterByID(srcNode.ID) srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) @@ -405,6 +404,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi complete.Error = err.Error() } - node := DecodeNode(instr.Coordinator) + node := instr.Coordinator return t.SendTo(node, complete) } From cd8c63c125935bba86a0b658eac13de50a4ffbc1 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 4 Jul 2018 21:43:18 -0500 Subject: [PATCH 03/18] tests passing --- api.go | 17 ++++++++--------- cluster_internal_test.go | 41 ++++++++++++++++++++-------------------- field.go | 2 +- view.go | 3 +-- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/api.go b/api.go index 65ff90870..73bdd6665 100644 --- a/api.go +++ b/api.go @@ -185,12 +185,11 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index } // Send the create index message to all nodes. err = api.server.SendSync( - &internal.CreateIndexMessage{ + &CreateIndexMessage{ Index: indexName, - Meta: options.Encode(), + Meta: &options, }) if err != nil { - api.server.logger.Printf("problem sending CreateIndex message: %s", err) return nil, errors.Wrap(err, "sending CreateIndex message") } api.holder.Stats.Count("createIndex", 1, 1.0) @@ -224,7 +223,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // Send the delete index message to all nodes. err = api.server.SendSync( - &internal.DeleteIndexMessage{ + &DeleteIndexMessage{ Index: indexName, }) if err != nil { @@ -264,10 +263,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str // Send the create field message to all nodes. err = api.server.SendSync( - &internal.CreateFieldMessage{ + &CreateFieldMessage{ Index: indexName, Field: fieldName, - Meta: fo.Encode(), + Meta: &fo, }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) @@ -311,7 +310,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str // Send the delete field message to all nodes. err := api.server.SendSync( - &internal.DeleteFieldMessage{ + &DeleteFieldMessage{ Index: indexName, Field: fieldName, }) @@ -489,7 +488,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return errors.Wrap(err, "validating api method") } - err := api.server.SendSync(&internal.RecalculateCaches{}) + err := api.server.SendSync(&RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") } @@ -568,7 +567,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri // Send the delete view message to all nodes. err := api.server.SendSync( - &internal.DeleteViewMessage{ + &DeleteViewMessage{ Index: indexName, Field: fieldName, View: viewName, diff --git a/cluster_internal_test.go b/cluster_internal_test.go index aee4ea09b..e209a8f80 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -24,7 +24,6 @@ import ( "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -175,19 +174,19 @@ func TestFragSources(t *testing.T) { from *cluster to *cluster idx *Index - expected map[string][]*internal.ResizeSource + expected map[string][]*ResizeSource err string }{ { from: c1, to: c2, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{}, - "node1": []*internal.ResizeSource{}, - "node2": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{}, + "node1": []*ResizeSource{}, + "node2": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -196,13 +195,13 @@ func TestFragSources(t *testing.T) { from: c4, to: c3, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -211,15 +210,15 @@ func TestFragSources(t *testing.T) { from: c5, to: c4, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, }, - "node2": []*internal.ResizeSource{}, + "node2": []*ResizeSource{}, }, err: "", }, diff --git a/field.go b/field.go index 76be9cf3f..eea6bb10e 100644 --- a/field.go +++ b/field.go @@ -605,7 +605,7 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) { if created { // Broadcast view creation to the cluster. err = f.broadcaster.SendSync( - &internal.CreateViewMessage{ + &CreateViewMessage{ Index: f.index, Field: f.name, View: name, diff --git a/view.go b/view.go index fd5306b85..0f2e189cd 100644 --- a/view.go +++ b/view.go @@ -22,7 +22,6 @@ import ( "strings" "sync" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -232,7 +231,7 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) { // Send the create shard message to all nodes. err := v.broadcaster.SendSync( - &internal.CreateShardMessage{ + &CreateShardMessage{ Index: v.index, Shard: shard, }) From 1b2aaa26bf13c1b42b902a09ff125eee7ca32080 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 09:26:58 -0500 Subject: [PATCH 04/18] export NodeEvent --- broadcast.go | 2 +- cluster.go | 14 +++++++------- event.go | 4 ++-- server.go | 2 +- utils_internal_test.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/broadcast.go b/broadcast.go index 0a33d4ca7..d3c4a2562 100644 --- a/broadcast.go +++ b/broadcast.go @@ -148,7 +148,7 @@ func encode(m Message) proto.Message { return encodeNodeStateMessage(mt) case *RecalculateCaches: return encodeRecalculateCaches(mt) - case *nodeEvent: + case *NodeEvent: return encodeNodeEventMessage(mt) case *NodeStatus: return encodeNodeStatus(mt) diff --git a/cluster.go b/cluster.go index e97ba46e4..2d25ebeaa 100644 --- a/cluster.go +++ b/cluster.go @@ -856,7 +856,7 @@ func (c *cluster) waitForStarted() error { // TODO: Because the normal code path already sends a NodeJoin event (via // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. - msg := &nodeEvent{ + msg := &NodeEvent{ Event: NodeJoin, Node: c.Node, } @@ -1540,7 +1540,7 @@ func (c *cluster) considerTopology() error { } // ReceiveEvent represents an implementation of EventHandler. -func (c *cluster) ReceiveEvent(e *nodeEvent) error { +func (c *cluster) ReceiveEvent(e *NodeEvent) error { // Ignore events sent from this node. if e.Node.ID == c.Node.ID { return nil @@ -1966,8 +1966,8 @@ func DecodeNode(node *internal.Node) *Node { } } -func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ +func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { + return &NodeEvent{ Event: NodeEventType(ne.Event), Node: DecodeNode(ne.Node), } @@ -2223,15 +2223,15 @@ func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage { } } -func encodeNodeEventMessage(m *nodeEvent) *internal.NodeEventMessage { +func encodeNodeEventMessage(m *NodeEvent) *internal.NodeEventMessage { return &internal.NodeEventMessage{ Event: uint32(m.Event), Node: EncodeNode(m.Node), } } -func decodeNodeEventMessage(pb *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ +func decodeNodeEventMessage(pb *internal.NodeEventMessage) *NodeEvent { + return &NodeEvent{ Event: NodeEventType(pb.Event), Node: DecodeNode(pb.Node), } diff --git a/event.go b/event.go index aa1e0e890..0d5e59e99 100644 --- a/event.go +++ b/event.go @@ -23,8 +23,8 @@ const ( NodeUpdate ) -// nodeEvent is a single event related to node activity in the cluster. -type nodeEvent struct { +// NodeEvent is a single event related to node activity in the cluster. +type NodeEvent struct { Event NodeEventType Node *Node } diff --git a/server.go b/server.go index 7199b7178..b57935acb 100644 --- a/server.go +++ b/server.go @@ -506,7 +506,7 @@ func (s *Server) receiveMessage(m Message) error { } case *RecalculateCaches: s.holder.RecalculateCaches() - case *nodeEvent: + case *NodeEvent: s.cluster.ReceiveEvent(obj) case *NodeStatus: s.handleRemoteStatus(obj) diff --git a/utils_internal_test.go b/utils_internal_test.go index f7c11d6a7..4b9f6870a 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -161,7 +161,7 @@ func (t *ClusterCluster) addNode() error { // Send NodeJoin event to coordinator. if id > 0 { coord := t.Clusters[0] - ev := &nodeEvent{ + ev := &NodeEvent{ Event: NodeJoin, Node: c.Node, } From 2cec75e39931f46d31842d16f273d67826fcc00c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 12:00:42 -0500 Subject: [PATCH 05/18] add proto encoding subpackage and use for send and receive message --- api.go | 9 +- broadcast.go | 116 +++++--- client.go | 5 +- cluster.go | 4 +- encoding/proto/proto.go | 594 ++++++++++++++++++++++++++++++++++++++++ field.go | 6 +- gossip/gossip.go | 6 +- holder.go | 2 +- http/client.go | 14 +- server.go | 26 +- server/server.go | 14 +- uri.go | 74 ++--- uri_internal_test.go | 24 +- view.go | 6 +- 14 files changed, 778 insertions(+), 122 deletions(-) create mode 100644 encoding/proto/proto.go diff --git a/api.go b/api.go index 73bdd6665..09f0531d5 100644 --- a/api.go +++ b/api.go @@ -509,14 +509,15 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { return errors.Wrap(err, "reading body") } - // Marshal into request object. - pb, err := UnmarshalMessage(body) + typ := body[0] + msg := getMessage(typ) + err = api.server.serializer.Unmarshal(body[1:], msg) if err != nil { - return errors.Wrap(err, "unmarshaling message") + return errors.Wrap(err, "deserializing cluster message") } // Forward the error message. - if err := api.server.receiveMessage(decode(pb)); err != nil { + if err := api.server.receiveMessage(msg); err != nil { return errors.Wrap(err, "receiving message") } return nil diff --git a/broadcast.go b/broadcast.go index d3c4a2562..1da6ca57f 100644 --- a/broadcast.go +++ b/broadcast.go @@ -23,6 +23,12 @@ import ( "github.com/pkg/errors" ) +// Serializer is an interface for serializing pilosa types to bytes and back. +type Serializer interface { + Marshal(Message) ([]byte, error) + Unmarshal([]byte, Message) error +} + // broadcaster is an interface for broadcasting messages. type broadcaster interface { SendSync(Message) error @@ -118,42 +124,82 @@ func MarshalMessage(m proto.Message) ([]byte, error) { return append([]byte{typ}, buf...), nil } -func encode(m Message) proto.Message { - switch mt := m.(type) { - case *CreateShardMessage: - return encodeCreateShardMessage(mt) - case *CreateIndexMessage: - return encodeCreateIndexMessage(mt) - case *DeleteIndexMessage: - return encodeDeleteIndexMessage(mt) - case *CreateFieldMessage: - return encodeCreateFieldMessage(mt) - case *DeleteFieldMessage: - return encodeDeleteFieldMessage(mt) - case *CreateViewMessage: - return encodeCreateViewMessage(mt) - case *DeleteViewMessage: - return encodeDeleteViewMessage(mt) - case *ClusterStatus: - return encodeClusterStatus(mt) - case *ResizeInstruction: - return encodeResizeInstruction(mt) - case *ResizeInstructionComplete: - return encodeResizeInstructionComplete(mt) - case *SetCoordinatorMessage: - return encodeSetCoordinatorMessage(mt) - case *UpdateCoordinatorMessage: - return encodeUpdateCoordinatorMessage(mt) - case *NodeStateMessage: - return encodeNodeStateMessage(mt) - case *RecalculateCaches: - return encodeRecalculateCaches(mt) - case *NodeEvent: - return encodeNodeEventMessage(mt) - case *NodeStatus: - return encodeNodeStatus(mt) +func getMessage(typ byte) Message { + switch typ { + case messageTypeCreateShard: + return &CreateShardMessage{} + case messageTypeCreateIndex: + return &CreateIndexMessage{} + case messageTypeDeleteIndex: + return &DeleteIndexMessage{} + case messageTypeCreateField: + return &CreateFieldMessage{} + case messageTypeDeleteField: + return &DeleteFieldMessage{} + case messageTypeCreateView: + return &CreateViewMessage{} + case messageTypeDeleteView: + return &DeleteViewMessage{} + case messageTypeClusterStatus: + return &ClusterStatus{} + case messageTypeResizeInstruction: + return &ResizeInstruction{} + case messageTypeResizeInstructionComplete: + return &ResizeInstructionComplete{} + case messageTypeSetCoordinator: + return &SetCoordinatorMessage{} + case messageTypeUpdateCoordinator: + return &UpdateCoordinatorMessage{} + case messageTypeNodeState: + return &NodeStateMessage{} + case messageTypeRecalculateCaches: + return &RecalculateCaches{} + case messageTypeNodeEvent: + return &NodeEvent{} + case messageTypeNodeStatus: + return &NodeStatus{} + default: + panic(fmt.Sprintf("unknown message type %d", typ)) + } +} + +func getMessageType(m Message) byte { + switch m.(type) { + case *CreateShardMessage: + return messageTypeCreateShard + case *CreateIndexMessage: + return messageTypeCreateIndex + case *DeleteIndexMessage: + return messageTypeDeleteIndex + case *CreateFieldMessage: + return messageTypeCreateField + case *DeleteFieldMessage: + return messageTypeDeleteField + case *CreateViewMessage: + return messageTypeCreateView + case *DeleteViewMessage: + return messageTypeDeleteView + case *ClusterStatus: + return messageTypeClusterStatus + case *ResizeInstruction: + return messageTypeResizeInstruction + case *ResizeInstructionComplete: + return messageTypeResizeInstructionComplete + case *SetCoordinatorMessage: + return messageTypeSetCoordinator + case *UpdateCoordinatorMessage: + return messageTypeUpdateCoordinator + case *NodeStateMessage: + return messageTypeNodeState + case *RecalculateCaches: + return messageTypeRecalculateCaches + case *NodeEvent: + return messageTypeNodeEvent + case *NodeStatus: + return messageTypeNodeStatus + default: + panic(fmt.Sprintf("don't have type for message %#v", m)) } - return nil } // UnmarshalMessage decodes the byte slice into a protobuf message. diff --git a/client.go b/client.go index 59e3ad59b..01e0b847a 100644 --- a/client.go +++ b/client.go @@ -4,7 +4,6 @@ import ( "context" "io" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -49,7 +48,7 @@ type InternalClient interface { BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) } @@ -128,7 +127,7 @@ func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index s func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { +func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { return nil } func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { diff --git a/cluster.go b/cluster.go index 2d25ebeaa..e2f29fde5 100644 --- a/cluster.go +++ b/cluster.go @@ -1890,10 +1890,10 @@ func decodeField(f *internal.Field) *FieldInfo { fi := &FieldInfo{ Name: f.Name, Options: *decodeFieldOptions(f.Meta), - Views: make([]*viewInfo, 0, len(f.Views)), + Views: make([]*ViewInfo, 0, len(f.Views)), } for _, viewname := range f.Views { - fi.Views = append(fi.Views, &viewInfo{Name: viewname}) + fi.Views = append(fi.Views, &ViewInfo{Name: viewname}) } return fi } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go new file mode 100644 index 000000000..1c83ba4ea --- /dev/null +++ b/encoding/proto/proto.go @@ -0,0 +1,594 @@ +package proto + +import ( + "fmt" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// Serializer implements pilosa.Serializer for protobufs. +type Serializer struct{} + +// Marshal turns pilosa messages into protobuf serialized bytes. +func (Serializer) Marshal(m pilosa.Message) ([]byte, error) { + pm := encodeToProto(m) + if pm == nil { + return nil, errors.New("passed invalid pilosa.Message") + } + buf, err := proto.Marshal(pm) + return buf, errors.Wrap(err, "marshalling") +} + +// Unmarshal takes byte slices and protobuf deserializes them into a pilosa Message. +func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + msg := &internal.CreateShardMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateShardMessage") + } + decodeCreateShardMessage(msg, mt) + return nil + case *pilosa.CreateIndexMessage: + msg := &internal.CreateIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateIndexMessage") + } + decodeCreateIndexMessage(msg, mt) + return nil + case *pilosa.DeleteIndexMessage: + msg := &internal.DeleteIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteIndexMessage") + } + decodeDeleteIndexMessage(msg, mt) + return nil + case *pilosa.CreateFieldMessage: + msg := &internal.CreateFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateFieldMessage") + } + decodeCreateFieldMessage(msg, mt) + return nil + case *pilosa.DeleteFieldMessage: + msg := &internal.DeleteFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteFieldMessage") + } + decodeDeleteFieldMessage(msg, mt) + return nil + case *pilosa.CreateViewMessage: + msg := &internal.CreateViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateViewMessage") + } + decodeCreateViewMessage(msg, mt) + return nil + case *pilosa.DeleteViewMessage: + msg := &internal.DeleteViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteViewMessage") + } + decodeDeleteViewMessage(msg, mt) + return nil + case *pilosa.ClusterStatus: + msg := &internal.ClusterStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ClusterStatus") + } + decodeClusterStatus(msg, mt) + return nil + case *pilosa.ResizeInstruction: + msg := &internal.ResizeInstruction{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstruction") + } + decodeResizeInstruction(msg, mt) + return nil + case *pilosa.ResizeInstructionComplete: + msg := &internal.ResizeInstructionComplete{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstructionComplete") + } + decodeResizeInstructionComplete(msg, mt) + return nil + case *pilosa.SetCoordinatorMessage: + msg := &internal.SetCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling SetCoordinatorMessage") + } + decodeSetCoordinatorMessage(msg, mt) + return nil + case *pilosa.UpdateCoordinatorMessage: + msg := &internal.UpdateCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling UpdateCoordinatorMessage") + } + decodeUpdateCoordinatorMessage(msg, mt) + return nil + case *pilosa.NodeStateMessage: + msg := &internal.NodeStateMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStateMessage") + } + decodeNodeStateMessage(msg, mt) + return nil + case *pilosa.RecalculateCaches: + msg := &internal.RecalculateCaches{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling RecalculateCaches") + } + decodeRecalculateCaches(msg, mt) + return nil + case *pilosa.NodeEvent: + msg := &internal.NodeEventMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeEvent") + } + decodeNodeEventMessage(msg, mt) + return nil + case *pilosa.NodeStatus: + msg := &internal.NodeStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStatus") + } + decodeNodeStatus(msg, mt) + return nil + default: + panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) + } +} + +func encodeToProto(m pilosa.Message) proto.Message { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + return encodeCreateShardMessage(mt) + case *pilosa.CreateIndexMessage: + return encodeCreateIndexMessage(mt) + case *pilosa.DeleteIndexMessage: + return encodeDeleteIndexMessage(mt) + case *pilosa.CreateFieldMessage: + return encodeCreateFieldMessage(mt) + case *pilosa.DeleteFieldMessage: + return encodeDeleteFieldMessage(mt) + case *pilosa.CreateViewMessage: + return encodeCreateViewMessage(mt) + case *pilosa.DeleteViewMessage: + return encodeDeleteViewMessage(mt) + case *pilosa.ClusterStatus: + return encodeClusterStatus(mt) + case *pilosa.ResizeInstruction: + return encodeResizeInstruction(mt) + case *pilosa.ResizeInstructionComplete: + return encodeResizeInstructionComplete(mt) + case *pilosa.SetCoordinatorMessage: + return encodeSetCoordinatorMessage(mt) + case *pilosa.UpdateCoordinatorMessage: + return encodeUpdateCoordinatorMessage(mt) + case *pilosa.NodeStateMessage: + return encodeNodeStateMessage(mt) + case *pilosa.RecalculateCaches: + return encodeRecalculateCaches(mt) + case *pilosa.NodeEvent: + return encodeNodeEventMessage(mt) + case *pilosa.NodeStatus: + return encodeNodeStatus(mt) + } + return nil +} + +func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { + return &internal.ResizeInstruction{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Coordinator: EncodeNode(m.Coordinator), + Sources: encodeResizeSources(m.Sources), + Schema: encodeSchema(m.Schema), + ClusterStatus: encodeClusterStatus(m.ClusterStatus), + } +} + +func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource { + new := make([]*internal.ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, encodeResizeSource(src)) + } + return new +} + +func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource { + return &internal.ResizeSource{ + Node: EncodeNode(m.Node), + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + } +} + +func encodeSchema(m *pilosa.Schema) *internal.Schema { + return &internal.Schema{ + Indexes: encodeIndexInfos(m.Indexes), + } +} + +func encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index { + new := make([]*internal.Index, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, encodeIndexInfo(idx)) + } + return new +} + +func encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { + return &internal.Index{ + Name: idx.Name, + Fields: encodeFieldInfos(idx.Fields), + } +} + +func encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { + new := make([]*internal.Field, 0, len(fs)) + for _, f := range fs { + new = append(new, encodeFieldInfo(f)) + } + return new +} + +func encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { + ifield := &internal.Field{ + Name: f.Name, + Meta: encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), + } + + for _, viewinfo := range f.Views { + ifield.Views = append(ifield.Views, viewinfo.Name) + } + return ifield +} + +func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { + if o == nil { + return nil + } + return &internal.FieldOptions{ + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + } +} + +// EncodeNodes converts a slice of Nodes into its internal representation. +func EncodeNodes(a []*pilosa.Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = EncodeNode(a[i]) + } + return other +} + +// EncodeNode converts a Node into its internal representation. +func EncodeNode(n *pilosa.Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: n.URI.Encode(), + IsCoordinator: n.IsCoordinator, + } +} + +func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { + return &internal.ClusterStatus{ + State: m.State, + ClusterID: m.ClusterID, + Nodes: EncodeNodes(m.Nodes), + } +} + +func encodeCreateShardMessage(m *pilosa.CreateShardMessage) *internal.CreateShardMessage { + return &internal.CreateShardMessage{ + Index: m.Index, + Shard: m.Shard, + } +} + +func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { + return &internal.CreateIndexMessage{ + Index: m.Index, + Meta: encodeIndexMeta(m.Meta), + } +} + +func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta { + return &internal.IndexMeta{ + Keys: m.Keys, + } +} + +func encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *internal.DeleteIndexMessage { + return &internal.DeleteIndexMessage{ + Index: m.Index, + } +} + +func encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { + return &internal.CreateFieldMessage{ + Index: m.Index, + Field: m.Field, + Meta: encodeFieldOptions(m.Meta), + } +} + +func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFieldMessage { + return &internal.DeleteFieldMessage{ + Index: m.Index, + Field: m.Field, + } +} + +func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage { + return &internal.CreateViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMessage { + return &internal.DeleteViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete { + return &internal.ResizeInstructionComplete{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Error: m.Error, + } +} + +func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage { + return &internal.SetCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { + return &internal.UpdateCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessage { + return &internal.NodeStateMessage{ + NodeID: m.NodeID, + State: m.State, + } +} + +func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage { + return &internal.NodeEventMessage{ + Event: uint32(m.Event), + Node: EncodeNode(m.Node), + } +} + +func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus { + return &internal.NodeStatus{ + Node: EncodeNode(m.Node), + MaxShards: &internal.MaxShards{Standard: m.MaxShards}, + Schema: encodeSchema(m.Schema), + } +} + +func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCaches { + return &internal.RecalculateCaches{} +} + +func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { + m.JobID = ri.JobID + m.Node = &pilosa.Node{} + decodeNode(ri.Node, m.Node) + m.Coordinator = &pilosa.Node{} + decodeNode(ri.Coordinator, m.Coordinator) + m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) + decodeResizeSources(ri.Sources, m.Sources) + m.Schema = &pilosa.Schema{} + decodeSchema(ri.Schema, m.Schema) + m.ClusterStatus = &pilosa.ClusterStatus{} + decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus) +} + +func decodeResizeSources(srcs []*internal.ResizeSource, m []*pilosa.ResizeSource) { + for i := range srcs { + m[i] = &pilosa.ResizeSource{} + decodeResizeSource(srcs[i], m[i]) + } +} + +func decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) { + m.Node = &pilosa.Node{} + decodeNode(rs.Node, m.Node) + m.Index = rs.Index + m.Field = rs.Field + m.View = rs.View + m.Shard = rs.Shard +} + +func decodeSchema(s *internal.Schema, m *pilosa.Schema) { + m.Indexes = make([]*pilosa.IndexInfo, len(s.Indexes)) + decodeIndexes(s.Indexes, m.Indexes) +} + +func decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) { + for i := range idxs { + m[i] = &pilosa.IndexInfo{} + decodeIndex(idxs[i], m[i]) + } +} + +func decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { + m.Name = idx.Name + m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) + decodeFields(idx.Fields, m.Fields) +} + +func decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { + for i := range fs { + m[i] = &pilosa.FieldInfo{} + decodeField(fs[i], m[i]) + } +} + +func decodeField(f *internal.Field, m *pilosa.FieldInfo) { + m.Name = f.Name + m.Options = pilosa.FieldOptions{} + decodeFieldOptions(f.Meta, &m.Options) + m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) + for _, viewname := range f.Views { + m.Views = append(m.Views, &pilosa.ViewInfo{Name: viewname}) + } +} + +func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) { + m.Type = options.Type + m.CacheType = options.CacheType + m.CacheSize = options.CacheSize + m.Min = options.Min + m.Max = options.Max + m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) + m.Keys = options.Keys +} + +func decodeNodes(a []*internal.Node, m []*pilosa.Node) { + for i := range a { + m[i] = &pilosa.Node{} + decodeNode(a[i], m[i]) + } +} + +func decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) { + m.State = cs.State + m.ClusterID = cs.ClusterID + m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) + decodeNodes(cs.Nodes, m.Nodes) +} + +func decodeNode(node *internal.Node, m *pilosa.Node) { + m.ID = node.ID + decodeURI(node.URI, &m.URI) + m.IsCoordinator = node.IsCoordinator +} + +func decodeURI(i *internal.URI, m *pilosa.URI) { + m.Scheme = i.Scheme + m.Host = i.Host + m.Port = uint16(i.Port) +} + +func decodeCreateShardMessage(pb *internal.CreateShardMessage, m *pilosa.CreateShardMessage) { + m.Index = pb.Index + m.Shard = pb.Shard +} + +func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { + m.Index = pb.Index + m.Meta = &pilosa.IndexOptions{} + decodeIndexMeta(pb.Meta, m.Meta) +} + +func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) { + m.Keys = pb.Keys +} + +func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) { + m.Index = pb.Index +} + +func decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.Meta = &pilosa.FieldOptions{} + decodeFieldOptions(pb.Meta, m.Meta) +} + +func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field +} + +func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *pilosa.DeleteViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) { + m.JobID = pb.JobID + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.Error = pb.Error +} + +func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pilosa.NodeStateMessage) { + m.NodeID = pb.NodeID + m.State = pb.State +} + +func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) { + m.Event = pilosa.NodeEventType(pb.Event) + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) +} + +func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.MaxShards = pb.MaxShards.Standard + m.Schema = &pilosa.Schema{} + decodeSchema(pb.Schema, m.Schema) +} + +func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {} diff --git a/field.go b/field.go index eea6bb10e..fb7e9af4f 100644 --- a/field.go +++ b/field.go @@ -1077,13 +1077,13 @@ func (f *Field) MarshalJSON() ([]byte, error) { thing := struct { Name string Options FieldOptions - Views []*viewInfo + Views []*ViewInfo }{ Name: f.Name(), Options: f.Options(), } for _, viewname := range f.viewNames() { - thing.Views = append(thing.Views, &viewInfo{Name: viewname}) + thing.Views = append(thing.Views, &ViewInfo{Name: viewname}) } return json.Marshal(thing) } @@ -1117,7 +1117,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } type FieldInfo struct { Name string `json:"name"` Options FieldOptions `json:"options"` - Views []*viewInfo `json:"views,omitempty"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/gossip/gossip.go b/gossip/gossip.go index 3dc7c202b..cd1d9ad0b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -148,7 +148,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - host := api.Node().URI.Host() + host := api.Node().URI.GetHost() g := &GossipMemberSet{ papi: api, Logger: pilosa.NopLogger, @@ -193,10 +193,10 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO conf := memberlist.DefaultWANConfig() conf.Transport = g.transport.Net conf.Name = api.Node().ID - conf.BindAddr = api.Node().URI.Host() + conf.BindAddr = api.Node().URI.GetHost() conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = hostToIP(api.Node().URI.Host()) + conf.AdvertiseAddr = hostToIP(api.Node().URI.GetHost()) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult diff --git a/holder.go b/holder.go index 83ffd6967..10e147098 100644 --- a/holder.go +++ b/holder.go @@ -217,7 +217,7 @@ func (h *Holder) Schema() []*IndexInfo { for _, field := range index.Fields() { fi := &FieldInfo{Name: field.Name(), Options: field.Options()} for _, view := range field.views() { - fi.Views = append(fi.Views, &viewInfo{Name: view.name}) + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) diff --git a/http/client.go b/http/client.go index 27597a5cb..41602b778 100644 --- a/http/client.go +++ b/http/client.go @@ -31,6 +31,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + pilosaproto "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -43,6 +44,7 @@ type ClientOptions struct { // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { defaultURI *pilosa.URI + serializer pilosa.Serializer // The client to use for HTTP communication. HTTPClient *http.Client @@ -66,6 +68,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { return &InternalClient{ defaultURI: defaultURI, + serializer: pilosaproto.Serializer{}, HTTPClient: remoteClient, } } @@ -819,12 +822,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index } // SendMessage posts a message synchronously. -func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - +func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error { u := uriPathToURL(uri, "/internal/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) if err != nil { @@ -998,7 +996,7 @@ func pos(rowID, columnID uint64) uint64 { func uriPathToURL(uri *pilosa.URI, path string) url.URL { return url.URL{ - Scheme: uri.Scheme(), + Scheme: uri.GetScheme(), Host: uri.HostPort(), Path: path, } @@ -1006,7 +1004,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL { func nodePathToURL(node *pilosa.Node, path string) url.URL { return url.URL{ - Scheme: node.URI.Scheme(), + Scheme: node.URI.GetScheme(), Host: node.URI.HostPort(), Path: path, } diff --git a/server.go b/server.go index b57935acb..b3bc6e224 100644 --- a/server.go +++ b/server.go @@ -54,6 +54,7 @@ type Server struct { executor *executor hosts []string clusterDisabled bool + serializer Serializer // External systemInfo SystemInfo @@ -200,6 +201,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +func OptServerSerializer(ser Serializer) ServerOption { + return func(s *Server) error { + s.serializer = ser + return nil + } +} + func OptServerIsCoordinator(is bool) ServerOption { return func(s *Server) error { s.isCoordinator = is @@ -517,8 +525,12 @@ func (s *Server) receiveMessage(m Message) error { // SendSync represents an implementation of Broadcaster. func (s *Server) SendSync(m Message) error { - pb := encode(m) var eg errgroup.Group + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) for _, node := range s.cluster.Nodes { node := node s.logger.Printf("SendSync to: %s", node.URI) @@ -528,7 +540,7 @@ func (s *Server) SendSync(m Message) error { } eg.Go(func() error { - return s.defaultClient.SendMessage(context.Background(), &node.URI, pb) + return s.defaultClient.SendMessage(context.Background(), &node.URI, msg) }) } @@ -542,9 +554,13 @@ func (s *Server) SendAsync(m Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, m Message) error { - pb := encode(m) s.logger.Printf("SendTo: %s", to.URI) - return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) + return s.defaultClient.SendMessage(context.Background(), &to.URI, msg) } // node returns the pilosa.node object. It is used by membership protocols to @@ -613,7 +629,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) - s.diagnostics.Set("Host", s.uri.host) + s.diagnostics.Set("Host", s.uri.Host) s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) diff --git a/server/server.go b/server/server.go index c8de0664c..f46f6a64a 100644 --- a/server/server.go +++ b/server/server.go @@ -35,6 +35,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" @@ -202,7 +203,7 @@ func (m *Command) SetupServer() error { // Setup TLS var TLSConfig *tls.Config - if uri.Scheme() == "https" { + if uri.GetScheme() == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") } @@ -235,7 +236,7 @@ func (m *Command) SetupServer() error { } // If port is 0, get auto-allocated port from listener - if uri.Port() == 0 { + if uri.GetPort() == 0 { uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) } @@ -271,6 +272,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerSerializer(proto.Serializer{}), coordinatorOpt, } @@ -309,7 +311,7 @@ func (m *Command) SetupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.API.Node().URI.Host() + gossipHost := m.API.Node().URI.GetHost() m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return errors.Wrap(err, "getting transport") @@ -366,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { // getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS - if uri.Scheme() == "https" && tlsconf != nil { + if uri.GetScheme() == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) if err != nil { return nil, errors.Wrap(err, "tls.Listener") } - } else if uri.Scheme() == "http" { + } else if uri.GetScheme() == "http" { // Open HTTP listener to determine port (if specified as :0). ln, err = net.Listen("tcp", uri.HostPort()) if err != nil { return nil, errors.Wrap(err, "net.Listen") } } else { - return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + return nil, errors.Errorf("unsupported scheme: %s", uri.GetScheme()) } return ln, nil diff --git a/uri.go b/uri.go index 5d823d8b6..2058f70fa 100644 --- a/uri.go +++ b/uri.go @@ -43,17 +43,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a- // localhost // :10101 type URI struct { - scheme string `json:"scheme"` - host string `json:"host"` - port uint16 `json:"port"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` } // DefaultURI creates and returns the default URI. func DefaultURI() *URI { return &URI{ - scheme: "http", - host: "localhost", - port: 10101, + Scheme: "http", + Host: "localhost", + Port: 10101, } } @@ -83,9 +83,9 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// Scheme returns the scheme of this URI. -func (u *URI) Scheme() string { - return u.scheme +// GetScheme returns the scheme of this URI. +func (u *URI) GetScheme() string { + return u.Scheme } // SetScheme sets the scheme of this URI. @@ -94,13 +94,13 @@ func (u *URI) SetScheme(scheme string) error { if m == nil { return errors.New("invalid scheme") } - u.scheme = scheme + u.Scheme = scheme return nil } -// Host returns the host of this URI. -func (u *URI) Host() string { - return u.host +// GetHost returns the host of this URI. +func (u *URI) GetHost() string { + return u.Host } // SetHost sets the host of this URI. @@ -109,18 +109,18 @@ func (u *URI) SetHost(host string) error { if m == nil { return errors.New("invalid host") } - u.host = host + u.Host = host return nil } -// Port returns the port of this URI. -func (u *URI) Port() uint16 { - return u.port +// GetPort returns the port of this URI. +func (u *URI) GetPort() uint16 { + return u.Port } // SetPort sets the port of this URI. func (u *URI) SetPort(port uint16) { - u.port = port + u.Port = port } // HostPort returns `Host:Port` @@ -129,23 +129,23 @@ func (u *URI) HostPort() string { if u == nil { return "" } - s := fmt.Sprintf("%s:%d", u.host, u.port) + s := fmt.Sprintf("%s:%d", u.Host, u.Port) return s } // Normalize returns the address in a form usable by a HTTP client. func (u *URI) Normalize() string { - scheme := u.scheme + scheme := u.Scheme index := strings.Index(scheme, "+") if index >= 0 { scheme = scheme[:index] } - return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port) + return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port) } // String returns the address as a string. func (u URI) String() string { - return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port) + return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port) } // Equals returns true if the checked URI is equivalent to this URI. @@ -199,9 +199,9 @@ func parseAddress(address string) (uri *URI, err error) { } } uri = &URI{ - scheme: scheme, - host: host, - port: uint16(port), + Scheme: scheme, + Host: host, + Port: uint16(port), } return uri, nil } @@ -213,9 +213,9 @@ func (u URI) Encode() *internal.URI { func encodeURI(u URI) *internal.URI { return &internal.URI{ - Scheme: u.scheme, - Host: u.host, - Port: uint32(u.port), + Scheme: u.Scheme, + Host: u.Host, + Port: uint32(u.Port), } } @@ -228,9 +228,9 @@ func decodeURI(i *internal.URI) URI { return URI{} } return URI{ - scheme: i.Scheme, - host: i.Host, - port: uint16(i.Port), + Scheme: i.Scheme, + Host: i.Host, + Port: uint16(i.Port), } } @@ -241,9 +241,9 @@ func (u *URI) MarshalJSON() ([]byte, error) { Host string `json:"host,omitempty"` Port uint16 `json:"port,omitempty"` } - output.Scheme = u.scheme - output.Host = u.host - output.Port = u.port + output.Scheme = u.Scheme + output.Host = u.Host + output.Port = u.Port return json.Marshal(output) } @@ -257,8 +257,8 @@ func (u *URI) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &input); err != nil { return err } - u.scheme = input.Scheme - u.host = input.Host - u.port = input.Port + u.Scheme = input.Scheme + u.Host = input.Host + u.Port = input.Port return nil } diff --git a/uri_internal_test.go b/uri_internal_test.go index dbcbfa04d..2aac9651f 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -93,8 +93,8 @@ func TestSetScheme(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.Scheme() != target { - t.Fatalf("%s != %s", uri.Scheme(), target) + if uri.GetScheme() != target { + t.Fatalf("%s != %s", uri.GetScheme(), target) } } @@ -105,8 +105,8 @@ func TestSetHost(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.Host() != target { - t.Fatalf("%s != %s", uri.host, target) + if uri.GetHost() != target { + t.Fatalf("%s != %s", uri.Host, target) } } @@ -114,8 +114,8 @@ func TestSetPort(t *testing.T) { uri := DefaultURI() target := uint16(9999) uri.SetPort(target) - if uri.Port() != target { - t.Fatalf("%d != %d", uri.port, target) + if uri.GetPort() != target { + t.Fatalf("%d != %d", uri.Port, target) } } @@ -147,14 +147,14 @@ func TestHostPort(t *testing.T) { } func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { - if uri.Scheme() != scheme { - t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme) + if uri.GetScheme() != scheme { + t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme) } - if uri.Host() != host { - t.Fatalf("Host does not match: %s != %s", uri.host, host) + if uri.GetHost() != host { + t.Fatalf("Host does not match: %s != %s", uri.Host, host) } - if uri.Port() != port { - t.Fatalf("Port does not match: %d != %d", uri.port, port) + if uri.GetPort() != port { + t.Fatalf("Port does not match: %d != %d", uri.Port, port) } } diff --git a/view.go b/view.go index 0f2e189cd..609664304 100644 --- a/view.go +++ b/view.go @@ -421,12 +421,12 @@ func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (* return r, nil } -// viewInfo represents schema information for a view. -type viewInfo struct { +// ViewInfo represents schema information for a view. +type ViewInfo struct { Name string `json:"name"` } -type viewInfoSlice []*viewInfo +type viewInfoSlice []*ViewInfo func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p viewInfoSlice) Len() int { return len(p) } From 6309d3b7f79a3b2a222e2feb30e754076aa82f7d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 12:32:49 -0500 Subject: [PATCH 06/18] get gossip using serializer stuff, remove proto and internal --- api.go | 4 ++++ broadcast.go | 48 +++++--------------------------------- broadcast_test.go | 51 ----------------------------------------- encoding/proto/proto.go | 32 +++++++++++++++++--------- gossip/gossip.go | 49 ++++++++++++++++++++------------------- 5 files changed, 57 insertions(+), 127 deletions(-) delete mode 100644 broadcast_test.go diff --git a/api.go b/api.go index 09f0531d5..cae91b20f 100644 --- a/api.go +++ b/api.go @@ -149,6 +149,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return resp, nil } +func (api *API) Holder() *Holder { + return api.server.Holder() +} + // readColumnAttrSets returns a list of column attribute objects by id. func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { if index == nil { diff --git a/broadcast.go b/broadcast.go index 1da6ca57f..00835db64 100644 --- a/broadcast.go +++ b/broadcast.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "reflect" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -78,48 +77,13 @@ const ( messageTypeNodeStatus ) -// MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(m proto.Message) ([]byte, error) { - var typ uint8 - switch obj := m.(type) { - case *internal.CreateShardMessage: - typ = messageTypeCreateShard - case *internal.CreateIndexMessage: - typ = messageTypeCreateIndex - case *internal.DeleteIndexMessage: - typ = messageTypeDeleteIndex - case *internal.CreateFieldMessage: - typ = messageTypeCreateField - case *internal.DeleteFieldMessage: - typ = messageTypeDeleteField - case *internal.CreateViewMessage: - typ = messageTypeCreateView - case *internal.DeleteViewMessage: - typ = messageTypeDeleteView - case *internal.ClusterStatus: - typ = messageTypeClusterStatus - case *internal.ResizeInstruction: - typ = messageTypeResizeInstruction - case *internal.ResizeInstructionComplete: - typ = messageTypeResizeInstructionComplete - case *internal.SetCoordinatorMessage: - typ = messageTypeSetCoordinator - case *internal.UpdateCoordinatorMessage: - typ = messageTypeUpdateCoordinator - case *internal.NodeStateMessage: - typ = messageTypeNodeState - case *internal.RecalculateCaches: - typ = messageTypeRecalculateCaches - case *internal.NodeEventMessage: - typ = messageTypeNodeEvent - case *internal.NodeStatus: - typ = messageTypeNodeStatus - default: - return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) - } - buf, err := proto.Marshal(m) +// MarshalInternalMessage serializes the pilosa message and adds pilosa internal +// type info which is used by the internal messaging stuff. +func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) { + typ := getMessageType(m) + buf, err := s.Marshal(m) if err != nil { - return nil, errors.Wrap(err, "marshalling") + return nil, errors.Wrap(err, "marshaling") } return append([]byte{typ}, buf...), nil } diff --git a/broadcast_test.go b/broadcast_test.go deleted file mode 100644 index 415228718..000000000 --- a/broadcast_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "reflect" - "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" -) - -// Ensure a message can be marshaled and unmarshaled. -func TestMessage_Marshal(t *testing.T) { - - testMessageMarshal(t, &internal.CreateShardMessage{ - Index: "i", - Shard: 8, - }) - - testMessageMarshal(t, &internal.DeleteIndexMessage{ - Index: "i", - }) -} - -func testMessageMarshal(t *testing.T, m proto.Message) { - marshalled, err := pilosa.MarshalMessage(m) - if err != nil { - t.Fatal(err) - } - unmarshalled, err := pilosa.UnmarshalMessage(marshalled) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(unmarshalled, m) { - t.Fatalf("unexpected message marshalling: %s", unmarshalled) - } -} diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 1c83ba4ea..10cc13dc4 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -153,6 +153,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeNodeStatus(msg, mt) return nil + case *pilosa.Node: + msg := &internal.Node{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling Node") + } + decodeNode(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -192,6 +200,8 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeNodeEventMessage(mt) case *pilosa.NodeStatus: return encodeNodeStatus(mt) + case *pilosa.Node: + return encodeNode(mt) } return nil } @@ -199,8 +209,8 @@ func encodeToProto(m pilosa.Message) proto.Message { func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { return &internal.ResizeInstruction{ JobID: m.JobID, - Node: EncodeNode(m.Node), - Coordinator: EncodeNode(m.Coordinator), + Node: encodeNode(m.Node), + Coordinator: encodeNode(m.Coordinator), Sources: encodeResizeSources(m.Sources), Schema: encodeSchema(m.Schema), ClusterStatus: encodeClusterStatus(m.ClusterStatus), @@ -217,7 +227,7 @@ func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource { func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource { return &internal.ResizeSource{ - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), Index: m.Index, Field: m.Field, View: m.View, @@ -286,13 +296,13 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { func EncodeNodes(a []*pilosa.Node) []*internal.Node { other := make([]*internal.Node, len(a)) for i := range a { - other[i] = EncodeNode(a[i]) + other[i] = encodeNode(a[i]) } return other } -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *pilosa.Node) *internal.Node { +// encodeNode converts a Node into its internal representation. +func encodeNode(n *pilosa.Node) *internal.Node { return &internal.Node{ ID: n.ID, URI: n.URI.Encode(), @@ -368,20 +378,20 @@ func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMe func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete { return &internal.ResizeInstructionComplete{ JobID: m.JobID, - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), Error: m.Error, } } func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage { return &internal.SetCoordinatorMessage{ - New: EncodeNode(m.New), + New: encodeNode(m.New), } } func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { return &internal.UpdateCoordinatorMessage{ - New: EncodeNode(m.New), + New: encodeNode(m.New), } } @@ -395,13 +405,13 @@ func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessa func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage { return &internal.NodeEventMessage{ Event: uint32(m.Event), - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), } } func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus { return &internal.NodeStatus{ - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), MaxShards: &internal.MaxShards{Standard: m.MaxShards}, Schema: encodeSchema(m.Schema), } diff --git a/gossip/gossip.go b/gossip/gossip.go index cd1d9ad0b..251851077 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -26,10 +26,9 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -44,8 +43,9 @@ type GossipMemberSet struct { broadcasts *memberlist.TransmitLimitedQueue - papi *pilosa.API - config *gossipConfig + papi *pilosa.API + serializer pilosa.Serializer + config *gossipConfig Logger pilosa.Logger @@ -150,8 +150,9 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { host := api.Node().URI.GetHost() g := &GossipMemberSet{ - papi: api, - Logger: pilosa.NopLogger, + papi: api, + serializer: proto.Serializer{}, + Logger: pilosa.NopLogger, } // options @@ -222,7 +223,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := proto.Marshal(pilosa.EncodeNode(g.papi.Node())) + buf, err := g.serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} @@ -248,14 +249,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. func (g *GossipMemberSet) LocalState(join bool) []byte { - pb := &internal.NodeStatus{ - Node: pilosa.EncodeNode(g.papi.Node()), - MaxShards: &internal.MaxShards{Standard: g.papi.MaxShards(context.Background())}, - Schema: &internal.Schema{Indexes: pilosa.EncodeIndexes(g.papi.Schema(context.Background()))}, + m := &pilosa.NodeStatus{ + Node: g.papi.Node(), + MaxShards: g.papi.MaxShards(context.Background()), + Schema: &pilosa.Schema{Indexes: g.papi.Holder().Schema()}, } // Marshal nodestate data to bytes. - buf, err := pilosa.MarshalMessage(pb) + buf, err := pilosa.MarshalInternalMessage(m, g.serializer) if err != nil { g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} @@ -278,8 +279,9 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type gossipEventReceiver struct { - ch chan memberlist.NodeEvent - papi *pilosa.API + ch chan memberlist.NodeEvent + papi *pilosa.API + serializer pilosa.Serializer logger *log.Logger } @@ -287,9 +289,10 @@ type gossipEventReceiver struct { // newGossipEventReceiver returns a new instance of GossipEventReceiver. func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { ger := &gossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - logger: logger, - papi: papi, + ch: make(chan memberlist.NodeEvent, 1), + logger: logger, + papi: papi, + serializer: proto.Serializer{}, } go ger.listen() return ger @@ -323,16 +326,16 @@ func (g *gossipEventReceiver) listen() { } // Get the node from the event.Node meta data. - var n internal.Node - if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { - panic("failed to unmarshal event node meta data") + var n pilosa.Node + if err := g.serializer.Unmarshal(e.Node.Meta, &n); err != nil { + panic("failed to unmarshal event node meta into node") } - ne := &internal.NodeEventMessage{ - Event: uint32(nodeEventType), + ne := &pilosa.NodeEvent{ + Event: nodeEventType, Node: &n, } - buf, err := pilosa.MarshalMessage(ne) + buf, err := pilosa.MarshalInternalMessage(ne, g.serializer) if err != nil { panic(err) } From 59e80f9692aa2c687cd9365e61af6802bf8fccb6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 13:35:14 -0500 Subject: [PATCH 07/18] put Serializer on API, add QueryRequest/Response to serializer --- api.go | 3 + encoding/proto/proto.go | 288 ++++++++++++++++++++++++++++++++++++++++ gossip/gossip.go | 31 ++--- http/handler.go | 13 +- 4 files changed, 310 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index cae91b20f..a9bd02f2d 100644 --- a/api.go +++ b/api.go @@ -38,6 +38,8 @@ type API struct { holder *Holder cluster *cluster server *Server + + Serializer Serializer } // APIOption is a functional option type for pilosa.API @@ -48,6 +50,7 @@ func OptAPIServer(s *Server) APIOption { a.server = s a.holder = s.holder a.cluster = s.cluster + a.Serializer = s.serializer return nil } } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 10cc13dc4..0f2574f1b 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -2,6 +2,7 @@ package proto import ( "fmt" + "sort" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" @@ -161,6 +162,23 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeNode(msg, mt) return nil + case *pilosa.QueryRequest: + msg := &internal.QueryRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryRequest") + } + decodeQueryRequest(msg, mt) + return nil + case *pilosa.QueryResponse: + msg := &internal.QueryResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryResponse") + } + decodeQueryResponse(msg, mt) + return nil + default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -202,10 +220,62 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeNodeStatus(mt) case *pilosa.Node: return encodeNode(mt) + case *pilosa.QueryRequest: + return encodeQueryRequest(mt) + case *pilosa.QueryResponse: + return encodeQueryResponse(mt) } return nil } +func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { + return &internal.QueryRequest{ + Query: m.Query, + Shards: m.Shards, + ColumnAttrs: m.ColumnAttrs, + Remote: m.Remote, + ExcludeRowAttrs: m.ExcludeRowAttrs, + ExcludeColumns: m.ExcludeColumns, + } +} + +func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(m.Results)), + ColumnAttrSets: EncodeColumnAttrSets(m.ColumnAttrSets), + } + + for i := range m.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := m.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = queryResultTypeRow + pb.Results[i].Row = EncodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = queryResultTypePairs + pb.Results[i].Pairs = EncodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = queryResultTypeValCount + pb.Results[i].ValCount = EncodeValCount(result) + case uint64: + pb.Results[i].Type = queryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = queryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = queryResultTypeNil + } + } + + if m.Err != nil { + pb.Err = m.Err.Error() + } + + return pb +} + func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { return &internal.ResizeInstruction{ JobID: m.JobID, @@ -602,3 +672,221 @@ func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { } func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {} + +func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { + m.Query = pb.Query + m.Shards = pb.Shards + m.ColumnAttrs = pb.ColumnAttrs + m.Remote = pb.Remote + m.ExcludeRowAttrs = pb.ExcludeRowAttrs + m.ExcludeColumns = pb.ExcludeColumns +} + +func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { + m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) + decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) + m.Err = errors.New(pb.Err) + m.Results = make([]interface{}, len(pb.Results)) + decodeQueryResults(pb.Results, m.Results) + +} + +func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { + for i := range pb { + decodeColumnAttrSet(pb[i], m[i]) + } +} + +func decodeColumnAttrSet(pb *internal.ColumnAttrSet, m *pilosa.ColumnAttrSet) { + m.ID = pb.ID + m.Key = pb.Key + m.Attrs = decodeAttrs(pb.Attrs) +} + +func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) { + for i := range pb { + m[i] = decodeQueryResult(pb[i]) + } +} + +// QueryResult types. +const ( + queryResultTypeNil uint32 = iota + queryResultTypeRow + queryResultTypePairs + queryResultTypeValCount + queryResultTypeUint64 + queryResultTypeBool +) + +func decodeQueryResult(pb *internal.QueryResult) interface{} { + switch pb.Type { + case queryResultTypeRow: + return decodeRow(pb.Row) + case queryResultTypePairs: + return decodePairs(pb.Pairs) + case queryResultTypeValCount: + return decodeValCount(pb.ValCount) + case queryResultTypeUint64: + return pb.N + case queryResultTypeBool: + return pb.Changed + case queryResultTypeNil: + return nil + } + panic(fmt.Sprintf("unknown type: %d", pb.Type)) +} + +// DecodeRow converts r from its internal representation. +func decodeRow(pr *internal.Row) *pilosa.Row { + if pr == nil { + return nil + } + + r := pilosa.NewRow() + r.Attrs = decodeAttrs(pr.Attrs) + for _, v := range pr.Columns { + r.SetBit(v) + } + return r +} + +func decodeAttrs(pb []*internal.Attr) map[string]interface{} { + m := make(map[string]interface{}, len(pb)) + for i := range pb { + key, value := decodeAttr(pb[i]) + m[key] = value + } + return m +} + +const ( + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 +) + +func decodeAttr(attr *internal.Attr) (key string, value interface{}) { + switch attr.Type { + case attrTypeString: + return attr.Key, attr.StringValue + case attrTypeInt: + return attr.Key, attr.IntValue + case attrTypeBool: + return attr.Key, attr.BoolValue + case attrTypeFloat: + return attr.Key, attr.FloatValue + default: + return attr.Key, nil + } +} + +func decodePairs(a []*internal.Pair) []pilosa.Pair { + other := make([]pilosa.Pair, len(a)) + for i := range a { + other[i] = decodePair(a[i]) + } + return other +} + +func decodePair(pb *internal.Pair) pilosa.Pair { + return pilosa.Pair{ + ID: pb.ID, + Key: pb.Key, + Count: pb.Count, + } +} + +func decodeValCount(pb *internal.ValCount) pilosa.ValCount { + return pilosa.ValCount{ + Val: pb.Val, + Count: pb.Count, + } +} + +func EncodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { + other := make([]*internal.ColumnAttrSet, len(a)) + for i := range a { + other[i] = EncodeColumnAttrSet(a[i]) + } + return other +} + +func EncodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { + return &internal.ColumnAttrSet{ + ID: set.ID, + Attrs: encodeAttrs(set.Attrs), + } +} + +func EncodeRow(r *pilosa.Row) *internal.Row { + if r == nil { + return nil + } + + return &internal.Row{ + Columns: r.Columns(), + Attrs: encodeAttrs(r.Attrs), + } +} + +func EncodePairs(a pilosa.Pairs) []*internal.Pair { + other := make([]*internal.Pair, len(a)) + for i := range a { + other[i] = encodePair(a[i]) + } + return other +} + +func encodePair(p pilosa.Pair) *internal.Pair { + return &internal.Pair{ + ID: p.ID, + Key: p.Key, + Count: p.Count, + } +} + +func EncodeValCount(vc pilosa.ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} + +func encodeAttrs(m map[string]interface{}) []*internal.Attr { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + a := make([]*internal.Attr, len(keys)) + for i := range keys { + a[i] = encodeAttr(keys[i], m[keys[i]]) + } + return a +} + +// encodeAttr converts a key/value pair into an Attr internal representation. +func encodeAttr(key string, value interface{}) *internal.Attr { + pb := &internal.Attr{Key: key} + switch value := value.(type) { + case string: + pb.Type = attrTypeString + pb.StringValue = value + case float64: + pb.Type = attrTypeFloat + pb.FloatValue = value + case uint64: + pb.Type = attrTypeInt + pb.IntValue = int64(value) + case int64: + pb.Type = attrTypeInt + pb.IntValue = value + case bool: + pb.Type = attrTypeBool + pb.BoolValue = value + } + return pb +} diff --git a/gossip/gossip.go b/gossip/gossip.go index 251851077..28b3ac690 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -28,7 +28,6 @@ import ( "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -43,9 +42,8 @@ type GossipMemberSet struct { broadcasts *memberlist.TransmitLimitedQueue - papi *pilosa.API - serializer pilosa.Serializer - config *gossipConfig + papi *pilosa.API + config *gossipConfig Logger pilosa.Logger @@ -150,9 +148,8 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { host := api.Node().URI.GetHost() g := &GossipMemberSet{ - papi: api, - serializer: proto.Serializer{}, - Logger: pilosa.NopLogger, + papi: api, + Logger: pilosa.NopLogger, } // options @@ -223,7 +220,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := g.serializer.Marshal(g.papi.Node()) + buf, err := g.papi.Serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} @@ -256,7 +253,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte { } // Marshal nodestate data to bytes. - buf, err := pilosa.MarshalInternalMessage(m, g.serializer) + buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer) if err != nil { g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} @@ -279,9 +276,8 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type gossipEventReceiver struct { - ch chan memberlist.NodeEvent - papi *pilosa.API - serializer pilosa.Serializer + ch chan memberlist.NodeEvent + papi *pilosa.API logger *log.Logger } @@ -289,10 +285,9 @@ type gossipEventReceiver struct { // newGossipEventReceiver returns a new instance of GossipEventReceiver. func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { ger := &gossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - logger: logger, - papi: papi, - serializer: proto.Serializer{}, + ch: make(chan memberlist.NodeEvent, 1), + logger: logger, + papi: papi, } go ger.listen() return ger @@ -327,7 +322,7 @@ func (g *gossipEventReceiver) listen() { // Get the node from the event.Node meta data. var n pilosa.Node - if err := g.serializer.Unmarshal(e.Node.Meta, &n); err != nil { + if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil { panic("failed to unmarshal event node meta into node") } @@ -335,7 +330,7 @@ func (g *gossipEventReceiver) listen() { Event: nodeEventType, Node: &n, } - buf, err := pilosa.MarshalInternalMessage(ne, g.serializer) + buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer) if err != nil { panic(err) } diff --git a/http/handler.go b/http/handler.go index 9195f05e9..84832b1d4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -815,13 +815,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques return nil, errors.Wrap(err, "reading") } - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") + qreq := &pilosa.QueryRequest{} + err = h.API.Serializer.Unmarshal(body, qreq) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling query request") } - - return decodeQueryRequest(&req), nil + return qreq, nil } // readURLQueryRequest parses query parameters from URL parameters from r. @@ -860,7 +859,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res // writeProtobufQueryResponse writes the response from the executor to w as protobuf. func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { + if buf, err := h.API.Serializer.Marshal(resp); err != nil { return errors.Wrap(err, "marshalling") } else if _, err := w.Write(buf); err != nil { return errors.Wrap(err, "writing") From e76a90e69b5241d40ea20b56d80e82554428dc91 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 15:02:33 -0500 Subject: [PATCH 08/18] change Query and QueryNode to use pilosa.* Query structs --- cache.go | 8 ------- client.go | 14 +++++------- encoding/proto/proto.go | 6 ++++- executor.go | 47 ++++++-------------------------------- executor_test.go | 2 +- fragment.go | 2 +- http/client.go | 17 +++++++------- http/client_test.go | 9 ++++---- http/handler.go | 50 ----------------------------------------- 9 files changed, 32 insertions(+), 123 deletions(-) diff --git a/cache.go b/cache.go index 192f38cc7..4c826f5d7 100644 --- a/cache.go +++ b/cache.go @@ -409,14 +409,6 @@ func (p Pairs) String() string { return buf.String() } -func EncodePairs(a Pairs) []*internal.Pair { - other := make([]*internal.Pair, len(a)) - for i := range a { - other[i] = encodePair(a[i]) - } - return other -} - func decodePairs(a []*internal.Pair) []Pair { other := make([]Pair, len(a)) for i := range a { diff --git a/client.go b/client.go index 01e0b847a..5c51ae63f 100644 --- a/client.go +++ b/client.go @@ -3,8 +3,6 @@ package pilosa import ( "context" "io" - - "github.com/pilosa/pilosa/internal" ) // Bit represents the intersection of a row and a column. It can be specifed by @@ -35,8 +33,8 @@ type InternalClient interface { Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) - Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error @@ -55,12 +53,12 @@ type InternalClient interface { //=============== type InternalQueryClient interface { - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) } type NopInternalQueryClient struct{} -func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } @@ -90,10 +88,10 @@ func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt In func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { return nil, nil } -func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 0f2574f1b..341ad46f7 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -685,7 +685,11 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) - m.Err = errors.New(pb.Err) + if pb.Err == "" { + m.Err = nil + } else { + m.Err = errors.New(pb.Err) + } m.Results = make([]interface{}, len(pb.Results)) decodeQueryResults(pb.Results, m.Results) diff --git a/executor.go b/executor.go index 1a81e92bd..aa2c2fb88 100644 --- a/executor.go +++ b/executor.go @@ -337,7 +337,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrap(err, "map reduce") } // Attach attributes for Row() calls. @@ -1392,7 +1392,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p // exec executes a PQL query remotely for a set of shards on a node. func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) { // Encode request object. - pbreq := &internal.QueryRequest{ + pbreq := &QueryRequest{ Query: q.String(), Shards: shards, Remote: true, @@ -1403,40 +1403,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q * return nil, err } - // Return an error, if specified on response. - if err := decodeError(pb.Err); err != nil { - return nil, err - } - - // Return appropriate data for the query. - results = make([]interface{}, len(q.Calls)) - for i, call := range q.Calls { - var v interface{} - var err error - - switch call.Name { - case "Average", "Sum": - v, err = decodeValCount(pb.Results[i].GetValCount()), nil - case "TopN": - v, err = decodePairs(pb.Results[i].GetPairs()), nil - case "Count": - v, err = pb.Results[i].N, nil - case "Set": - v, err = pb.Results[i].Changed, nil - case "Clear": - v, err = pb.Results[i].Changed, nil - case "SetRowAttrs": - case "SetColumnAttrs": - default: - v, err = DecodeRow(pb.Results[i].GetRow()), nil - } - if err != nil { - return nil, err - } - - results[i] = v - } - return results, nil + return pb.Results, pb.Err } // shardsByNode returns a mapping of nodes to shards. @@ -1490,7 +1457,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, for { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, errors.Wrap(ctx.Err(), "context done") case resp := <-ch: // On error retry against remaining nodes. If an error returns then // the context will cancel and cause all open goroutines to return. @@ -1500,10 +1467,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { return nil, resp.err } else if err != nil { - return nil, err + return nil, errors.Wrap(err, "calling mapper") } continue } @@ -1524,7 +1491,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return err + return errors.Wrap(err, "shards by node") } // Execute each node in a separate goroutine. diff --git a/executor_test.go b/executor_test.go index f54fb7e33..fc72b00f3 100644 --- a/executor_test.go +++ b/executor_test.go @@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) { hldr.SetBit("i", "f", 1, 0) if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { - t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err)) + t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err) } } diff --git a/fragment.go b/fragment.go index b502e1597..1ae3c5bb3 100644 --- a/fragment.go +++ b/fragment.go @@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { } // Execute query. - queryRequest := &internal.QueryRequest{ + queryRequest := &QueryRequest{ Query: buffers[k].String(), Remote: true, } diff --git a/http/client.go b/http/client.go index 41602b778..d8fb7f7ef 100644 --- a/http/client.go +++ b/http/client.go @@ -220,22 +220,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard } // Query executes query against the index. -func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { if index == "" { return nil, pilosa.ErrIndexRequired } else if queryRequest.Query == "" { return nil, pilosa.ErrQueryRequired } - // Encode request object. - buf, err := proto.Marshal(queryRequest) + buf, err := c.serializer.Marshal(queryRequest) if err != nil { - return nil, errors.Wrap(err, "marshaling") + return nil, errors.Wrap(err, "marshaling queryRequest") } // Create HTTP request. @@ -265,11 +264,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s return nil, errors.New(string(body)) } - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { + qresp := &pilosa.QueryResponse{} + if err := c.serializer.Unmarshal(body, qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) + } else if qresp.Err != nil { + return nil, qresp.Err } return qresp, nil diff --git a/http/client_test.go b/http/client_test.go index fb1105a27..38a677673 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -24,7 +24,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" @@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) { client[2] = MustNewClient(c[2].URL(), defaultClient) topN := 4 - queryRequest := &internal.QueryRequest{ + queryRequest := &pilosa.QueryRequest{ Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), Remote: false, } @@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) { } // Test must return exactly N results. - if len(result.Results[0].Pairs) != topN { + if len(result.Results[0].([]pilosa.Pair)) != topN { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } - p := []*internal.Pair{ + p := []pilosa.Pair{ {ID: 100, Count: 12}, {ID: 22, Count: 10}, {ID: 98, Count: 8}, {ID: 99, Count: 7}} // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.Results[0].Pairs, p) { + if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } diff --git a/http/handler.go b/http/handler.go index 84832b1d4..33085879c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1102,56 +1102,6 @@ const ( QueryResultTypeBool ) -func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { - req := &pilosa.QueryRequest{ - Query: pb.Query, - Shards: pb.Shards, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - -func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *pilosa.Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = pilosa.EncodeRow(result) - case []pilosa.Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = pilosa.EncodePairs(result) - case pilosa.ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = pilosa.EncodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb -} - // parseUint64Slice returns a slice of uint64s from a comma-delimited string. func parseUint64Slice(s string) ([]uint64, error) { var a []uint64 From bbb93abd57cd1f0a7195d504f5257c391e247b78 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 15:35:32 -0500 Subject: [PATCH 09/18] remove lots of unused code --- api.go | 4 +- broadcast.go | 87 --------- cluster.go | 410 ---------------------------------------- encoding/proto/proto.go | 10 +- executor.go | 15 -- field.go | 34 ---- holder.go | 15 -- index.go | 24 --- row.go | 27 --- uri.go | 29 --- 10 files changed, 11 insertions(+), 644 deletions(-) diff --git a/api.go b/api.go index a9bd02f2d..1abe4502c 100644 --- a/api.go +++ b/api.go @@ -770,8 +770,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // Send the set-coordinator message to new node. err = api.server.SendTo( newNode, - &internal.SetCoordinatorMessage{ - New: EncodeNode(newNode), + &SetCoordinatorMessage{ + New: newNode, }) if err != nil { return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) diff --git a/broadcast.go b/broadcast.go index 00835db64..a3ea01a4f 100644 --- a/broadcast.go +++ b/broadcast.go @@ -17,8 +17,6 @@ package pilosa import ( "fmt" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -165,88 +163,3 @@ func getMessageType(m Message) byte { panic(fmt.Sprintf("don't have type for message %#v", m)) } } - -// UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (proto.Message, error) { - typ, buf := buf[0], buf[1:] - var m proto.Message - switch typ { - case messageTypeCreateShard: - m = &internal.CreateShardMessage{} - case messageTypeCreateIndex: - m = &internal.CreateIndexMessage{} - case messageTypeDeleteIndex: - m = &internal.DeleteIndexMessage{} - case messageTypeCreateField: - m = &internal.CreateFieldMessage{} - case messageTypeDeleteField: - m = &internal.DeleteFieldMessage{} - case messageTypeCreateView: - m = &internal.CreateViewMessage{} - case messageTypeDeleteView: - m = &internal.DeleteViewMessage{} - case messageTypeClusterStatus: - m = &internal.ClusterStatus{} - case messageTypeResizeInstruction: - m = &internal.ResizeInstruction{} - case messageTypeResizeInstructionComplete: - m = &internal.ResizeInstructionComplete{} - case messageTypeSetCoordinator: - m = &internal.SetCoordinatorMessage{} - case messageTypeUpdateCoordinator: - m = &internal.UpdateCoordinatorMessage{} - case messageTypeNodeState: - m = &internal.NodeStateMessage{} - case messageTypeRecalculateCaches: - m = &internal.RecalculateCaches{} - case messageTypeNodeEvent: - m = &internal.NodeEventMessage{} - case messageTypeNodeStatus: - m = &internal.NodeStatus{} - default: - return nil, fmt.Errorf("invalid message type: %d", typ) - } - - if err := proto.Unmarshal(buf, m); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - return m, nil -} - -func decode(m proto.Message) Message { - switch mt := m.(type) { - case *internal.CreateShardMessage: - return decodeCreateShardMessage(mt) - case *internal.CreateIndexMessage: - return decodeCreateIndexMessage(mt) - case *internal.DeleteIndexMessage: - return decodeDeleteIndexMessage(mt) - case *internal.CreateFieldMessage: - return decodeCreateFieldMessage(mt) - case *internal.DeleteFieldMessage: - return decodeDeleteFieldMessage(mt) - case *internal.CreateViewMessage: - return decodeCreateViewMessage(mt) - case *internal.DeleteViewMessage: - return decodeDeleteViewMessage(mt) - case *internal.ClusterStatus: - return decodeClusterStatus(mt) - case *internal.ResizeInstruction: - return decodeResizeInstruction(mt) - case *internal.ResizeInstructionComplete: - return decodeResizeInstructionComplete(mt) - case *internal.SetCoordinatorMessage: - return decodeSetCoordinatorMessage(mt) - case *internal.UpdateCoordinatorMessage: - return decodeUpdateCoordinatorMessage(mt) - case *internal.NodeStateMessage: - return decodeNodeStateMessage(mt) - case *internal.RecalculateCaches: - return decodeRecalculateCaches(mt) - case *internal.NodeEventMessage: - return decodeNodeEventMessage(mt) - case *internal.NodeStatus: - return decodeNodeStatus(mt) - } - return nil -} diff --git a/cluster.go b/cluster.go index e2f29fde5..541c83642 100644 --- a/cluster.go +++ b/cluster.go @@ -1757,28 +1757,6 @@ type ResizeInstruction struct { ClusterStatus *ClusterStatus } -func decodeResizeInstruction(ri *internal.ResizeInstruction) *ResizeInstruction { - return &ResizeInstruction{ - JobID: ri.JobID, - Node: DecodeNode(ri.Node), - Coordinator: DecodeNode(ri.Coordinator), - Sources: decodeResizeSources(ri.Sources), - Schema: decodeSchema(ri.Schema), - ClusterStatus: decodeClusterStatus(ri.ClusterStatus), - } -} - -func encodeResizeInstruction(m *ResizeInstruction) *internal.ResizeInstruction { - return &internal.ResizeInstruction{ - JobID: m.JobID, - Node: EncodeNode(m.Node), - Coordinator: EncodeNode(m.Coordinator), - Sources: encodeResizeSources(m.Sources), - Schema: encodeSchema(m.Schema), - ClusterStatus: encodeClusterStatus(m.ClusterStatus), - } -} - type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -1787,192 +1765,11 @@ type ResizeSource struct { Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource { - new := make([]*ResizeSource, 0, len(srcs)) - for _, src := range srcs { - new = append(new, decodeResizeSource(src)) - } - return new -} - -func encodeResizeSources(srcs []*ResizeSource) []*internal.ResizeSource { - new := make([]*internal.ResizeSource, 0, len(srcs)) - for _, src := range srcs { - new = append(new, encodeResizeSource(src)) - } - return new -} - -func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { - return &ResizeSource{ - Node: DecodeNode(rs.Node), - Index: rs.Index, - Field: rs.Field, - View: rs.View, - Shard: rs.Shard, - } -} - -func encodeResizeSource(m *ResizeSource) *internal.ResizeSource { - return &internal.ResizeSource{ - Node: EncodeNode(m.Node), - Index: m.Index, - Field: m.Field, - View: m.View, - Shard: m.Shard, - } -} - // Schema is a schema type Schema struct { Indexes []*IndexInfo } -func decodeSchema(s *internal.Schema) *Schema { - return &Schema{ - Indexes: decodeIndexes(s.Indexes), - } -} - -func encodeSchema(m *Schema) *internal.Schema { - return &internal.Schema{ - Indexes: encodeIndexInfos(m.Indexes), - } -} - -func decodeIndexes(idxs []*internal.Index) []*IndexInfo { - new := make([]*IndexInfo, 0, len(idxs)) - for _, idx := range idxs { - new = append(new, decodeIndex(idx)) - } - return new -} - -func encodeIndexInfos(idxs []*IndexInfo) []*internal.Index { - new := make([]*internal.Index, 0, len(idxs)) - for _, idx := range idxs { - new = append(new, encodeIndexInfo(idx)) - } - return new -} - -func decodeIndex(idx *internal.Index) *IndexInfo { - return &IndexInfo{ - Name: idx.Name, - Fields: decodeFields(idx.Fields), - } -} - -func encodeIndexInfo(idx *IndexInfo) *internal.Index { - return &internal.Index{ - Name: idx.Name, - Fields: encodeFieldInfos(idx.Fields), - } -} - -func decodeFields(fs []*internal.Field) []*FieldInfo { - new := make([]*FieldInfo, 0, len(fs)) - for _, f := range fs { - new = append(new, decodeField(f)) - } - return new -} - -func encodeFieldInfos(fs []*FieldInfo) []*internal.Field { - new := make([]*internal.Field, 0, len(fs)) - for _, f := range fs { - new = append(new, encodeFieldInfo(f)) - } - return new -} - -func decodeField(f *internal.Field) *FieldInfo { - fi := &FieldInfo{ - Name: f.Name, - Options: *decodeFieldOptions(f.Meta), - Views: make([]*ViewInfo, 0, len(f.Views)), - } - for _, viewname := range f.Views { - fi.Views = append(fi.Views, &ViewInfo{Name: viewname}) - } - return fi -} - -func encodeFieldInfo(f *FieldInfo) *internal.Field { - ifield := &internal.Field{ - Name: f.Name, - Meta: encodeFieldOptions(&f.Options), - Views: make([]string, 0, len(f.Views)), - } - - for _, viewinfo := range f.Views { - ifield.Views = append(ifield.Views, viewinfo.Name) - } - return ifield -} - -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*Node) []*internal.Node { - other := make([]*internal.Node, len(a)) - for i := range a { - other[i] = EncodeNode(a[i]) - } - return other -} - -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *Node) *internal.Node { - return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), - IsCoordinator: n.IsCoordinator, - } -} - -// DecodeNodes converts a proto message into a slice of Nodes. -func DecodeNodes(a []*internal.Node) []*Node { - if len(a) == 0 { - return nil - } - other := make([]*Node, len(a)) - for i := range a { - other[i] = DecodeNode(a[i]) - } - return other -} - -func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus { - return &ClusterStatus{ - State: cs.State, - ClusterID: cs.ClusterID, - Nodes: DecodeNodes(cs.Nodes), - } -} - -func encodeClusterStatus(m *ClusterStatus) *internal.ClusterStatus { - return &internal.ClusterStatus{ - State: m.State, - ClusterID: m.ClusterID, - Nodes: EncodeNodes(m.Nodes), - } -} - -// DecodeNode converts a proto message into a Node. -func DecodeNode(node *internal.Node) *Node { - return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), - IsCoordinator: node.IsCoordinator, - } -} - -func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { - return &NodeEvent{ - Event: NodeEventType(ne.Event), - Node: DecodeNode(ne.Node), - } -} - func encodeTopology(topology *Topology) *internal.Topology { if topology == nil { return nil @@ -2004,267 +1801,60 @@ type CreateShardMessage struct { Shard uint64 } -func encodeCreateShardMessage(m *CreateShardMessage) *internal.CreateShardMessage { - return &internal.CreateShardMessage{ - Index: m.Index, - Shard: m.Shard, - } -} - -func decodeCreateShardMessage(pb *internal.CreateShardMessage) *CreateShardMessage { - return &CreateShardMessage{ - Index: pb.Index, - Shard: pb.Shard, - } -} - type CreateIndexMessage struct { Index string Meta *IndexOptions } -func encodeCreateIndexMessage(m *CreateIndexMessage) *internal.CreateIndexMessage { - return &internal.CreateIndexMessage{ - Index: m.Index, - Meta: encodeIndexMeta(m.Meta), - } -} - -func decodeCreateIndexMessage(pb *internal.CreateIndexMessage) *CreateIndexMessage { - return &CreateIndexMessage{ - Index: pb.Index, - Meta: decodeIndexMeta(pb.Meta), - } -} - -func encodeIndexMeta(m *IndexOptions) *internal.IndexMeta { - return &internal.IndexMeta{ - Keys: m.Keys, - } -} - -func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions { - return &IndexOptions{ - Keys: pb.Keys, - } -} - type DeleteIndexMessage struct { Index string } -func encodeDeleteIndexMessage(m *DeleteIndexMessage) *internal.DeleteIndexMessage { - return &internal.DeleteIndexMessage{ - Index: m.Index, - } -} - -func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage) *DeleteIndexMessage { - return &DeleteIndexMessage{ - Index: pb.Index, - } -} - type CreateFieldMessage struct { Index string Field string Meta *FieldOptions } -func encodeCreateFieldMessage(m *CreateFieldMessage) *internal.CreateFieldMessage { - return &internal.CreateFieldMessage{ - Index: m.Index, - Field: m.Field, - Meta: encodeFieldOptions(m.Meta), - } -} - -func decodeCreateFieldMessage(pb *internal.CreateFieldMessage) *CreateFieldMessage { - return &CreateFieldMessage{ - Index: pb.Index, - Field: pb.Field, - Meta: decodeFieldOptions(pb.Meta), - } -} - type DeleteFieldMessage struct { Index string Field string } -func encodeDeleteFieldMessage(m *DeleteFieldMessage) *internal.DeleteFieldMessage { - return &internal.DeleteFieldMessage{ - Index: m.Index, - Field: m.Field, - } -} - -func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage) *DeleteFieldMessage { - return &DeleteFieldMessage{ - Index: pb.Index, - Field: pb.Field, - } -} - type CreateViewMessage struct { Index string Field string View string } - -func encodeCreateViewMessage(m *CreateViewMessage) *internal.CreateViewMessage { - return &internal.CreateViewMessage{ - Index: m.Index, - Field: m.Field, - View: m.View, - } -} - -func decodeCreateViewMessage(pb *internal.CreateViewMessage) *CreateViewMessage { - return &CreateViewMessage{ - Index: pb.Index, - Field: pb.Field, - View: pb.View, - } -} - type DeleteViewMessage struct { Index string Field string View string } -func encodeDeleteViewMessage(m *DeleteViewMessage) *internal.DeleteViewMessage { - return &internal.DeleteViewMessage{ - Index: m.Index, - Field: m.Field, - View: m.View, - } -} - -func decodeDeleteViewMessage(pb *internal.DeleteViewMessage) *DeleteViewMessage { - return &DeleteViewMessage{ - Index: pb.Index, - Field: pb.Field, - View: pb.View, - } -} - type ResizeInstructionComplete struct { JobID int64 Node *Node Error string } -func encodeResizeInstructionComplete(m *ResizeInstructionComplete) *internal.ResizeInstructionComplete { - return &internal.ResizeInstructionComplete{ - JobID: m.JobID, - Node: EncodeNode(m.Node), - Error: m.Error, - } -} - -func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete) *ResizeInstructionComplete { - return &ResizeInstructionComplete{ - JobID: pb.JobID, - Node: DecodeNode(pb.Node), - Error: pb.Error, - } -} - type SetCoordinatorMessage struct { New *Node } -func encodeSetCoordinatorMessage(m *SetCoordinatorMessage) *internal.SetCoordinatorMessage { - return &internal.SetCoordinatorMessage{ - New: EncodeNode(m.New), - } -} - -func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage) *SetCoordinatorMessage { - return &SetCoordinatorMessage{ - New: DecodeNode(pb.New), - } -} - type UpdateCoordinatorMessage struct { New *Node } -func encodeUpdateCoordinatorMessage(m *UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { - return &internal.UpdateCoordinatorMessage{ - New: EncodeNode(m.New), - } -} - -func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage) *UpdateCoordinatorMessage { - return &UpdateCoordinatorMessage{ - New: DecodeNode(pb.New), - } -} - type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } -func encodeNodeStateMessage(m *NodeStateMessage) *internal.NodeStateMessage { - return &internal.NodeStateMessage{ - NodeID: m.NodeID, - State: m.State, - } -} - -func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage { - return &NodeStateMessage{ - NodeID: pb.NodeID, - State: pb.State, - } -} - -func encodeNodeEventMessage(m *NodeEvent) *internal.NodeEventMessage { - return &internal.NodeEventMessage{ - Event: uint32(m.Event), - Node: EncodeNode(m.Node), - } -} - -func decodeNodeEventMessage(pb *internal.NodeEventMessage) *NodeEvent { - return &NodeEvent{ - Event: NodeEventType(pb.Event), - Node: DecodeNode(pb.Node), - } -} - type NodeStatus struct { Node *Node MaxShards map[string]uint64 Schema *Schema } -func encodeNodeStatus(m *NodeStatus) *internal.NodeStatus { - return &internal.NodeStatus{ - Node: EncodeNode(m.Node), - MaxShards: &internal.MaxShards{Standard: m.MaxShards}, - Schema: encodeSchema(m.Schema), - } -} - -func decodeNodeStatus(pb *internal.NodeStatus) *NodeStatus { - return &NodeStatus{ - Node: DecodeNode(pb.Node), - MaxShards: pb.MaxShards.Standard, - Schema: decodeSchema(pb.Schema), - } -} - type RecalculateCaches struct{} - -func decodeRecalculateCaches(pb *internal.RecalculateCaches) *RecalculateCaches { - return &RecalculateCaches{} -} - -func encodeRecalculateCaches(*RecalculateCaches) *internal.RecalculateCaches { - return &internal.RecalculateCaches{} -} diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 341ad46f7..003e9cd94 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -375,11 +375,19 @@ func EncodeNodes(a []*pilosa.Node) []*internal.Node { func encodeNode(n *pilosa.Node) *internal.Node { return &internal.Node{ ID: n.ID, - URI: n.URI.Encode(), + URI: encodeURI(n.URI), IsCoordinator: n.IsCoordinator, } } +func encodeURI(u pilosa.URI) *internal.URI { + return &internal.URI{ + Scheme: u.Scheme, + Host: u.Host, + Port: uint32(u.Port), + } +} + func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { return &internal.ClusterStatus{ State: m.State, diff --git a/executor.go b/executor.go index aa2c2fb88..399da685d 100644 --- a/executor.go +++ b/executor.go @@ -20,7 +20,6 @@ import ( "sort" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -1738,20 +1737,6 @@ func (vc *ValCount) Add(other ValCount) ValCount { } } -func EncodeValCount(vc ValCount) *internal.ValCount { - return &internal.ValCount{ - Val: vc.Val, - Count: vc.Count, - } -} - -func decodeValCount(pb *internal.ValCount) ValCount { - return ValCount{ - Val: pb.Val, - Count: pb.Count, - } -} - // Smaller returns the smaller of the two ValCounts. func (vc *ValCount) Smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { diff --git a/field.go b/field.go index fb7e9af4f..0ef5630e3 100644 --- a/field.go +++ b/field.go @@ -1088,25 +1088,6 @@ func (f *Field) MarshalJSON() ([]byte, error) { return json.Marshal(thing) } -// encodeFields converts a into its internal representation. -func encodeFields(a []*Field) []*internal.Field { - other := make([]*internal.Field, len(a)) - for i := range a { - other[i] = encodeField(a[i]) - } - return other -} - -// encodeField converts f into its internal representation. -func encodeField(f *Field) *internal.Field { - fo := f.options - return &internal.Field{ - Name: f.name, - Meta: fo.Encode(), - Views: f.viewNames(), - } -} - type fieldSlice []*Field func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -1170,21 +1151,6 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { } } -func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { - if options == nil { - return nil - } - return &FieldOptions{ - Type: options.Type, - CacheType: options.CacheType, - CacheSize: options.CacheSize, - Min: options.Min, - Max: options.Max, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Keys: options.Keys, - } -} - func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { case FieldTypeSet: diff --git a/holder.go b/holder.go index 10e147098..cb067db92 100644 --- a/holder.go +++ b/holder.go @@ -27,7 +27,6 @@ import ( "syscall" "time" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -256,20 +255,6 @@ func (h *Holder) applySchema(schema *Schema) error { return nil } -// encodeMaxShards creates and internal representation of max shards. -func (h *Holder) encodeMaxShards() *internal.MaxShards { - return &internal.MaxShards{ - Standard: h.maxShards(), - } -} - -// encodeSchema creates an internal representation of schema. -func (h *Holder) encodeSchema() *internal.Schema { - return &internal.Schema{ - Indexes: EncodeIndexes(h.Indexes()), - } -} - // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } diff --git a/index.go b/index.go index 98f50eced..7451030cd 100644 --- a/index.go +++ b/index.go @@ -403,35 +403,11 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// EncodeIndexes converts a into its internal representation. -func EncodeIndexes(a []*Index) []*internal.Index { - other := make([]*internal.Index, len(a)) - for i := range a { - other[i] = encodeIndex(a[i]) - } - return other -} - -// encodeIndex converts d into its internal representation. -func encodeIndex(d *Index) *internal.Index { - return &internal.Index{ - Name: d.name, - Fields: encodeFields(d.Fields()), - } -} - // IndexOptions represents options to set when initializing an index. type IndexOptions struct { Keys bool `json:"keys"` } -// Encode converts i into its internal representation. -func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{ - Keys: i.Keys, - } -} - // hasTime returns true if a contains a non-nil time. func hasTime(a []*time.Time) bool { for _, t := range a { diff --git a/row.go b/row.go index cbfa6b270..2134026e9 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,6 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" ) @@ -271,32 +270,6 @@ func (r *Row) Columns() []uint64 { return a } -// EncodeRow converts r into its internal representation. -func EncodeRow(r *Row) *internal.Row { - if r == nil { - return nil - } - - return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), - } -} - -// DecodeRow converts r from its internal representation. -func DecodeRow(pr *internal.Row) *Row { - if pr == nil { - return nil - } - - r := NewRow() - r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Columns { - r.SetBit(v) - } - return r -} - // Union performs a union on a slice of rows. func Union(rows []*Row) *Row { other := rows[0] diff --git a/uri.go b/uri.go index 2058f70fa..b58678381 100644 --- a/uri.go +++ b/uri.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -206,34 +205,6 @@ func parseAddress(address string) (uri *URI, err error) { return uri, nil } -// Encode converts o into its internal representation. -func (u URI) Encode() *internal.URI { - return encodeURI(u) -} - -func encodeURI(u URI) *internal.URI { - return &internal.URI{ - Scheme: u.Scheme, - Host: u.Host, - Port: uint32(u.Port), - } -} - -func DecodeURI(i *internal.URI) URI { - return decodeURI(i) -} - -func decodeURI(i *internal.URI) URI { - if i == nil { - return URI{} - } - return URI{ - Scheme: i.Scheme, - Host: i.Host, - Port: uint16(i.Port), - } -} - // MarshalJSON marshals URI into a JSON-encoded byte slice. func (u *URI) MarshalJSON() ([]byte, error) { var output struct { From db2a53223d591a396c5d50964d3e675069616af9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:02:18 -0500 Subject: [PATCH 10/18] remove internal references from api and http/* --- api.go | 14 ++--- encoding/proto/proto.go | 134 +++++++++++++++++++++++++++++++++++++++- handler.go | 37 +++++++++++ http/client.go | 34 +++++----- http/handler.go | 12 ++-- 5 files changed, 197 insertions(+), 34 deletions(-) diff --git a/api.go b/api.go index 1abe4502c..181d1b3fb 100644 --- a/api.go +++ b/api.go @@ -26,8 +26,6 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -437,8 +435,8 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, if err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read body error")) } - var req internal.BlockDataRequest - if err := proto.Unmarshal(reqBytes, &req); err != nil { + var req BlockDataRequest + if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error")) } @@ -448,11 +446,11 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, return nil, ErrFragmentNotFound } - var resp = internal.BlockDataResponse{} + var resp = BlockDataResponse{} resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. - buf, err := proto.Marshal(&resp) + buf, err := api.Serializer.Marshal(&resp) if err != nil { return nil, errors.Wrap(err, "merge block response encoding error") } @@ -657,7 +655,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s } // Import bulk imports data into a particular index,field,shard. -func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { +func (api *API) Import(ctx context.Context, req *ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } @@ -686,7 +684,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { } // ImportValue bulk imports values into a particular field. -func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { +func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { return errors.Wrap(err, "validating api method") } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 003e9cd94..ed751247e 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -178,7 +178,46 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeQueryResponse(msg, mt) return nil - + case *pilosa.ImportRequest: + msg := &internal.ImportRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportRequest") + } + decodeImportRequest(msg, mt) + return nil + case *pilosa.ImportValueRequest: + msg := &internal.ImportValueRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportValueRequest") + } + decodeImportValueRequest(msg, mt) + return nil + case *pilosa.ImportResponse: + msg := &internal.ImportResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportResponse") + } + decodeImportResponse(msg, mt) + return nil + case *pilosa.BlockDataRequest: + msg := &internal.BlockDataRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataRequest") + } + decodeBlockDataRequest(msg, mt) + return nil + case *pilosa.BlockDataResponse: + msg := &internal.BlockDataResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataResponse") + } + decodeBlockDataResponse(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -224,10 +263,66 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeQueryRequest(mt) case *pilosa.QueryResponse: return encodeQueryResponse(mt) + case *pilosa.ImportRequest: + return encodeImportRequest(mt) + case *pilosa.ImportValueRequest: + return encodeImportValueRequest(mt) + case *pilosa.ImportResponse: + return encodeImportResponse(mt) + case *pilosa.BlockDataRequest: + return encodeBlockDataRequest(mt) + case *pilosa.BlockDataResponse: + return encodeBlockDataResponse(mt) } return nil } +func encodeBlockDataRequest(m *pilosa.BlockDataRequest) *internal.BlockDataRequest { + return &internal.BlockDataRequest{ + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + Block: m.Block, + } +} +func encodeBlockDataResponse(m *pilosa.BlockDataResponse) *internal.BlockDataResponse { + return &internal.BlockDataResponse{ + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + } +} + +func encodeImportResponse(m *pilosa.ImportResponse) *internal.ImportResponse { + return &internal.ImportResponse{ + Err: m.Err, + } +} + +func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { + return &internal.ImportRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, + } +} + +func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { + return &internal.ImportValueRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + } +} + func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { return &internal.QueryRequest{ Query: m.Query, @@ -690,6 +785,43 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { m.ExcludeColumns = pb.ExcludeColumns } +func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs + m.RowKeys = pb.RowKeys + m.ColumnKeys = pb.ColumnKeys + m.Timestamps = pb.Timestamps +} + +func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.ColumnIDs = pb.ColumnIDs + m.ColumnKeys = pb.ColumnKeys + m.Values = pb.Values +} + +func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) { + m.Err = pb.Err +} + +func decodeBlockDataRequest(pb *internal.BlockDataRequest, m *pilosa.BlockDataRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View + m.Shard = pb.Shard + m.Block = pb.Block +} + +func decodeBlockDataResponse(pb *internal.BlockDataResponse, m *pilosa.BlockDataResponse) { + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs +} + func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) diff --git a/handler.go b/handler.go index 1f3d04300..9fc3af368 100644 --- a/handler.go +++ b/handler.go @@ -75,3 +75,40 @@ func (n nopHandler) Close() error { } var NopHandler Handler = nopHandler{} + +type ImportValueRequest struct { + Index string + Field string + Shard uint64 + ColumnIDs []uint64 + ColumnKeys []string + Values []int64 +} + +type ImportRequest struct { + Index string + Field string + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +type ImportResponse struct { + Err string +} + +type BlockDataRequest struct { + Index string + Field string + View string + Shard uint64 + Block uint64 +} + +type BlockDataResponse struct { + RowIDs []uint64 + ColumnIDs []uint64 +} diff --git a/http/client.go b/http/client.go index d8fb7f7ef..19bf82815 100644 --- a/http/client.go +++ b/http/client.go @@ -29,10 +29,8 @@ import ( "crypto/tls" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - pilosaproto "github.com/pilosa/pilosa/encoding/proto" - "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pkg/errors" ) @@ -68,7 +66,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { return &InternalClient{ defaultURI: defaultURI, - serializer: pilosaproto.Serializer{}, + serializer: proto.Serializer{}, HTTPClient: remoteClient, } } @@ -282,7 +280,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard return pilosa.ErrFieldRequired } - buf, err := marshalImportPayload(index, field, shard, bits) + buf, err := c.marshalImportPayload(index, field, shard, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -311,7 +309,7 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, colum return pilosa.ErrFieldRequired } - buf, err := marshalImportPayloadK(index, field, columns) + buf, err := c.marshalImportPayloadK(index, field, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -345,14 +343,14 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { +func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ Index: index, Field: field, Shard: shard, @@ -367,14 +365,14 @@ func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) } // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { +func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowKeys := Bits(bits).RowKeys() columnKeys := Bits(bits).ColumnKeys() timestamps := Bits(bits).Timestamps() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ Index: index, Field: field, RowKeys: rowKeys, @@ -416,8 +414,8 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde return errors.New(string(body)) } - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { + var isresp pilosa.ImportResponse + if err := c.serializer.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) } else if s := isresp.Err; s != "" { return errors.New(s) @@ -434,7 +432,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s return pilosa.ErrFieldRequired } - buf, err := marshalImportValuePayload(index, field, shard, vals) + buf, err := c.marshalImportValuePayload(index, field, shard, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -456,13 +454,13 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { +func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{ Index: index, Field: field, Shard: shard, @@ -685,7 +683,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, if uri == nil { panic("need to pass a URI to BlockData") } - buf, err := proto.Marshal(&internal.BlockDataRequest{ + buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{ Index: index, Field: field, Shard: shard, @@ -721,10 +719,10 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, } // Decode response object. - var rsp internal.BlockDataResponse + var rsp pilosa.BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { + } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { return nil, nil, errors.Wrap(err, "unmarshalling") } return rsp.RowIDs, rsp.ColumnIDs, nil diff --git a/http/handler.go b/http/handler.go index 33085879c..2b3d3ebd8 100644 --- a/http/handler.go +++ b/http/handler.go @@ -33,11 +33,9 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -911,8 +909,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if field.Type() == pilosa.FieldTypeInt { // Field type: Int // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { + req := &pilosa.ImportValueRequest{} + if err := h.API.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -929,8 +927,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } else { // Field type: Set, Time // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { + req := &pilosa.ImportRequest{} + if err := h.API.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -947,7 +945,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: ""}) + buf, e := h.API.Serializer.Marshal(&pilosa.ImportResponse{Err: ""}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) return From a164233c92f890c45bab13fa837073abe3d96feb Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:21:27 -0500 Subject: [PATCH 11/18] fix handler tests not to use internal and fix bug --- encoding/proto/proto.go | 1 + server/handler_test.go | 80 +++++++++++++++++++---------------------- 2 files changed, 37 insertions(+), 44 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index ed751247e..0020260ab 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -837,6 +837,7 @@ func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { for i := range pb { + m[i] = &pilosa.ColumnAttrSet{} decodeColumnAttrSet(pb[i], m[i]) } } diff --git a/server/handler_test.go b/server/handler_test.go index e49b1eb0f..f0eeb18c8 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -27,10 +27,8 @@ import ( gohttp "net/http" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -144,7 +142,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Shards args protobuf", func(t *testing.T) { // Generate request body. - reqBody, err := proto.Marshal(&internal.QueryRequest{ + reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ Query: "Count(Row(f0=30))", Shards: []uint64{0, 1}, }) @@ -196,13 +194,11 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if n := resp.Results[0].N; n != 3 { - t.Fatalf("unexpected n: %d", n) + } else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) } }) @@ -244,27 +240,25 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + } else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) } }) t.Run("Row columnattrs protobuf", func(t *testing.T) { // Encode request body. - buf, err := proto.Marshal(&internal.QueryRequest{ + buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ Query: "Row(f0=30)", ColumnAttrs: true, }) @@ -281,22 +275,22 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + } else if _, ok := resp.Results[0].(*pilosa.Row); !ok { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) } if a := resp.ColumnAttrSets; len(a) != 2 { @@ -305,8 +299,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected column attr length: %d", len(a)) - } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if a[0].Attrs["x"] != "y" { + t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"]) } }) @@ -329,12 +323,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if a := resp.Results[0].GetPairs(); len(a) != 2 { + } else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } }) @@ -358,10 +350,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if s := resp.Err; s != `executing: field not found` { + } else if s := resp.Err.Error(); s != `executing: field not found` { t.Fatalf("unexpected error: %s", s) } }) From b7a583d6a8fc8ca687611d40885b1c59f7a5f0d6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 16:24:40 -0500 Subject: [PATCH 12/18] use Set() instead of SetValue() for integer fields --- executor.go | 123 ++++++++++++++++++++--------------------------- executor_test.go | 58 +++++++++++----------- pql/ast.go | 20 ++++++++ 3 files changed, 102 insertions(+), 99 deletions(-) diff --git a/executor.go b/executor.go index 1a81e92bd..96e327f43 100644 --- a/executor.go +++ b/executor.go @@ -185,8 +185,6 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeCount(ctx, index, c, shards, opt) case "Set": return e.executeSetBit(ctx, index, c, opt) - case "SetValue": - return nil, e.executeSetValue(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -1077,14 +1075,7 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, ErrFieldNotFound } - // Read fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Set() row: %v", err) - } else if !ok { - return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) - } - + // Read colID using labels. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { return false, fmt.Errorf("reading Set() column: %v", err) @@ -1092,20 +1083,40 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, fmt.Errorf("Set() column argument '%v' required", columnLabel) } - var timestamp *time.Time - sTimestamp, ok := c.Args["_timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) + if f.Type() == FieldTypeInt { + // Read remaining fields using labels. + rowVal, ok, err := c.IntArg(fieldName) if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) } - timestamp = &t - } - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) + } else { + // Read remaining fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + } } -// executeSetBitField executes a Set() call for a specific view. +// executeSetBitField executes a Set() call for a specific field. func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { shard := colID / ShardWidth ret := false @@ -1137,64 +1148,36 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. return ret, nil } -// executeSetValue executes a SetValue() call. -func (e *executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { - // Parse labels. - columnID, ok, err := c.UintArg(columnLabel) - if err != nil { - return fmt.Errorf("reading SetValue() column: %v", err) - } else if !ok { - return fmt.Errorf("SetValue() column field '%v' required", columnLabel) - } +// executeSetValueField executes a Set() call for a specific int field. +func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { + shard := colID / ShardWidth + ret := false - // Copy args and remove reserved fields. - args := pql.CopyArgs(c.Args) - // While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. - // Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility. - delete(args, columnLabel) - - // Set values. - for name, value := range args { - // Retrieve field. - field := e.Holder.Field(index, name) - if field == nil { - return ErrFieldNotFound - } - - switch value := value.(type) { - case int64: - if _, err := field.SetValue(columnID, value); err != nil { - return err + for _, node := range e.Cluster.shardNodes(index, shard) { + // Update locally if host matches. + if node.ID == e.Node.ID { + val, err := f.SetValue(colID, value) + if err != nil { + return false, err + } else if val { + ret = true } - default: - return ErrInvalidBSIGroupValueType + continue } - field.Stats.Count("SetValue", 1, 1.0) - } - // Do not forward call if this is already being forwarded. - if opt.Remote { - return nil - } + // Do not forward call if this is already being forwarded. + if opt.Remote { + continue + } - // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) - resp := make(chan error, len(nodes)) - for _, node := range nodes { - go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) - resp <- err - }(node) - } - - // Return first error. - for range nodes { - if err := <-resp; err != nil { - return err + // Forward call to remote node otherwise. + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + return false, err + } else { + ret = res[0].(bool) } } - - return nil + return ret, nil } // executeSetRowAttrs executes a SetRowAttrs() call. diff --git a/executor_test.go b/executor_test.go index c5e2f24a8..1bbeafddd 100644 --- a/executor_test.go +++ b/executor_test.go @@ -405,9 +405,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil { + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil { t.Fatal(err) } @@ -440,19 +440,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) @@ -748,14 +748,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) { Set(1, x=1) Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) - SetValue(col=0, f=20) - SetValue(col=1, f=-5) - SetValue(col=2, f=-5) - SetValue(col=3, f=10) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60) + Set(0, f=20) + Set(1, f=-5) + Set(2, f=-5) + Set(3, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) `}); err != nil { t.Fatal(err) } @@ -844,13 +844,13 @@ func TestExecutor_Execute_Sum(t *testing.T) { Set(0, x=0) Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - SetValue(col=0, foo=20) - SetValue(col=0, bar=2000) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) - SetValue(col=0, other=1000) + Set(0, foo=20) + Set(0, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) `}); err != nil { t.Fatal(err) } @@ -959,15 +959,15 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) - SetValue(col=50, foo=20) - SetValue(col=50, bar=2000) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) - SetValue(col=0, other=1000) - SetValue(col=0, edge=100) - SetValue(col=1, edge=-100) + Set(50, foo=20) + Set(50, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + Set(0, edge=100) + Set(1, edge=-100) `}); err != nil { t.Fatal(err) } diff --git a/pql/ast.go b/pql/ast.go index 0bcc582d4..4539bc95f 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -285,6 +285,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } } +// IntArg is for reading the value at key from call.Args as a uint64. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a uint64 or an int64 and +// then cast to a uint64. An error is returned if the value is not an int64 or +// uint64. +func (c *Call) IntArg(key string) (int64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return tval, true, nil + case uint64: + return int64(tval), true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.IntArg", tval, tval) + } +} + // UintSliceArg reads the value at key from call.Args as a slice of uint64. If // the key is not in Call.Args, the value of the returned bool will be false, // and the error will be nil. If the value is a slice of int64 it will convert From def3be0f17df00ccdfde76d829930f16aaf2e021 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:25:04 -0500 Subject: [PATCH 13/18] remove more dead code --- cache.go | 25 ------------------------- pilosa.go | 19 ------------------- 2 files changed, 44 deletions(-) diff --git a/cache.go b/cache.go index 4c826f5d7..044d85f3c 100644 --- a/cache.go +++ b/cache.go @@ -22,7 +22,6 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/lru" ) @@ -318,22 +317,6 @@ type Pair struct { Count uint64 `json:"count"` } -func encodePair(p Pair) *internal.Pair { - return &internal.Pair{ - ID: p.ID, - Key: p.Key, - Count: p.Count, - } -} - -func decodePair(pb *internal.Pair) Pair { - return Pair{ - ID: pb.ID, - Key: pb.Key, - Count: pb.Count, - } -} - // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -409,14 +392,6 @@ func (p Pairs) String() string { return buf.String() } -func decodePairs(a []*internal.Pair) []Pair { - other := make([]Pair, len(a)) - for i := range a { - other[i] = decodePair(a[i]) - } - return other -} - // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/pilosa.go b/pilosa.go index ebc2be438..9615e88b8 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,8 +17,6 @@ package pilosa import ( "errors" "regexp" - - "github.com/pilosa/pilosa/internal" ) // System errors. @@ -122,23 +120,6 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } -// EncodeColumnAttrSets converts a into its internal representation. -func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { - other := make([]*internal.ColumnAttrSet, len(a)) - for i := range a { - other[i] = EncodeColumnAttrSet(a[i]) - } - return other -} - -// EncodeColumnAttrSet converts set into its internal representation. -func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { - return &internal.ColumnAttrSet{ - ID: set.ID, - Attrs: encodeAttrs(set.Attrs), - } -} - // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" From 462b27d9a9f8b1e8d50fab7ccd105e25a3bfbf22 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 16:31:06 -0500 Subject: [PATCH 14/18] change executeSetBit to executeSet --- executor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 96e327f43..8ebc1df90 100644 --- a/executor.go +++ b/executor.go @@ -184,7 +184,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) case "Set": - return e.executeSetBit(ctx, index, c, opt) + return e.executeSet(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -1058,8 +1058,8 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq return ret, nil } -// executeSetBit executes a Set() call. -func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { +// executeSet executes a Set() call. +func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Set() argument required: field") From 5717e32310912583c31f5e74ac7152b3f850ad02 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 16:35:27 -0500 Subject: [PATCH 15/18] fix comment for IntArg --- pql/ast.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 4539bc95f..dc16f4cdf 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -285,10 +285,10 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } } -// IntArg is for reading the value at key from call.Args as a uint64. If the +// IntArg is for reading the value at key from call.Args as an int64. If the // key is not in Call.Args, the value of the returned bool will be false, and -// the error will be nil. The value is assumed to be a uint64 or an int64 and -// then cast to a uint64. An error is returned if the value is not an int64 or +// the error will be nil. The value is assumed to be a unt64 or an int64 and +// then cast to an int64. An error is returned if the value is not an int64 or // uint64. func (c *Call) IntArg(key string) (int64, bool, error) { val, ok := c.Args[key] @@ -301,7 +301,7 @@ func (c *Call) IntArg(key string) (int64, bool, error) { case uint64: return int64(tval), true, nil default: - return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.IntArg", tval, tval) + return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval) } } From d0485a3a1985a7eebd119c3f5b13240077207f44 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:51:23 -0500 Subject: [PATCH 16/18] remove unused code in row.go --- row.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/row.go b/row.go index 4d9ba3662..d6a0037cc 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,6 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" ) @@ -252,41 +251,6 @@ func (r *Row) Columns() []uint64 { return a } -// Union performs a union on a slice of rows. -func Union(rows []*Row) *Row { - other := rows[0] - for _, r := range rows[1:] { - other = other.Union(r) - } - return other -} - -// EncodeRow converts r into its internal representation. -func EncodeRow(r *Row) *internal.Row { - if r == nil { - return nil - } - - return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), - } -} - -// DecodeRow converts r from its internal representation. -func DecodeRow(pr *internal.Row) *Row { - if pr == nil { - return nil - } - - r := NewRow() - r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Columns { - r.SetBit(v) - } - return r -} - // RowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The // width of the segment will always match the shard width. From e33682cfd27a3d1efdaf6fb8888e3cc9d66f5bea Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 17:38:49 -0500 Subject: [PATCH 17/18] address feedback --- cluster.go | 4 ++-- utils_internal_test.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 6d8cf38b4..f61173c3b 100644 --- a/cluster.go +++ b/cluster.go @@ -459,7 +459,7 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { return nil } -// Status returns the the cluster's status including what nodes it contains, it's ID, and current state. +// Status returns the the cluster's status including what nodes it contains, its ID, and current state. func (c *cluster) Status() *ClusterStatus { return &ClusterStatus{ ClusterID: c.id, @@ -1764,7 +1764,7 @@ type ResizeSource struct { Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } -// Schema is a schema +// Schema contains information about indexes and their configuration. type Schema struct { Indexes []*IndexInfo } diff --git a/utils_internal_test.go b/utils_internal_test.go index e0a42bdc1..df1b7d2e0 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -364,8 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error } for _, src := range instr.Sources { - srcNode := src.Node - srcCluster := t.clusterByID(srcNode.ID) + srcCluster := t.clusterByID(src.Node.ID) srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) From 21cf6b6e57b1cd18c26a8a51609f452cab49f482 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 17:59:18 -0500 Subject: [PATCH 18/18] remove URI getters since the fields were exported for serialization --- gossip/gossip.go | 6 +++--- http/client.go | 4 ++-- server/server.go | 12 ++++++------ uri.go | 15 --------------- uri_internal_test.go | 14 +++++++------- 5 files changed, 18 insertions(+), 33 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 28b3ac690..7a11d01d4 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -146,7 +146,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - host := api.Node().URI.GetHost() + host := api.Node().URI.Host g := &GossipMemberSet{ papi: api, Logger: pilosa.NopLogger, @@ -191,10 +191,10 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO conf := memberlist.DefaultWANConfig() conf.Transport = g.transport.Net conf.Name = api.Node().ID - conf.BindAddr = api.Node().URI.GetHost() + conf.BindAddr = api.Node().URI.Host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = hostToIP(api.Node().URI.GetHost()) + conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult diff --git a/http/client.go b/http/client.go index 19bf82815..de6eb89f6 100644 --- a/http/client.go +++ b/http/client.go @@ -993,7 +993,7 @@ func pos(rowID, columnID uint64) uint64 { func uriPathToURL(uri *pilosa.URI, path string) url.URL { return url.URL{ - Scheme: uri.GetScheme(), + Scheme: uri.Scheme, Host: uri.HostPort(), Path: path, } @@ -1001,7 +1001,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL { func nodePathToURL(node *pilosa.Node, path string) url.URL { return url.URL{ - Scheme: node.URI.GetScheme(), + Scheme: node.URI.Scheme, Host: node.URI.HostPort(), Path: path, } diff --git a/server/server.go b/server/server.go index f46f6a64a..401da09e8 100644 --- a/server/server.go +++ b/server/server.go @@ -203,7 +203,7 @@ func (m *Command) SetupServer() error { // Setup TLS var TLSConfig *tls.Config - if uri.GetScheme() == "https" { + if uri.Scheme == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") } @@ -236,7 +236,7 @@ func (m *Command) SetupServer() error { } // If port is 0, get auto-allocated port from listener - if uri.GetPort() == 0 { + if uri.Port == 0 { uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) } @@ -311,7 +311,7 @@ func (m *Command) SetupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.API.Node().URI.GetHost() + gossipHost := m.API.Node().URI.Host m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return errors.Wrap(err, "getting transport") @@ -368,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { // getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS - if uri.GetScheme() == "https" && tlsconf != nil { + if uri.Scheme == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) if err != nil { return nil, errors.Wrap(err, "tls.Listener") } - } else if uri.GetScheme() == "http" { + } else if uri.Scheme == "http" { // Open HTTP listener to determine port (if specified as :0). ln, err = net.Listen("tcp", uri.HostPort()) if err != nil { return nil, errors.Wrap(err, "net.Listen") } } else { - return nil, errors.Errorf("unsupported scheme: %s", uri.GetScheme()) + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme) } return ln, nil diff --git a/uri.go b/uri.go index e01bf8bcc..8577c8238 100644 --- a/uri.go +++ b/uri.go @@ -82,11 +82,6 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// GetScheme returns the scheme of this URI. -func (u *URI) GetScheme() string { - return u.Scheme -} - // SetScheme sets the scheme of this URI. func (u *URI) SetScheme(scheme string) error { m := schemeRegexp.FindStringSubmatch(scheme) @@ -97,11 +92,6 @@ func (u *URI) SetScheme(scheme string) error { return nil } -// GetHost returns the host of this URI. -func (u *URI) GetHost() string { - return u.Host -} - // SetHost sets the host of this URI. func (u *URI) SetHost(host string) error { m := hostRegexp.FindStringSubmatch(host) @@ -112,11 +102,6 @@ func (u *URI) SetHost(host string) error { return nil } -// GetPort returns the port of this URI. -func (u *URI) GetPort() uint16 { - return u.Port -} - // SetPort sets the port of this URI. func (u *URI) SetPort(port uint16) { u.Port = port diff --git a/uri_internal_test.go b/uri_internal_test.go index 2587223f8..3c9631661 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -83,8 +83,8 @@ func TestSetScheme(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.GetScheme() != target { - t.Fatalf("%s != %s", uri.GetScheme(), target) + if uri.Scheme != target { + t.Fatalf("%s != %s", uri.Scheme, target) } } @@ -95,7 +95,7 @@ func TestSetHost(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.GetHost() != target { + if uri.Host != target { t.Fatalf("%s != %s", uri.Host, target) } } @@ -104,7 +104,7 @@ func TestSetPort(t *testing.T) { uri := DefaultURI() target := uint16(9999) uri.SetPort(target) - if uri.GetPort() != target { + if uri.Port != target { t.Fatalf("%d != %d", uri.Port, target) } } @@ -137,13 +137,13 @@ func TestHostPort(t *testing.T) { } func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { - if uri.GetScheme() != scheme { + if uri.Scheme != scheme { t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme) } - if uri.GetHost() != host { + if uri.Host != host { t.Fatalf("Host does not match: %s != %s", uri.Host, host) } - if uri.GetPort() != port { + if uri.Port != port { t.Fatalf("Port does not match: %d != %d", uri.Port, port) } }