From 4ef8fff9b6dbce49f951bf4c3e75481c3b2b7327 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 4 Jul 2018 07:29:16 -0500 Subject: [PATCH 001/166] WIP, broken. refactoring to isolate intneral structs and define core structs --- broadcast.go | 33 ++++-- cluster.go | 306 ++++++++++++++++++++++++++++++++++++++------------- server.go | 6 +- 3 files changed, 259 insertions(+), 86 deletions(-) diff --git a/broadcast.go b/broadcast.go index 19e927c18..3f6b960e4 100644 --- a/broadcast.go +++ b/broadcast.go @@ -25,11 +25,15 @@ import ( // broadcaster is an interface for broadcasting messages. type broadcaster interface { - SendSync(pb proto.Message) error - SendAsync(pb proto.Message) error - SendTo(to *Node, pb proto.Message) error + SendSync(Message) error + SendAsync(Message) error + SendTo(*Node, Message) error } +// Message is the interface implemented by all core pilosa types which can be serialized to messages. +// TODO add at least a single "isMessage()" method. +type Message interface{} + func init() { NopBroadcaster = &nopBroadcaster{} } @@ -40,13 +44,13 @@ var NopBroadcaster broadcaster type nopBroadcaster struct{} // SendSync A no-op implementation of Broadcaster SendSync method. -func (n nopBroadcaster) SendSync(pb proto.Message) error { return nil } +func (nopBroadcaster) SendSync(Message) error { return nil } // SendAsync A no-op implementation of Broadcaster SendAsync method. -func (n nopBroadcaster) SendAsync(pb proto.Message) error { return nil } +func (nopBroadcaster) SendAsync(Message) error { return nil } // SendTo is a no-op implementation of Broadcaster SendTo method. -func (c nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil } +func (nopBroadcaster) SendTo(*Node, Message) error { return nil } // Broadcast message types. const ( @@ -69,7 +73,8 @@ const ( ) // MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(m proto.Message) ([]byte, error) { +func MarshalMessage(pm Message) ([]byte, error) { + m := encode(pm) var typ uint8 switch obj := m.(type) { case *internal.CreateShardMessage: @@ -114,8 +119,20 @@ func MarshalMessage(m proto.Message) ([]byte, error) { return append([]byte{typ}, buf...), nil } +func encode(m Message) proto.Message { + var pm proto.Message + switch mt := m.(type) { + case *CreateShardMessage: + return encodeCreateShardMessage(mt) + case *CreateIndexMessage: + return encodeCreateIndexMessage(mt) + // TODO, the rest + } + return nil +} + // UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (proto.Message, error) { +func UnmarshalMessage(buf []byte) (Message, error) { typ, buf := buf[0], buf[1:] var m proto.Message switch typ { diff --git a/cluster.go b/cluster.go index acf06dae0..fcfb08d73 100644 --- a/cluster.go +++ b/cluster.go @@ -69,52 +69,6 @@ func (n Node) String() string { return fmt.Sprintf("Node: %s", n.ID) } -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*Node) []*internal.Node { - other := make([]*internal.Node, len(a)) - for i := range a { - other[i] = EncodeNode(a[i]) - } - return other -} - -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *Node) *internal.Node { - return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), - IsCoordinator: n.IsCoordinator, - } -} - -// DecodeNodes converts a proto message into a slice of Nodes. -func DecodeNodes(a []*internal.Node) []*Node { - if len(a) == 0 { - return nil - } - other := make([]*Node, len(a)) - for i := range a { - other[i] = DecodeNode(a[i]) - } - return other -} - -// DecodeNode converts a proto message into a Node. -func DecodeNode(node *internal.Node) *Node { - return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), - IsCoordinator: node.IsCoordinator, - } -} - -func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ - Event: NodeEventType(ne.Event), - Node: DecodeNode(ne.Node), - } -} - // Nodes represents a list of nodes. type Nodes []*Node @@ -1176,7 +1130,7 @@ func (c *cluster) completeCurrentJob(state string) error { } // followResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) error { +func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { c.logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. @@ -1553,32 +1507,6 @@ func (c *cluster) saveTopology() error { return nil } -func encodeTopology(topology *Topology) *internal.Topology { - if topology == nil { - return nil - } - return &internal.Topology{ - ClusterID: topology.ClusterID, - NodeIDs: topology.NodeIDs, - } -} - -func decodeTopology(topology *internal.Topology) (*Topology, error) { - if topology == nil { - return nil, nil - } - - t := NewTopology() - t.ClusterID = topology.ClusterID - t.NodeIDs = topology.NodeIDs - sort.Slice(t.NodeIDs, - func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] - }) - - return t, nil -} - func (c *cluster) considerTopology() error { // Create ClusterID if one does not already exist. if c.id == "" { @@ -1752,7 +1680,7 @@ func (c *cluster) nodeLeave(node *Node) error { return nil } -func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() c.logger.Printf("merge cluster status: %v", cs) @@ -1764,7 +1692,7 @@ func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { // Set ClusterID. c.setID(cs.ClusterID) - officialNodes := DecodeNodes(cs.Nodes) + officialNodes := cs.Nodes // Add all nodes from the coordinator. for _, node := range officialNodes { @@ -1813,3 +1741,231 @@ func (c *cluster) setStatic(hosts []string) error { } return nil } + +type ClusterStatus struct { + ClusterID string + State string + Nodes []*Node +} + +type ResizeInstruction struct { + JobID int64 + Node *Node + Coordinator *Node + Sources []*ResizeSource + Schema *Schema + ClusterStatus *ClusterStatus +} + +func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction { + return &ResizeInstruction{ + JobID: ri.JobID, + Node: DecodeNode(ri.Node), + Coordinator: DecodeNode(ri.Coordinator), + Sources: decodeResizeSources(ri.Sources), + Schema: decodeSchema(ri.Schema), + ClusterStatus: decodeClusterStatus(ri.ClusterStatus), + } +} + +type ResizeSource struct { + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` +} + +func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource { + new := make([]*ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, decodeResizeSource(src)) + } + return new +} + +func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { + return &ResizeSource{ + Node: DecodeNode(rs.Node), + Index: rs.Index, + Field: rs.Field, + View: rs.View, + Shard: rs.Shard, + } +} + +// Schema is a schema +type Schema struct { + Indexes []*IndexInfo +} + +func decodeSchema(s *internal.Schema) *Schema { + return &Schema{ + Indexes: decodeIndexes(s.Indexes), + } +} + +func decodeIndexes(idxs []*internal.Index) []*IndexInfo { + new := make([]*IndexInfo, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, decodeIndex(idx)) + } + return new +} + +func decodeIndex(idx *internal.Index) *IndexInfo { + return &IndexInfo{ + Name: idx.Name, + Fields: decodeFields(idx.Fields), + } +} + +func decodeFields(fs []*internal.Field) []*FieldInfo { + new := make([]*FieldInfo, 0, len(fs)) + for _, f := range fs { + new = append(new, decodeField(f)) + } + return new +} + +func decodeField(f *internal.Field) *FieldInfo { + fi := &FieldInfo{ + Name: f.Name, + Options: *decodeFieldOptions(f.Meta), + Views: make([]*viewInfo, 0, len(f.Views)), + } + for _, viewname := range f.Views { + fi.Views = append(fi.Views, &viewInfo{Name: viewname}) + } + return fi +} + +// EncodeNodes converts a slice of Nodes into its internal representation. +func EncodeNodes(a []*Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = EncodeNode(a[i]) + } + return other +} + +// EncodeNode converts a Node into its internal representation. +func EncodeNode(n *Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: n.URI.Encode(), + IsCoordinator: n.IsCoordinator, + } +} + +// DecodeNodes converts a proto message into a slice of Nodes. +func DecodeNodes(a []*internal.Node) []*Node { + if len(a) == 0 { + return nil + } + other := make([]*Node, len(a)) + for i := range a { + other[i] = DecodeNode(a[i]) + } + return other +} + +func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus { + return &ClusterStatus{ + State: cs.State, + ClusterID: cs.ClusterID, + Nodes: DecodeNodes(cs.Nodes), + } +} + +// DecodeNode converts a proto message into a Node. +func DecodeNode(node *internal.Node) *Node { + return &Node{ + ID: node.ID, + URI: decodeURI(node.URI), + IsCoordinator: node.IsCoordinator, + } +} + +func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { + return &nodeEvent{ + Event: NodeEventType(ne.Event), + Node: DecodeNode(ne.Node), + } +} + +func encodeTopology(topology *Topology) *internal.Topology { + if topology == nil { + return nil + } + return &internal.Topology{ + ClusterID: topology.ClusterID, + NodeIDs: topology.NodeIDs, + } +} + +func decodeTopology(topology *internal.Topology) (*Topology, error) { + if topology == nil { + return nil, nil + } + + t := NewTopology() + t.ClusterID = topology.ClusterID + t.NodeIDs = topology.NodeIDs + sort.Slice(t.NodeIDs, + func(i, j int) bool { + return t.NodeIDs[i] < t.NodeIDs[j] + }) + + return t, nil +} + +type CreateShardMessage struct { + Index string + Shard uint64 +} + +func encodeCreateShardMessage(m *CreateShardMessage) *internal.CreateShardMessage { + return &internal.CreateShardMessage{ + Index: m.Index, + Shard: m.Shard, + } +} + +func decodeCreateShardMessage(pb *internal.CreateShardMessage) *CreateShardMessage { + return &CreateShardMessage{ + Index: pb.Index, + Shard: pb.Shard, + } +} + +type CreateIndexMessage struct { + Index string + Meta *IndexOptions +} + +func encodeCreateIndexMessage(m *CreateIndexMessage) *internal.CreateIndexMessage { + return &internal.CreateIndexMessage{ + Index: m.Index, + Meta: encodeIndexMeta(m.Meta), + } +} + +func decodeCreateIndexMessage(pb *internal.CreateIndexMessage) *CreateIndexMessage { + return &CreateIndexMessage{ + Index: pb.Index, + Meta: decodeIndexMeta(pb.Meta), + } +} + +func encodeIndexMeta(m *IndexOptions) *internal.IndexMeta { + return &internal.IndexMeta{ + Keys: m.Keys, + } +} + +func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions { + return &IndexOptions{ + Keys: pb.Keys, + } +} diff --git a/server.go b/server.go index 56195b427..dffeb94a0 100644 --- a/server.go +++ b/server.go @@ -483,12 +483,12 @@ func (s *Server) receiveMessage(pb proto.Message) error { return err } case *internal.ClusterStatus: - err := s.cluster.mergeClusterStatus(obj) + err := s.cluster.mergeClusterStatus(decodeClusterStatus(obj)) if err != nil { return err } case *internal.ResizeInstruction: - err := s.cluster.followResizeInstruction(obj) + err := s.cluster.followResizeInstruction(decodeResizeInstruction(obj)) if err != nil { return err } @@ -518,7 +518,7 @@ func (s *Server) receiveMessage(pb proto.Message) error { } // SendSync represents an implementation of Broadcaster. -func (s *Server) SendSync(pb proto.Message) error { +func (s *Server) SendSync(m Message) error { var eg errgroup.Group for _, node := range s.cluster.Nodes { node := node From 6417f468bb70a292d97bdcf6d7897d353440d5e0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 4 Jul 2018 16:50:20 -0500 Subject: [PATCH 002/166] wip implement more... still quite broken --- api.go | 2 +- broadcast.go | 73 ++++++++- cluster.go | 357 +++++++++++++++++++++++++++++++++++++---- holder.go | 7 +- server.go | 68 ++++---- utils_internal_test.go | 27 ++-- 6 files changed, 447 insertions(+), 87 deletions(-) diff --git a/api.go b/api.go index abf2a84ea..65ff90870 100644 --- a/api.go +++ b/api.go @@ -517,7 +517,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { } // Forward the error message. - if err := api.server.receiveMessage(pb); err != nil { + if err := api.server.receiveMessage(decode(pb)); err != nil { return errors.Wrap(err, "receiving message") } return nil diff --git a/broadcast.go b/broadcast.go index 3f6b960e4..0a33d4ca7 100644 --- a/broadcast.go +++ b/broadcast.go @@ -73,8 +73,7 @@ const ( ) // MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(pm Message) ([]byte, error) { - m := encode(pm) +func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { case *internal.CreateShardMessage: @@ -120,19 +119,45 @@ func MarshalMessage(pm Message) ([]byte, error) { } func encode(m Message) proto.Message { - var pm proto.Message switch mt := m.(type) { case *CreateShardMessage: return encodeCreateShardMessage(mt) case *CreateIndexMessage: return encodeCreateIndexMessage(mt) - // TODO, the rest + case *DeleteIndexMessage: + return encodeDeleteIndexMessage(mt) + case *CreateFieldMessage: + return encodeCreateFieldMessage(mt) + case *DeleteFieldMessage: + return encodeDeleteFieldMessage(mt) + case *CreateViewMessage: + return encodeCreateViewMessage(mt) + case *DeleteViewMessage: + return encodeDeleteViewMessage(mt) + case *ClusterStatus: + return encodeClusterStatus(mt) + case *ResizeInstruction: + return encodeResizeInstruction(mt) + case *ResizeInstructionComplete: + return encodeResizeInstructionComplete(mt) + case *SetCoordinatorMessage: + return encodeSetCoordinatorMessage(mt) + case *UpdateCoordinatorMessage: + return encodeUpdateCoordinatorMessage(mt) + case *NodeStateMessage: + return encodeNodeStateMessage(mt) + case *RecalculateCaches: + return encodeRecalculateCaches(mt) + case *nodeEvent: + return encodeNodeEventMessage(mt) + case *NodeStatus: + return encodeNodeStatus(mt) } return nil } // UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (Message, error) { +func UnmarshalMessage(buf []byte) (proto.Message, error) { typ, buf := buf[0], buf[1:] var m proto.Message switch typ { @@ -177,3 +202,41 @@ func UnmarshalMessage(buf []byte) (Message, error) { } return m, nil } + +func decode(m proto.Message) Message { + switch mt := m.(type) { + case *internal.CreateShardMessage: + return decodeCreateShardMessage(mt) + case *internal.CreateIndexMessage: + return decodeCreateIndexMessage(mt) + case *internal.DeleteIndexMessage: + return decodeDeleteIndexMessage(mt) + case *internal.CreateFieldMessage: + return decodeCreateFieldMessage(mt) + case *internal.DeleteFieldMessage: + return decodeDeleteFieldMessage(mt) + case *internal.CreateViewMessage: + return decodeCreateViewMessage(mt) + case *internal.DeleteViewMessage: + return decodeDeleteViewMessage(mt) + case *internal.ClusterStatus: + return decodeClusterStatus(mt) + case *internal.ResizeInstruction: + return decodeResizeInstruction(mt) + case *internal.ResizeInstructionComplete: + return decodeResizeInstructionComplete(mt) + case *internal.SetCoordinatorMessage: + return decodeSetCoordinatorMessage(mt) + case *internal.UpdateCoordinatorMessage: + return decodeUpdateCoordinatorMessage(mt) + case *internal.NodeStateMessage: + return decodeNodeStateMessage(mt) + case *internal.RecalculateCaches: + return decodeRecalculateCaches(mt) + case *internal.NodeEventMessage: + return decodeNodeEventMessage(mt) + case *internal.NodeStatus: + return decodeNodeStatus(mt) + } + return nil +} diff --git a/cluster.go b/cluster.go index fcfb08d73..e97ba46e4 100644 --- a/cluster.go +++ b/cluster.go @@ -268,8 +268,8 @@ func (c *cluster) setCoordinator(n *Node) error { c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.broadcaster.SendSync( - &internal.UpdateCoordinatorMessage{ - New: EncodeNode(n), + &UpdateCoordinatorMessage{ + New: n, }) if err != nil { return fmt.Errorf("problem sending UpdateCoordinator message: %v", err) @@ -423,7 +423,7 @@ func (c *cluster) setNodeState(state string) error { } // Send node state to coordinator. - ns := &internal.NodeStateMessage{ + ns := &NodeStateMessage{ NodeID: c.Node.ID, State: state, } @@ -460,12 +460,12 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { return nil } -// Status returns the internal ClusterStatus representation. -func (c *cluster) Status() *internal.ClusterStatus { - return &internal.ClusterStatus{ +// Status returns the the cluster's status including what nodes it contains, it's ID, and current state. +func (c *cluster) Status() *ClusterStatus { + return &ClusterStatus{ ClusterID: c.id, State: c.state, - Nodes: EncodeNodes(c.Nodes), + Nodes: c.Nodes, } } @@ -640,8 +640,8 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error) // fragSources returns a list of ResizeSources - for each node in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.ResizeSource, error) { - m := make(map[string][]*internal.ResizeSource) +func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) { + m := make(map[string][]*ResizeSource) // Determine if a node is being added or removed. action, diffNodeID, err := c.diff(to) @@ -700,7 +700,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R // Get the ResizeSource for each diff. for nodeID, diff := range diffs { - m[nodeID] = []*internal.ResizeSource{} + m[nodeID] = []*ResizeSource{} for _, frag := range diff { // If there is no valid source node ID for a fragment, // it likely means that the replica factor was not @@ -711,8 +711,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)") } - src := &internal.ResizeSource{ - Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)), + src := &ResizeSource{ + Node: c.unprotectedNodeByID(srcNodeID), Index: idx.Name(), Field: frag.field, View: frag.view, @@ -856,9 +856,9 @@ func (c *cluster) waitForStarted() error { // TODO: Because the normal code path already sends a NodeJoin event (via // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. - msg := &internal.NodeEventMessage{ - Event: uint32(NodeJoin), - Node: EncodeNode(c.Node), + msg := &nodeEvent{ + Event: NodeJoin, + Node: c.Node, } if err := c.broadcaster.SendSync(msg); err != nil { return fmt.Errorf("sending restart NodeJoin: %v", err) @@ -968,8 +968,8 @@ func (c *cluster) setStateAndBroadcast(state string) error { return c.broadcaster.SendSync(c.Status()) } -func (c *cluster) sendTo(node *Node, msg proto.Message) error { - if err := c.broadcaster.SendTo(node, msg); err != nil { +func (c *cluster) sendTo(node *Node, m Message) error { + if err := c.broadcaster.SendTo(node, m); err != nil { return errors.Wrap(err, "sending") } return nil @@ -1075,7 +1075,7 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, } // multiIndex is a map of sources initialized with all the nodes in toCluster. - multiIndex := make(map[string][]*internal.ResizeSource) + multiIndex := make(map[string][]*ResizeSource) for _, n := range toCluster.Nodes { multiIndex[n.ID] = nil @@ -1099,12 +1099,12 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, j.IDs[id] = true continue } - instr := &internal.ResizeInstruction{ + instr := &ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.unprotectedNodeByID(id)), - Coordinator: EncodeNode(c.coordinatorNode()), + Node: toCluster.unprotectedNodeByID(id), + Coordinator: c.coordinatorNode(), Sources: sources, - Schema: c.holder.encodeSchema(), // Include the schema to ensure it's in sync on the receiving node. + Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), } j.Instructions = append(j.Instructions, instr) @@ -1148,7 +1148,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { <-c.holder.opened // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -1167,7 +1167,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { for _, src := range instr.Sources { c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - srcURI := decodeURI(src.Node.URI) + srcURI := src.Node.URI // Retrieve field. f := c.holder.Field(src.Index, src.Field) @@ -1219,14 +1219,14 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { complete.Error = err.Error() } - if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { + if err := c.sendTo(instr.Coordinator, complete); err != nil { c.logger.Printf("sending resizeInstructionComplete error: err=%s", err) } }() return nil } -func (c *cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error { j := c.job(complete.JobID) @@ -1263,7 +1263,7 @@ func (c *cluster) job(id int64) *resizeJob { type resizeJob struct { ID int64 IDs map[string]bool - Instructions []*internal.ResizeInstruction + Instructions []*ResizeInstruction Broadcaster broadcaster action string @@ -1366,7 +1366,7 @@ func (j *resizeJob) distributeResizeInstructions() error { // a dummy node object to use in the SendTo() method. node := &Node{ ID: instr.Node.ID, - URI: decodeURI(instr.Node.URI), + URI: instr.Node.URI, } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -1757,7 +1757,7 @@ type ResizeInstruction struct { ClusterStatus *ClusterStatus } -func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction { +func decodeResizeInstruction(ri *internal.ResizeInstruction) *ResizeInstruction { return &ResizeInstruction{ JobID: ri.JobID, Node: DecodeNode(ri.Node), @@ -1768,6 +1768,17 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction) ResizeInstruction { } } +func encodeResizeInstruction(m *ResizeInstruction) *internal.ResizeInstruction { + return &internal.ResizeInstruction{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Coordinator: EncodeNode(m.Coordinator), + Sources: encodeResizeSources(m.Sources), + Schema: encodeSchema(m.Schema), + ClusterStatus: encodeClusterStatus(m.ClusterStatus), + } +} + type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -1784,6 +1795,14 @@ func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource { return new } +func encodeResizeSources(srcs []*ResizeSource) []*internal.ResizeSource { + new := make([]*internal.ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, encodeResizeSource(src)) + } + return new +} + func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { return &ResizeSource{ Node: DecodeNode(rs.Node), @@ -1794,6 +1813,16 @@ func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { } } +func encodeResizeSource(m *ResizeSource) *internal.ResizeSource { + return &internal.ResizeSource{ + Node: EncodeNode(m.Node), + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + } +} + // Schema is a schema type Schema struct { Indexes []*IndexInfo @@ -1805,6 +1834,12 @@ func decodeSchema(s *internal.Schema) *Schema { } } +func encodeSchema(m *Schema) *internal.Schema { + return &internal.Schema{ + Indexes: encodeIndexInfos(m.Indexes), + } +} + func decodeIndexes(idxs []*internal.Index) []*IndexInfo { new := make([]*IndexInfo, 0, len(idxs)) for _, idx := range idxs { @@ -1813,6 +1848,14 @@ func decodeIndexes(idxs []*internal.Index) []*IndexInfo { return new } +func encodeIndexInfos(idxs []*IndexInfo) []*internal.Index { + new := make([]*internal.Index, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, encodeIndexInfo(idx)) + } + return new +} + func decodeIndex(idx *internal.Index) *IndexInfo { return &IndexInfo{ Name: idx.Name, @@ -1820,6 +1863,13 @@ func decodeIndex(idx *internal.Index) *IndexInfo { } } +func encodeIndexInfo(idx *IndexInfo) *internal.Index { + return &internal.Index{ + Name: idx.Name, + Fields: encodeFieldInfos(idx.Fields), + } +} + func decodeFields(fs []*internal.Field) []*FieldInfo { new := make([]*FieldInfo, 0, len(fs)) for _, f := range fs { @@ -1828,6 +1878,14 @@ func decodeFields(fs []*internal.Field) []*FieldInfo { return new } +func encodeFieldInfos(fs []*FieldInfo) []*internal.Field { + new := make([]*internal.Field, 0, len(fs)) + for _, f := range fs { + new = append(new, encodeFieldInfo(f)) + } + return new +} + func decodeField(f *internal.Field) *FieldInfo { fi := &FieldInfo{ Name: f.Name, @@ -1840,6 +1898,19 @@ func decodeField(f *internal.Field) *FieldInfo { return fi } +func encodeFieldInfo(f *FieldInfo) *internal.Field { + ifield := &internal.Field{ + Name: f.Name, + Meta: encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), + } + + for _, viewinfo := range f.Views { + ifield.Views = append(ifield.Views, viewinfo.Name) + } + return ifield +} + // EncodeNodes converts a slice of Nodes into its internal representation. func EncodeNodes(a []*Node) []*internal.Node { other := make([]*internal.Node, len(a)) @@ -1878,6 +1949,14 @@ func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus { } } +func encodeClusterStatus(m *ClusterStatus) *internal.ClusterStatus { + return &internal.ClusterStatus{ + State: m.State, + ClusterID: m.ClusterID, + Nodes: EncodeNodes(m.Nodes), + } +} + // DecodeNode converts a proto message into a Node. func DecodeNode(node *internal.Node) *Node { return &Node{ @@ -1969,3 +2048,223 @@ func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions { Keys: pb.Keys, } } + +type DeleteIndexMessage struct { + Index string +} + +func encodeDeleteIndexMessage(m *DeleteIndexMessage) *internal.DeleteIndexMessage { + return &internal.DeleteIndexMessage{ + Index: m.Index, + } +} + +func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage) *DeleteIndexMessage { + return &DeleteIndexMessage{ + Index: pb.Index, + } +} + +type CreateFieldMessage struct { + Index string + Field string + Meta *FieldOptions +} + +func encodeCreateFieldMessage(m *CreateFieldMessage) *internal.CreateFieldMessage { + return &internal.CreateFieldMessage{ + Index: m.Index, + Field: m.Field, + Meta: encodeFieldOptions(m.Meta), + } +} + +func decodeCreateFieldMessage(pb *internal.CreateFieldMessage) *CreateFieldMessage { + return &CreateFieldMessage{ + Index: pb.Index, + Field: pb.Field, + Meta: decodeFieldOptions(pb.Meta), + } +} + +type DeleteFieldMessage struct { + Index string + Field string +} + +func encodeDeleteFieldMessage(m *DeleteFieldMessage) *internal.DeleteFieldMessage { + return &internal.DeleteFieldMessage{ + Index: m.Index, + Field: m.Field, + } +} + +func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage) *DeleteFieldMessage { + return &DeleteFieldMessage{ + Index: pb.Index, + Field: pb.Field, + } +} + +type CreateViewMessage struct { + Index string + Field string + View string +} + +func encodeCreateViewMessage(m *CreateViewMessage) *internal.CreateViewMessage { + return &internal.CreateViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func decodeCreateViewMessage(pb *internal.CreateViewMessage) *CreateViewMessage { + return &CreateViewMessage{ + Index: pb.Index, + Field: pb.Field, + View: pb.View, + } +} + +type DeleteViewMessage struct { + Index string + Field string + View string +} + +func encodeDeleteViewMessage(m *DeleteViewMessage) *internal.DeleteViewMessage { + return &internal.DeleteViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func decodeDeleteViewMessage(pb *internal.DeleteViewMessage) *DeleteViewMessage { + return &DeleteViewMessage{ + Index: pb.Index, + Field: pb.Field, + View: pb.View, + } +} + +type ResizeInstructionComplete struct { + JobID int64 + Node *Node + Error string +} + +func encodeResizeInstructionComplete(m *ResizeInstructionComplete) *internal.ResizeInstructionComplete { + return &internal.ResizeInstructionComplete{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Error: m.Error, + } +} + +func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete) *ResizeInstructionComplete { + return &ResizeInstructionComplete{ + JobID: pb.JobID, + Node: DecodeNode(pb.Node), + Error: pb.Error, + } +} + +type SetCoordinatorMessage struct { + New *Node +} + +func encodeSetCoordinatorMessage(m *SetCoordinatorMessage) *internal.SetCoordinatorMessage { + return &internal.SetCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage) *SetCoordinatorMessage { + return &SetCoordinatorMessage{ + New: DecodeNode(pb.New), + } +} + +type UpdateCoordinatorMessage struct { + New *Node +} + +func encodeUpdateCoordinatorMessage(m *UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { + return &internal.UpdateCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage) *UpdateCoordinatorMessage { + return &UpdateCoordinatorMessage{ + New: DecodeNode(pb.New), + } +} + +type NodeStateMessage struct { + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` +} + +func encodeNodeStateMessage(m *NodeStateMessage) *internal.NodeStateMessage { + return &internal.NodeStateMessage{ + NodeID: m.NodeID, + State: m.State, + } +} + +func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage { + return &NodeStateMessage{ + NodeID: pb.NodeID, + State: pb.State, + } +} + +func encodeNodeEventMessage(m *nodeEvent) *internal.NodeEventMessage { + return &internal.NodeEventMessage{ + Event: uint32(m.Event), + Node: EncodeNode(m.Node), + } +} + +func decodeNodeEventMessage(pb *internal.NodeEventMessage) *nodeEvent { + return &nodeEvent{ + Event: NodeEventType(pb.Event), + Node: DecodeNode(pb.Node), + } +} + +type NodeStatus struct { + Node *Node + MaxShards map[string]uint64 + Schema *Schema +} + +func encodeNodeStatus(m *NodeStatus) *internal.NodeStatus { + return &internal.NodeStatus{ + Node: EncodeNode(m.Node), + MaxShards: &internal.MaxShards{Standard: m.MaxShards}, + Schema: encodeSchema(m.Schema), + } +} + +func decodeNodeStatus(pb *internal.NodeStatus) *NodeStatus { + return &NodeStatus{ + Node: DecodeNode(pb.Node), + MaxShards: pb.MaxShards.Standard, + Schema: decodeSchema(pb.Schema), + } +} + +type RecalculateCaches struct{} + +func decodeRecalculateCaches(pb *internal.RecalculateCaches) *RecalculateCaches { + return &RecalculateCaches{} +} + +func encodeRecalculateCaches(*RecalculateCaches) *internal.RecalculateCaches { + return &internal.RecalculateCaches{} +} diff --git a/holder.go b/holder.go index e599d21f2..83ffd6967 100644 --- a/holder.go +++ b/holder.go @@ -230,7 +230,7 @@ func (h *Holder) Schema() []*IndexInfo { } // applySchema applies an internal Schema to Holder. -func (h *Holder) applySchema(schema *internal.Schema) error { +func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. for _, index := range schema.Indexes { opt := IndexOptions{} @@ -240,14 +240,13 @@ func (h *Holder) applySchema(schema *internal.Schema) error { } // Create fields that don't exist. for _, f := range index.Fields { - opt := decodeFieldOptions(f.Meta) - field, err := idx.CreateFieldIfNotExists(f.Name, *opt) + field, err := idx.CreateFieldIfNotExists(f.Name, f.Options) if err != nil { return errors.Wrap(err, "creating field") } // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v) + _, err := field.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/server.go b/server.go index dffeb94a0..7199b7178 100644 --- a/server.go +++ b/server.go @@ -27,8 +27,6 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -431,40 +429,40 @@ func (s *Server) monitorAntiEntropy() { } // receiveMessage represents an implementation of BroadcastHandler. -func (s *Server) receiveMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateShardMessage: +func (s *Server) receiveMessage(m Message) error { + switch obj := m.(type) { + case *CreateShardMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } idx.setRemoteMaxShard(obj.Shard) - case *internal.CreateIndexMessage: + case *CreateIndexMessage: opt := IndexOptions{} _, err := s.holder.CreateIndex(obj.Index, opt) if err != nil { return err } - case *internal.DeleteIndexMessage: + case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err } - case *internal.CreateFieldMessage: + case *CreateFieldMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := decodeFieldOptions(obj.Meta) + opt := obj.Meta _, err := idx.CreateField(obj.Field, *opt) if err != nil { return err } - case *internal.DeleteFieldMessage: + case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { return err } - case *internal.CreateViewMessage: + case *CreateViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) @@ -473,7 +471,7 @@ func (s *Server) receiveMessage(pb proto.Message) error { if err != nil { return err } - case *internal.DeleteViewMessage: + case *DeleteViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) @@ -482,36 +480,36 @@ func (s *Server) receiveMessage(pb proto.Message) error { if err != nil { return err } - case *internal.ClusterStatus: - err := s.cluster.mergeClusterStatus(decodeClusterStatus(obj)) + case *ClusterStatus: + err := s.cluster.mergeClusterStatus(obj) if err != nil { return err } - case *internal.ResizeInstruction: - err := s.cluster.followResizeInstruction(decodeResizeInstruction(obj)) + case *ResizeInstruction: + err := s.cluster.followResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: err := s.cluster.markResizeInstructionComplete(obj) if err != nil { return err } - case *internal.SetCoordinatorMessage: - s.cluster.setCoordinator(DecodeNode(obj.New)) - case *internal.UpdateCoordinatorMessage: - s.cluster.updateCoordinator(DecodeNode(obj.New)) - case *internal.NodeStateMessage: + case *SetCoordinatorMessage: + s.cluster.setCoordinator(obj.New) + case *UpdateCoordinatorMessage: + s.cluster.updateCoordinator(obj.New) + case *NodeStateMessage: err := s.cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } - case *internal.RecalculateCaches: + case *RecalculateCaches: s.holder.RecalculateCaches() - case *internal.NodeEventMessage: - s.cluster.ReceiveEvent(DecodeNodeEvent(obj)) - case *internal.NodeStatus: - s.handleRemoteStatus(pb) + case *nodeEvent: + s.cluster.ReceiveEvent(obj) + case *NodeStatus: + s.handleRemoteStatus(obj) } return nil @@ -519,6 +517,7 @@ func (s *Server) receiveMessage(pb proto.Message) error { // SendSync represents an implementation of Broadcaster. func (s *Server) SendSync(m Message) error { + pb := encode(m) var eg errgroup.Group for _, node := range s.cluster.Nodes { node := node @@ -537,12 +536,13 @@ func (s *Server) SendSync(m Message) error { } // SendAsync represents an implementation of Broadcaster. -func (s *Server) SendAsync(pb proto.Message) error { +func (s *Server) SendAsync(m Message) error { return ErrNotImplemented } // SendTo represents an implementation of Broadcaster. -func (s *Server) SendTo(to *Node, pb proto.Message) error { +func (s *Server) SendTo(to *Node, m Message) error { + pb := encode(m) s.logger.Printf("SendTo: %s", to.URI) return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) } @@ -554,7 +554,7 @@ func (s *Server) node() Node { } // handleRemoteStatus receives incoming NodeStatus from remote nodes. -func (s *Server) handleRemoteStatus(pb proto.Message) { +func (s *Server) handleRemoteStatus(pb Message) { // Ignore NodeStatus messages until the cluster is in a Normal state. if s.cluster.State() != ClusterStateNormal { return @@ -564,16 +564,16 @@ func (s *Server) handleRemoteStatus(pb proto.Message) { // Make sure the holder has opened. <-s.holder.opened - err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + err := s.mergeRemoteStatus(pb.(*NodeStatus)) if err != nil { s.logger.Printf("merge remote status: %s", err) } }() } -func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { +func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { // Ignore status updates from self. - if s.nodeID == DecodeNode(ns.Node).ID { + if s.nodeID == ns.Node.ID { return nil } @@ -584,7 +584,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // Sync maxShards. oldmaxshards := s.holder.maxShards() - for index, newMax := range ns.MaxShards.Standard { + for index, newMax := range ns.MaxShards { localIndex := s.holder.Index(index) // if we don't know about an index locally, log an error because // indexes should be created and synced prior to shard creation diff --git a/utils_internal_test.go b/utils_internal_test.go index ca7bd1fa7..f7c11d6a7 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -24,7 +24,6 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -304,9 +303,9 @@ func (t *ClusterCluster) Close() error { } // SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *ClusterCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: +func (t *ClusterCluster) SendSync(m Message) error { + switch obj := m.(type) { + case *ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { c.mergeClusterStatus(obj) @@ -322,19 +321,19 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { } // SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *ClusterCluster) SendAsync(pb proto.Message) error { +func (t *ClusterCluster) SendAsync(Message) error { return nil } // SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: +func (t *ClusterCluster) SendTo(to *Node, m Message) error { + switch obj := m.(type) { + case *ResizeInstruction: err := t.FollowResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: coord := t.clusterByID(to.ID) go coord.markResizeInstructionComplete(obj) } @@ -342,10 +341,10 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { } // FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error { // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -356,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi // figure out which node it was meant for, then call the operation on that cluster // basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI) - instrNode := DecodeNode(instr.Node) + instrNode := instr.Node destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. @@ -365,7 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi } for _, src := range instr.Sources { - srcNode := DecodeNode(src.Node) + srcNode := src.Node srcCluster := t.clusterByID(srcNode.ID) srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) @@ -405,6 +404,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi complete.Error = err.Error() } - node := DecodeNode(instr.Coordinator) + node := instr.Coordinator return t.SendTo(node, complete) } From cd8c63c125935bba86a0b658eac13de50a4ffbc1 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 4 Jul 2018 21:43:18 -0500 Subject: [PATCH 003/166] tests passing --- api.go | 17 ++++++++--------- cluster_internal_test.go | 41 ++++++++++++++++++++-------------------- field.go | 2 +- view.go | 3 +-- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/api.go b/api.go index 65ff90870..73bdd6665 100644 --- a/api.go +++ b/api.go @@ -185,12 +185,11 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index } // Send the create index message to all nodes. err = api.server.SendSync( - &internal.CreateIndexMessage{ + &CreateIndexMessage{ Index: indexName, - Meta: options.Encode(), + Meta: &options, }) if err != nil { - api.server.logger.Printf("problem sending CreateIndex message: %s", err) return nil, errors.Wrap(err, "sending CreateIndex message") } api.holder.Stats.Count("createIndex", 1, 1.0) @@ -224,7 +223,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // Send the delete index message to all nodes. err = api.server.SendSync( - &internal.DeleteIndexMessage{ + &DeleteIndexMessage{ Index: indexName, }) if err != nil { @@ -264,10 +263,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str // Send the create field message to all nodes. err = api.server.SendSync( - &internal.CreateFieldMessage{ + &CreateFieldMessage{ Index: indexName, Field: fieldName, - Meta: fo.Encode(), + Meta: &fo, }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) @@ -311,7 +310,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str // Send the delete field message to all nodes. err := api.server.SendSync( - &internal.DeleteFieldMessage{ + &DeleteFieldMessage{ Index: indexName, Field: fieldName, }) @@ -489,7 +488,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return errors.Wrap(err, "validating api method") } - err := api.server.SendSync(&internal.RecalculateCaches{}) + err := api.server.SendSync(&RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") } @@ -568,7 +567,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri // Send the delete view message to all nodes. err := api.server.SendSync( - &internal.DeleteViewMessage{ + &DeleteViewMessage{ Index: indexName, Field: fieldName, View: viewName, diff --git a/cluster_internal_test.go b/cluster_internal_test.go index aee4ea09b..e209a8f80 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -24,7 +24,6 @@ import ( "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -175,19 +174,19 @@ func TestFragSources(t *testing.T) { from *cluster to *cluster idx *Index - expected map[string][]*internal.ResizeSource + expected map[string][]*ResizeSource err string }{ { from: c1, to: c2, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{}, - "node1": []*internal.ResizeSource{}, - "node2": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{}, + "node1": []*ResizeSource{}, + "node2": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -196,13 +195,13 @@ func TestFragSources(t *testing.T) { from: c4, to: c3, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -211,15 +210,15 @@ func TestFragSources(t *testing.T) { from: c5, to: c4, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, }, - "node2": []*internal.ResizeSource{}, + "node2": []*ResizeSource{}, }, err: "", }, diff --git a/field.go b/field.go index 76be9cf3f..eea6bb10e 100644 --- a/field.go +++ b/field.go @@ -605,7 +605,7 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) { if created { // Broadcast view creation to the cluster. err = f.broadcaster.SendSync( - &internal.CreateViewMessage{ + &CreateViewMessage{ Index: f.index, Field: f.name, View: name, diff --git a/view.go b/view.go index fd5306b85..0f2e189cd 100644 --- a/view.go +++ b/view.go @@ -22,7 +22,6 @@ import ( "strings" "sync" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -232,7 +231,7 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) { // Send the create shard message to all nodes. err := v.broadcaster.SendSync( - &internal.CreateShardMessage{ + &CreateShardMessage{ Index: v.index, Shard: shard, }) From 1b2aaa26bf13c1b42b902a09ff125eee7ca32080 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 09:26:58 -0500 Subject: [PATCH 004/166] export NodeEvent --- broadcast.go | 2 +- cluster.go | 14 +++++++------- event.go | 4 ++-- server.go | 2 +- utils_internal_test.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/broadcast.go b/broadcast.go index 0a33d4ca7..d3c4a2562 100644 --- a/broadcast.go +++ b/broadcast.go @@ -148,7 +148,7 @@ func encode(m Message) proto.Message { return encodeNodeStateMessage(mt) case *RecalculateCaches: return encodeRecalculateCaches(mt) - case *nodeEvent: + case *NodeEvent: return encodeNodeEventMessage(mt) case *NodeStatus: return encodeNodeStatus(mt) diff --git a/cluster.go b/cluster.go index e97ba46e4..2d25ebeaa 100644 --- a/cluster.go +++ b/cluster.go @@ -856,7 +856,7 @@ func (c *cluster) waitForStarted() error { // TODO: Because the normal code path already sends a NodeJoin event (via // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. - msg := &nodeEvent{ + msg := &NodeEvent{ Event: NodeJoin, Node: c.Node, } @@ -1540,7 +1540,7 @@ func (c *cluster) considerTopology() error { } // ReceiveEvent represents an implementation of EventHandler. -func (c *cluster) ReceiveEvent(e *nodeEvent) error { +func (c *cluster) ReceiveEvent(e *NodeEvent) error { // Ignore events sent from this node. if e.Node.ID == c.Node.ID { return nil @@ -1966,8 +1966,8 @@ func DecodeNode(node *internal.Node) *Node { } } -func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ +func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { + return &NodeEvent{ Event: NodeEventType(ne.Event), Node: DecodeNode(ne.Node), } @@ -2223,15 +2223,15 @@ func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage { } } -func encodeNodeEventMessage(m *nodeEvent) *internal.NodeEventMessage { +func encodeNodeEventMessage(m *NodeEvent) *internal.NodeEventMessage { return &internal.NodeEventMessage{ Event: uint32(m.Event), Node: EncodeNode(m.Node), } } -func decodeNodeEventMessage(pb *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ +func decodeNodeEventMessage(pb *internal.NodeEventMessage) *NodeEvent { + return &NodeEvent{ Event: NodeEventType(pb.Event), Node: DecodeNode(pb.Node), } diff --git a/event.go b/event.go index aa1e0e890..0d5e59e99 100644 --- a/event.go +++ b/event.go @@ -23,8 +23,8 @@ const ( NodeUpdate ) -// nodeEvent is a single event related to node activity in the cluster. -type nodeEvent struct { +// NodeEvent is a single event related to node activity in the cluster. +type NodeEvent struct { Event NodeEventType Node *Node } diff --git a/server.go b/server.go index 7199b7178..b57935acb 100644 --- a/server.go +++ b/server.go @@ -506,7 +506,7 @@ func (s *Server) receiveMessage(m Message) error { } case *RecalculateCaches: s.holder.RecalculateCaches() - case *nodeEvent: + case *NodeEvent: s.cluster.ReceiveEvent(obj) case *NodeStatus: s.handleRemoteStatus(obj) diff --git a/utils_internal_test.go b/utils_internal_test.go index f7c11d6a7..4b9f6870a 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -161,7 +161,7 @@ func (t *ClusterCluster) addNode() error { // Send NodeJoin event to coordinator. if id > 0 { coord := t.Clusters[0] - ev := &nodeEvent{ + ev := &NodeEvent{ Event: NodeJoin, Node: c.Node, } From e9503a443ebfe15b9642cb1d053f3224341db260 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 5 Jul 2018 10:27:49 -0500 Subject: [PATCH 005/166] Begin updating frame->field and slice->shard --- docs/data-model.md | 48 +++++++++++++++++++++++----------------------- docs/glossary.md | 26 ++++++++++++++----------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 9569b1f90..ad6a9ea9d 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -22,7 +22,7 @@ The central component of Pilosa's data model is a boolean matrix. Each cell in t Rows and columns can represent anything (they could even represent the same set of things - a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix. -Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *frames* and quickly retrieves the top rows in a frame sorted by the number of bits set in each row. +Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of bits set in each row. Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 263 on a single-node cluster, for example, will not work well due to memory limitations. @@ -35,15 +35,15 @@ The purpose of the Index is to represent a data namespace. You cannot perform cr ### Column -Column ids are sequential increasing integers and are common to all Frames within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable. +Column ids are sequential increasing integers and are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable. ### Row -Row ids are sequential increasing integers namespaced to each Frame within an Index. +Row ids are sequential increasing integers namespaced to each Field within an Index. -### Frame +### Field -Frames are used to segment rows within an index, for example to define different functional groups. A frame might correspond to a single field in a relational table, where each row in a standard frame represents a single possible value of the field. Similarly, a frame with BSI values could represent all possible integer values of a field . +Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, a field with BSI values could represent all possible integer values of a relational field. #### Relational Analogy @@ -56,7 +56,7 @@ Entities: Database | N/A *(internal: Holder)* Table | Index Row | Column - Column | Frame + Column | Field Value | Row Value (int) | Field.Value (see [BSI](#bsi-range-encoding)) @@ -68,7 +68,7 @@ Simple queries: `select ID from People where Age > 30` | `Range(frame=Default, Age > 30)` `select ID from People where Member = true` | `Bitmap(frame=Member, row=[true])` -In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple frames. For example, this SQL join: +In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: ```sql select AVG(p.Age) from People p @@ -87,37 +87,37 @@ This is one major component of Pilosa's ability to combine relationships from mu #### Ranked -Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation. +Ranked Fields maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Field creation. -![ranked frame diagram](/img/docs/frame-ranked.svg) -*Ranked frame diagram* +![ranked field diagram](/img/docs/field-ranked.svg) +*Ranked field diagram* #### LRU The LRU cache maintains the most recently accessed Rows. -![lru frame diagram](/img/docs/frame-lru.svg) -*LRU frame diagram* +![lru field diagram](/img/docs/field-lru.svg) +*LRU field diagram* ### Time Quantum -Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. +Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. ### Attribute Attributes are arbitrary key/value pairs that can be associated with either rows or columns. This metadata is stored in a separate BoltDB data structure. -Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all frames in an index. Row attributes apply to all bits in the corresponding row. +Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all fields in an index. Row attributes apply to all bits in the corresponding row. -### Slice +### Shard -Indexes are sharded into groups of columns called Slices. Each Slice contains a fixed number of columns, which is the SliceWidth. SliceWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 220. +Indexes are segmented into groups of columns called shards (previously known as slices). Each shard contains a fixed number of columns, which is the ShardWidth. ShardWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 220. Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm. ### View -Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. +Views represent the various data layouts within a Field. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. #### Standard @@ -125,21 +125,21 @@ The standard View contains the same Row/Column format as the input data. #### Time Quantums -If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the diagram below: +If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the diagram below: ``` SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00") SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") ``` -![time quantum frame diagram](/img/docs/frame-time-quantum.svg) -*Time quantum frame diagram* +![time quantum field diagram](/img/docs/field-time-quantum.svg) +*Time quantum fueld diagram* #### BSI Range-Encoding -Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. +Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI (TODO!!!!!) `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `SetFieldValue()` queries will result in the data described in the diagram below: @@ -152,7 +152,7 @@ SetFieldValue(col=2, frame="A", field1=1) SetFieldValue(col=3, frame="A", field1=6) ``` -![BSI frame diagram](/img/docs/frame-bsi.svg) -*BSI frame diagram* +![BSI field diagram](/img/docs/field-bsi.svg) +*BSI field diagram* Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa. diff --git a/docs/glossary.md b/docs/glossary.md index fb1a46690..78ca7c1fe 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -6,11 +6,11 @@ nav = [] ## Glossary -[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. +[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [shard](#shard) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. [Attribute](../data-model/#attribute): Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store. -[Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column). +[Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column). [Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. @@ -18,13 +18,15 @@ nav = [] Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. -[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). +[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index). [Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries. -Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). +Fragment: A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index). -[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. +[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. + +[Frame](../data-model/#field): Prior to Pilosa 1.0, fields were known as frames. [Gossip](https://en.wikipedia.org/wiki/Gossip_protocol): A protocol used by Pilosa for internal communication. @@ -34,7 +36,7 @@ nav = [] [Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). -MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. +MaxShard: The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. [Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). @@ -54,11 +56,13 @@ nav = [] [Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. -[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap). +[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap). -[Slice](../data-model/#slice): [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). +[Slice](../data-model/#slice): Prior to Pilosa 1.0, shards were known as slices. -SliceWidth: This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. +[Shard](../data-model/#shard): [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). + +ShardWidth: This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. [Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field). @@ -68,6 +72,6 @@ nav = [] [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). -[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [field](#field). -[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[View](../data-model/#view): Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. From 2cec75e39931f46d31842d16f273d67826fcc00c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 12:00:42 -0500 Subject: [PATCH 006/166] add proto encoding subpackage and use for send and receive message --- api.go | 9 +- broadcast.go | 116 +++++--- client.go | 5 +- cluster.go | 4 +- encoding/proto/proto.go | 594 ++++++++++++++++++++++++++++++++++++++++ field.go | 6 +- gossip/gossip.go | 6 +- holder.go | 2 +- http/client.go | 14 +- server.go | 26 +- server/server.go | 14 +- uri.go | 74 ++--- uri_internal_test.go | 24 +- view.go | 6 +- 14 files changed, 778 insertions(+), 122 deletions(-) create mode 100644 encoding/proto/proto.go diff --git a/api.go b/api.go index 73bdd6665..09f0531d5 100644 --- a/api.go +++ b/api.go @@ -509,14 +509,15 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { return errors.Wrap(err, "reading body") } - // Marshal into request object. - pb, err := UnmarshalMessage(body) + typ := body[0] + msg := getMessage(typ) + err = api.server.serializer.Unmarshal(body[1:], msg) if err != nil { - return errors.Wrap(err, "unmarshaling message") + return errors.Wrap(err, "deserializing cluster message") } // Forward the error message. - if err := api.server.receiveMessage(decode(pb)); err != nil { + if err := api.server.receiveMessage(msg); err != nil { return errors.Wrap(err, "receiving message") } return nil diff --git a/broadcast.go b/broadcast.go index d3c4a2562..1da6ca57f 100644 --- a/broadcast.go +++ b/broadcast.go @@ -23,6 +23,12 @@ import ( "github.com/pkg/errors" ) +// Serializer is an interface for serializing pilosa types to bytes and back. +type Serializer interface { + Marshal(Message) ([]byte, error) + Unmarshal([]byte, Message) error +} + // broadcaster is an interface for broadcasting messages. type broadcaster interface { SendSync(Message) error @@ -118,42 +124,82 @@ func MarshalMessage(m proto.Message) ([]byte, error) { return append([]byte{typ}, buf...), nil } -func encode(m Message) proto.Message { - switch mt := m.(type) { - case *CreateShardMessage: - return encodeCreateShardMessage(mt) - case *CreateIndexMessage: - return encodeCreateIndexMessage(mt) - case *DeleteIndexMessage: - return encodeDeleteIndexMessage(mt) - case *CreateFieldMessage: - return encodeCreateFieldMessage(mt) - case *DeleteFieldMessage: - return encodeDeleteFieldMessage(mt) - case *CreateViewMessage: - return encodeCreateViewMessage(mt) - case *DeleteViewMessage: - return encodeDeleteViewMessage(mt) - case *ClusterStatus: - return encodeClusterStatus(mt) - case *ResizeInstruction: - return encodeResizeInstruction(mt) - case *ResizeInstructionComplete: - return encodeResizeInstructionComplete(mt) - case *SetCoordinatorMessage: - return encodeSetCoordinatorMessage(mt) - case *UpdateCoordinatorMessage: - return encodeUpdateCoordinatorMessage(mt) - case *NodeStateMessage: - return encodeNodeStateMessage(mt) - case *RecalculateCaches: - return encodeRecalculateCaches(mt) - case *NodeEvent: - return encodeNodeEventMessage(mt) - case *NodeStatus: - return encodeNodeStatus(mt) +func getMessage(typ byte) Message { + switch typ { + case messageTypeCreateShard: + return &CreateShardMessage{} + case messageTypeCreateIndex: + return &CreateIndexMessage{} + case messageTypeDeleteIndex: + return &DeleteIndexMessage{} + case messageTypeCreateField: + return &CreateFieldMessage{} + case messageTypeDeleteField: + return &DeleteFieldMessage{} + case messageTypeCreateView: + return &CreateViewMessage{} + case messageTypeDeleteView: + return &DeleteViewMessage{} + case messageTypeClusterStatus: + return &ClusterStatus{} + case messageTypeResizeInstruction: + return &ResizeInstruction{} + case messageTypeResizeInstructionComplete: + return &ResizeInstructionComplete{} + case messageTypeSetCoordinator: + return &SetCoordinatorMessage{} + case messageTypeUpdateCoordinator: + return &UpdateCoordinatorMessage{} + case messageTypeNodeState: + return &NodeStateMessage{} + case messageTypeRecalculateCaches: + return &RecalculateCaches{} + case messageTypeNodeEvent: + return &NodeEvent{} + case messageTypeNodeStatus: + return &NodeStatus{} + default: + panic(fmt.Sprintf("unknown message type %d", typ)) + } +} + +func getMessageType(m Message) byte { + switch m.(type) { + case *CreateShardMessage: + return messageTypeCreateShard + case *CreateIndexMessage: + return messageTypeCreateIndex + case *DeleteIndexMessage: + return messageTypeDeleteIndex + case *CreateFieldMessage: + return messageTypeCreateField + case *DeleteFieldMessage: + return messageTypeDeleteField + case *CreateViewMessage: + return messageTypeCreateView + case *DeleteViewMessage: + return messageTypeDeleteView + case *ClusterStatus: + return messageTypeClusterStatus + case *ResizeInstruction: + return messageTypeResizeInstruction + case *ResizeInstructionComplete: + return messageTypeResizeInstructionComplete + case *SetCoordinatorMessage: + return messageTypeSetCoordinator + case *UpdateCoordinatorMessage: + return messageTypeUpdateCoordinator + case *NodeStateMessage: + return messageTypeNodeState + case *RecalculateCaches: + return messageTypeRecalculateCaches + case *NodeEvent: + return messageTypeNodeEvent + case *NodeStatus: + return messageTypeNodeStatus + default: + panic(fmt.Sprintf("don't have type for message %#v", m)) } - return nil } // UnmarshalMessage decodes the byte slice into a protobuf message. diff --git a/client.go b/client.go index 59e3ad59b..01e0b847a 100644 --- a/client.go +++ b/client.go @@ -4,7 +4,6 @@ import ( "context" "io" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -49,7 +48,7 @@ type InternalClient interface { BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) } @@ -128,7 +127,7 @@ func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index s func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { +func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { return nil } func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { diff --git a/cluster.go b/cluster.go index 2d25ebeaa..e2f29fde5 100644 --- a/cluster.go +++ b/cluster.go @@ -1890,10 +1890,10 @@ func decodeField(f *internal.Field) *FieldInfo { fi := &FieldInfo{ Name: f.Name, Options: *decodeFieldOptions(f.Meta), - Views: make([]*viewInfo, 0, len(f.Views)), + Views: make([]*ViewInfo, 0, len(f.Views)), } for _, viewname := range f.Views { - fi.Views = append(fi.Views, &viewInfo{Name: viewname}) + fi.Views = append(fi.Views, &ViewInfo{Name: viewname}) } return fi } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go new file mode 100644 index 000000000..1c83ba4ea --- /dev/null +++ b/encoding/proto/proto.go @@ -0,0 +1,594 @@ +package proto + +import ( + "fmt" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// Serializer implements pilosa.Serializer for protobufs. +type Serializer struct{} + +// Marshal turns pilosa messages into protobuf serialized bytes. +func (Serializer) Marshal(m pilosa.Message) ([]byte, error) { + pm := encodeToProto(m) + if pm == nil { + return nil, errors.New("passed invalid pilosa.Message") + } + buf, err := proto.Marshal(pm) + return buf, errors.Wrap(err, "marshalling") +} + +// Unmarshal takes byte slices and protobuf deserializes them into a pilosa Message. +func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + msg := &internal.CreateShardMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateShardMessage") + } + decodeCreateShardMessage(msg, mt) + return nil + case *pilosa.CreateIndexMessage: + msg := &internal.CreateIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateIndexMessage") + } + decodeCreateIndexMessage(msg, mt) + return nil + case *pilosa.DeleteIndexMessage: + msg := &internal.DeleteIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteIndexMessage") + } + decodeDeleteIndexMessage(msg, mt) + return nil + case *pilosa.CreateFieldMessage: + msg := &internal.CreateFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateFieldMessage") + } + decodeCreateFieldMessage(msg, mt) + return nil + case *pilosa.DeleteFieldMessage: + msg := &internal.DeleteFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteFieldMessage") + } + decodeDeleteFieldMessage(msg, mt) + return nil + case *pilosa.CreateViewMessage: + msg := &internal.CreateViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateViewMessage") + } + decodeCreateViewMessage(msg, mt) + return nil + case *pilosa.DeleteViewMessage: + msg := &internal.DeleteViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteViewMessage") + } + decodeDeleteViewMessage(msg, mt) + return nil + case *pilosa.ClusterStatus: + msg := &internal.ClusterStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ClusterStatus") + } + decodeClusterStatus(msg, mt) + return nil + case *pilosa.ResizeInstruction: + msg := &internal.ResizeInstruction{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstruction") + } + decodeResizeInstruction(msg, mt) + return nil + case *pilosa.ResizeInstructionComplete: + msg := &internal.ResizeInstructionComplete{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstructionComplete") + } + decodeResizeInstructionComplete(msg, mt) + return nil + case *pilosa.SetCoordinatorMessage: + msg := &internal.SetCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling SetCoordinatorMessage") + } + decodeSetCoordinatorMessage(msg, mt) + return nil + case *pilosa.UpdateCoordinatorMessage: + msg := &internal.UpdateCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling UpdateCoordinatorMessage") + } + decodeUpdateCoordinatorMessage(msg, mt) + return nil + case *pilosa.NodeStateMessage: + msg := &internal.NodeStateMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStateMessage") + } + decodeNodeStateMessage(msg, mt) + return nil + case *pilosa.RecalculateCaches: + msg := &internal.RecalculateCaches{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling RecalculateCaches") + } + decodeRecalculateCaches(msg, mt) + return nil + case *pilosa.NodeEvent: + msg := &internal.NodeEventMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeEvent") + } + decodeNodeEventMessage(msg, mt) + return nil + case *pilosa.NodeStatus: + msg := &internal.NodeStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStatus") + } + decodeNodeStatus(msg, mt) + return nil + default: + panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) + } +} + +func encodeToProto(m pilosa.Message) proto.Message { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + return encodeCreateShardMessage(mt) + case *pilosa.CreateIndexMessage: + return encodeCreateIndexMessage(mt) + case *pilosa.DeleteIndexMessage: + return encodeDeleteIndexMessage(mt) + case *pilosa.CreateFieldMessage: + return encodeCreateFieldMessage(mt) + case *pilosa.DeleteFieldMessage: + return encodeDeleteFieldMessage(mt) + case *pilosa.CreateViewMessage: + return encodeCreateViewMessage(mt) + case *pilosa.DeleteViewMessage: + return encodeDeleteViewMessage(mt) + case *pilosa.ClusterStatus: + return encodeClusterStatus(mt) + case *pilosa.ResizeInstruction: + return encodeResizeInstruction(mt) + case *pilosa.ResizeInstructionComplete: + return encodeResizeInstructionComplete(mt) + case *pilosa.SetCoordinatorMessage: + return encodeSetCoordinatorMessage(mt) + case *pilosa.UpdateCoordinatorMessage: + return encodeUpdateCoordinatorMessage(mt) + case *pilosa.NodeStateMessage: + return encodeNodeStateMessage(mt) + case *pilosa.RecalculateCaches: + return encodeRecalculateCaches(mt) + case *pilosa.NodeEvent: + return encodeNodeEventMessage(mt) + case *pilosa.NodeStatus: + return encodeNodeStatus(mt) + } + return nil +} + +func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { + return &internal.ResizeInstruction{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Coordinator: EncodeNode(m.Coordinator), + Sources: encodeResizeSources(m.Sources), + Schema: encodeSchema(m.Schema), + ClusterStatus: encodeClusterStatus(m.ClusterStatus), + } +} + +func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource { + new := make([]*internal.ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, encodeResizeSource(src)) + } + return new +} + +func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource { + return &internal.ResizeSource{ + Node: EncodeNode(m.Node), + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + } +} + +func encodeSchema(m *pilosa.Schema) *internal.Schema { + return &internal.Schema{ + Indexes: encodeIndexInfos(m.Indexes), + } +} + +func encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index { + new := make([]*internal.Index, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, encodeIndexInfo(idx)) + } + return new +} + +func encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { + return &internal.Index{ + Name: idx.Name, + Fields: encodeFieldInfos(idx.Fields), + } +} + +func encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { + new := make([]*internal.Field, 0, len(fs)) + for _, f := range fs { + new = append(new, encodeFieldInfo(f)) + } + return new +} + +func encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { + ifield := &internal.Field{ + Name: f.Name, + Meta: encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), + } + + for _, viewinfo := range f.Views { + ifield.Views = append(ifield.Views, viewinfo.Name) + } + return ifield +} + +func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { + if o == nil { + return nil + } + return &internal.FieldOptions{ + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + } +} + +// EncodeNodes converts a slice of Nodes into its internal representation. +func EncodeNodes(a []*pilosa.Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = EncodeNode(a[i]) + } + return other +} + +// EncodeNode converts a Node into its internal representation. +func EncodeNode(n *pilosa.Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: n.URI.Encode(), + IsCoordinator: n.IsCoordinator, + } +} + +func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { + return &internal.ClusterStatus{ + State: m.State, + ClusterID: m.ClusterID, + Nodes: EncodeNodes(m.Nodes), + } +} + +func encodeCreateShardMessage(m *pilosa.CreateShardMessage) *internal.CreateShardMessage { + return &internal.CreateShardMessage{ + Index: m.Index, + Shard: m.Shard, + } +} + +func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { + return &internal.CreateIndexMessage{ + Index: m.Index, + Meta: encodeIndexMeta(m.Meta), + } +} + +func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta { + return &internal.IndexMeta{ + Keys: m.Keys, + } +} + +func encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *internal.DeleteIndexMessage { + return &internal.DeleteIndexMessage{ + Index: m.Index, + } +} + +func encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { + return &internal.CreateFieldMessage{ + Index: m.Index, + Field: m.Field, + Meta: encodeFieldOptions(m.Meta), + } +} + +func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFieldMessage { + return &internal.DeleteFieldMessage{ + Index: m.Index, + Field: m.Field, + } +} + +func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage { + return &internal.CreateViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMessage { + return &internal.DeleteViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete { + return &internal.ResizeInstructionComplete{ + JobID: m.JobID, + Node: EncodeNode(m.Node), + Error: m.Error, + } +} + +func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage { + return &internal.SetCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { + return &internal.UpdateCoordinatorMessage{ + New: EncodeNode(m.New), + } +} + +func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessage { + return &internal.NodeStateMessage{ + NodeID: m.NodeID, + State: m.State, + } +} + +func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage { + return &internal.NodeEventMessage{ + Event: uint32(m.Event), + Node: EncodeNode(m.Node), + } +} + +func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus { + return &internal.NodeStatus{ + Node: EncodeNode(m.Node), + MaxShards: &internal.MaxShards{Standard: m.MaxShards}, + Schema: encodeSchema(m.Schema), + } +} + +func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCaches { + return &internal.RecalculateCaches{} +} + +func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { + m.JobID = ri.JobID + m.Node = &pilosa.Node{} + decodeNode(ri.Node, m.Node) + m.Coordinator = &pilosa.Node{} + decodeNode(ri.Coordinator, m.Coordinator) + m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) + decodeResizeSources(ri.Sources, m.Sources) + m.Schema = &pilosa.Schema{} + decodeSchema(ri.Schema, m.Schema) + m.ClusterStatus = &pilosa.ClusterStatus{} + decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus) +} + +func decodeResizeSources(srcs []*internal.ResizeSource, m []*pilosa.ResizeSource) { + for i := range srcs { + m[i] = &pilosa.ResizeSource{} + decodeResizeSource(srcs[i], m[i]) + } +} + +func decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) { + m.Node = &pilosa.Node{} + decodeNode(rs.Node, m.Node) + m.Index = rs.Index + m.Field = rs.Field + m.View = rs.View + m.Shard = rs.Shard +} + +func decodeSchema(s *internal.Schema, m *pilosa.Schema) { + m.Indexes = make([]*pilosa.IndexInfo, len(s.Indexes)) + decodeIndexes(s.Indexes, m.Indexes) +} + +func decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) { + for i := range idxs { + m[i] = &pilosa.IndexInfo{} + decodeIndex(idxs[i], m[i]) + } +} + +func decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { + m.Name = idx.Name + m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) + decodeFields(idx.Fields, m.Fields) +} + +func decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { + for i := range fs { + m[i] = &pilosa.FieldInfo{} + decodeField(fs[i], m[i]) + } +} + +func decodeField(f *internal.Field, m *pilosa.FieldInfo) { + m.Name = f.Name + m.Options = pilosa.FieldOptions{} + decodeFieldOptions(f.Meta, &m.Options) + m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) + for _, viewname := range f.Views { + m.Views = append(m.Views, &pilosa.ViewInfo{Name: viewname}) + } +} + +func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) { + m.Type = options.Type + m.CacheType = options.CacheType + m.CacheSize = options.CacheSize + m.Min = options.Min + m.Max = options.Max + m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) + m.Keys = options.Keys +} + +func decodeNodes(a []*internal.Node, m []*pilosa.Node) { + for i := range a { + m[i] = &pilosa.Node{} + decodeNode(a[i], m[i]) + } +} + +func decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) { + m.State = cs.State + m.ClusterID = cs.ClusterID + m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) + decodeNodes(cs.Nodes, m.Nodes) +} + +func decodeNode(node *internal.Node, m *pilosa.Node) { + m.ID = node.ID + decodeURI(node.URI, &m.URI) + m.IsCoordinator = node.IsCoordinator +} + +func decodeURI(i *internal.URI, m *pilosa.URI) { + m.Scheme = i.Scheme + m.Host = i.Host + m.Port = uint16(i.Port) +} + +func decodeCreateShardMessage(pb *internal.CreateShardMessage, m *pilosa.CreateShardMessage) { + m.Index = pb.Index + m.Shard = pb.Shard +} + +func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { + m.Index = pb.Index + m.Meta = &pilosa.IndexOptions{} + decodeIndexMeta(pb.Meta, m.Meta) +} + +func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) { + m.Keys = pb.Keys +} + +func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) { + m.Index = pb.Index +} + +func decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.Meta = &pilosa.FieldOptions{} + decodeFieldOptions(pb.Meta, m.Meta) +} + +func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field +} + +func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *pilosa.DeleteViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) { + m.JobID = pb.JobID + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.Error = pb.Error +} + +func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pilosa.NodeStateMessage) { + m.NodeID = pb.NodeID + m.State = pb.State +} + +func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) { + m.Event = pilosa.NodeEventType(pb.Event) + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) +} + +func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.MaxShards = pb.MaxShards.Standard + m.Schema = &pilosa.Schema{} + decodeSchema(pb.Schema, m.Schema) +} + +func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {} diff --git a/field.go b/field.go index eea6bb10e..fb7e9af4f 100644 --- a/field.go +++ b/field.go @@ -1077,13 +1077,13 @@ func (f *Field) MarshalJSON() ([]byte, error) { thing := struct { Name string Options FieldOptions - Views []*viewInfo + Views []*ViewInfo }{ Name: f.Name(), Options: f.Options(), } for _, viewname := range f.viewNames() { - thing.Views = append(thing.Views, &viewInfo{Name: viewname}) + thing.Views = append(thing.Views, &ViewInfo{Name: viewname}) } return json.Marshal(thing) } @@ -1117,7 +1117,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } type FieldInfo struct { Name string `json:"name"` Options FieldOptions `json:"options"` - Views []*viewInfo `json:"views,omitempty"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/gossip/gossip.go b/gossip/gossip.go index 3dc7c202b..cd1d9ad0b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -148,7 +148,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - host := api.Node().URI.Host() + host := api.Node().URI.GetHost() g := &GossipMemberSet{ papi: api, Logger: pilosa.NopLogger, @@ -193,10 +193,10 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO conf := memberlist.DefaultWANConfig() conf.Transport = g.transport.Net conf.Name = api.Node().ID - conf.BindAddr = api.Node().URI.Host() + conf.BindAddr = api.Node().URI.GetHost() conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = hostToIP(api.Node().URI.Host()) + conf.AdvertiseAddr = hostToIP(api.Node().URI.GetHost()) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult diff --git a/holder.go b/holder.go index 83ffd6967..10e147098 100644 --- a/holder.go +++ b/holder.go @@ -217,7 +217,7 @@ func (h *Holder) Schema() []*IndexInfo { for _, field := range index.Fields() { fi := &FieldInfo{Name: field.Name(), Options: field.Options()} for _, view := range field.views() { - fi.Views = append(fi.Views, &viewInfo{Name: view.name}) + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) diff --git a/http/client.go b/http/client.go index 27597a5cb..41602b778 100644 --- a/http/client.go +++ b/http/client.go @@ -31,6 +31,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + pilosaproto "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -43,6 +44,7 @@ type ClientOptions struct { // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { defaultURI *pilosa.URI + serializer pilosa.Serializer // The client to use for HTTP communication. HTTPClient *http.Client @@ -66,6 +68,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { return &InternalClient{ defaultURI: defaultURI, + serializer: pilosaproto.Serializer{}, HTTPClient: remoteClient, } } @@ -819,12 +822,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index } // SendMessage posts a message synchronously. -func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - +func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error { u := uriPathToURL(uri, "/internal/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) if err != nil { @@ -998,7 +996,7 @@ func pos(rowID, columnID uint64) uint64 { func uriPathToURL(uri *pilosa.URI, path string) url.URL { return url.URL{ - Scheme: uri.Scheme(), + Scheme: uri.GetScheme(), Host: uri.HostPort(), Path: path, } @@ -1006,7 +1004,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL { func nodePathToURL(node *pilosa.Node, path string) url.URL { return url.URL{ - Scheme: node.URI.Scheme(), + Scheme: node.URI.GetScheme(), Host: node.URI.HostPort(), Path: path, } diff --git a/server.go b/server.go index b57935acb..b3bc6e224 100644 --- a/server.go +++ b/server.go @@ -54,6 +54,7 @@ type Server struct { executor *executor hosts []string clusterDisabled bool + serializer Serializer // External systemInfo SystemInfo @@ -200,6 +201,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +func OptServerSerializer(ser Serializer) ServerOption { + return func(s *Server) error { + s.serializer = ser + return nil + } +} + func OptServerIsCoordinator(is bool) ServerOption { return func(s *Server) error { s.isCoordinator = is @@ -517,8 +525,12 @@ func (s *Server) receiveMessage(m Message) error { // SendSync represents an implementation of Broadcaster. func (s *Server) SendSync(m Message) error { - pb := encode(m) var eg errgroup.Group + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) for _, node := range s.cluster.Nodes { node := node s.logger.Printf("SendSync to: %s", node.URI) @@ -528,7 +540,7 @@ func (s *Server) SendSync(m Message) error { } eg.Go(func() error { - return s.defaultClient.SendMessage(context.Background(), &node.URI, pb) + return s.defaultClient.SendMessage(context.Background(), &node.URI, msg) }) } @@ -542,9 +554,13 @@ func (s *Server) SendAsync(m Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, m Message) error { - pb := encode(m) s.logger.Printf("SendTo: %s", to.URI) - return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) + return s.defaultClient.SendMessage(context.Background(), &to.URI, msg) } // node returns the pilosa.node object. It is used by membership protocols to @@ -613,7 +629,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) - s.diagnostics.Set("Host", s.uri.host) + s.diagnostics.Set("Host", s.uri.Host) s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) diff --git a/server/server.go b/server/server.go index c8de0664c..f46f6a64a 100644 --- a/server/server.go +++ b/server/server.go @@ -35,6 +35,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" @@ -202,7 +203,7 @@ func (m *Command) SetupServer() error { // Setup TLS var TLSConfig *tls.Config - if uri.Scheme() == "https" { + if uri.GetScheme() == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") } @@ -235,7 +236,7 @@ func (m *Command) SetupServer() error { } // If port is 0, get auto-allocated port from listener - if uri.Port() == 0 { + if uri.GetPort() == 0 { uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) } @@ -271,6 +272,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerSerializer(proto.Serializer{}), coordinatorOpt, } @@ -309,7 +311,7 @@ func (m *Command) SetupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.API.Node().URI.Host() + gossipHost := m.API.Node().URI.GetHost() m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return errors.Wrap(err, "getting transport") @@ -366,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { // getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS - if uri.Scheme() == "https" && tlsconf != nil { + if uri.GetScheme() == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) if err != nil { return nil, errors.Wrap(err, "tls.Listener") } - } else if uri.Scheme() == "http" { + } else if uri.GetScheme() == "http" { // Open HTTP listener to determine port (if specified as :0). ln, err = net.Listen("tcp", uri.HostPort()) if err != nil { return nil, errors.Wrap(err, "net.Listen") } } else { - return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + return nil, errors.Errorf("unsupported scheme: %s", uri.GetScheme()) } return ln, nil diff --git a/uri.go b/uri.go index 5d823d8b6..2058f70fa 100644 --- a/uri.go +++ b/uri.go @@ -43,17 +43,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a- // localhost // :10101 type URI struct { - scheme string `json:"scheme"` - host string `json:"host"` - port uint16 `json:"port"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` } // DefaultURI creates and returns the default URI. func DefaultURI() *URI { return &URI{ - scheme: "http", - host: "localhost", - port: 10101, + Scheme: "http", + Host: "localhost", + Port: 10101, } } @@ -83,9 +83,9 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// Scheme returns the scheme of this URI. -func (u *URI) Scheme() string { - return u.scheme +// GetScheme returns the scheme of this URI. +func (u *URI) GetScheme() string { + return u.Scheme } // SetScheme sets the scheme of this URI. @@ -94,13 +94,13 @@ func (u *URI) SetScheme(scheme string) error { if m == nil { return errors.New("invalid scheme") } - u.scheme = scheme + u.Scheme = scheme return nil } -// Host returns the host of this URI. -func (u *URI) Host() string { - return u.host +// GetHost returns the host of this URI. +func (u *URI) GetHost() string { + return u.Host } // SetHost sets the host of this URI. @@ -109,18 +109,18 @@ func (u *URI) SetHost(host string) error { if m == nil { return errors.New("invalid host") } - u.host = host + u.Host = host return nil } -// Port returns the port of this URI. -func (u *URI) Port() uint16 { - return u.port +// GetPort returns the port of this URI. +func (u *URI) GetPort() uint16 { + return u.Port } // SetPort sets the port of this URI. func (u *URI) SetPort(port uint16) { - u.port = port + u.Port = port } // HostPort returns `Host:Port` @@ -129,23 +129,23 @@ func (u *URI) HostPort() string { if u == nil { return "" } - s := fmt.Sprintf("%s:%d", u.host, u.port) + s := fmt.Sprintf("%s:%d", u.Host, u.Port) return s } // Normalize returns the address in a form usable by a HTTP client. func (u *URI) Normalize() string { - scheme := u.scheme + scheme := u.Scheme index := strings.Index(scheme, "+") if index >= 0 { scheme = scheme[:index] } - return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port) + return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port) } // String returns the address as a string. func (u URI) String() string { - return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port) + return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port) } // Equals returns true if the checked URI is equivalent to this URI. @@ -199,9 +199,9 @@ func parseAddress(address string) (uri *URI, err error) { } } uri = &URI{ - scheme: scheme, - host: host, - port: uint16(port), + Scheme: scheme, + Host: host, + Port: uint16(port), } return uri, nil } @@ -213,9 +213,9 @@ func (u URI) Encode() *internal.URI { func encodeURI(u URI) *internal.URI { return &internal.URI{ - Scheme: u.scheme, - Host: u.host, - Port: uint32(u.port), + Scheme: u.Scheme, + Host: u.Host, + Port: uint32(u.Port), } } @@ -228,9 +228,9 @@ func decodeURI(i *internal.URI) URI { return URI{} } return URI{ - scheme: i.Scheme, - host: i.Host, - port: uint16(i.Port), + Scheme: i.Scheme, + Host: i.Host, + Port: uint16(i.Port), } } @@ -241,9 +241,9 @@ func (u *URI) MarshalJSON() ([]byte, error) { Host string `json:"host,omitempty"` Port uint16 `json:"port,omitempty"` } - output.Scheme = u.scheme - output.Host = u.host - output.Port = u.port + output.Scheme = u.Scheme + output.Host = u.Host + output.Port = u.Port return json.Marshal(output) } @@ -257,8 +257,8 @@ func (u *URI) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &input); err != nil { return err } - u.scheme = input.Scheme - u.host = input.Host - u.port = input.Port + u.Scheme = input.Scheme + u.Host = input.Host + u.Port = input.Port return nil } diff --git a/uri_internal_test.go b/uri_internal_test.go index dbcbfa04d..2aac9651f 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -93,8 +93,8 @@ func TestSetScheme(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.Scheme() != target { - t.Fatalf("%s != %s", uri.Scheme(), target) + if uri.GetScheme() != target { + t.Fatalf("%s != %s", uri.GetScheme(), target) } } @@ -105,8 +105,8 @@ func TestSetHost(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.Host() != target { - t.Fatalf("%s != %s", uri.host, target) + if uri.GetHost() != target { + t.Fatalf("%s != %s", uri.Host, target) } } @@ -114,8 +114,8 @@ func TestSetPort(t *testing.T) { uri := DefaultURI() target := uint16(9999) uri.SetPort(target) - if uri.Port() != target { - t.Fatalf("%d != %d", uri.port, target) + if uri.GetPort() != target { + t.Fatalf("%d != %d", uri.Port, target) } } @@ -147,14 +147,14 @@ func TestHostPort(t *testing.T) { } func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { - if uri.Scheme() != scheme { - t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme) + if uri.GetScheme() != scheme { + t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme) } - if uri.Host() != host { - t.Fatalf("Host does not match: %s != %s", uri.host, host) + if uri.GetHost() != host { + t.Fatalf("Host does not match: %s != %s", uri.Host, host) } - if uri.Port() != port { - t.Fatalf("Port does not match: %d != %d", uri.port, port) + if uri.GetPort() != port { + t.Fatalf("Port does not match: %d != %d", uri.Port, port) } } diff --git a/view.go b/view.go index 0f2e189cd..609664304 100644 --- a/view.go +++ b/view.go @@ -421,12 +421,12 @@ func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (* return r, nil } -// viewInfo represents schema information for a view. -type viewInfo struct { +// ViewInfo represents schema information for a view. +type ViewInfo struct { Name string `json:"name"` } -type viewInfoSlice []*viewInfo +type viewInfoSlice []*ViewInfo func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p viewInfoSlice) Len() int { return len(p) } From 6309d3b7f79a3b2a222e2feb30e754076aa82f7d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 12:32:49 -0500 Subject: [PATCH 007/166] get gossip using serializer stuff, remove proto and internal --- api.go | 4 ++++ broadcast.go | 48 +++++--------------------------------- broadcast_test.go | 51 ----------------------------------------- encoding/proto/proto.go | 32 +++++++++++++++++--------- gossip/gossip.go | 49 ++++++++++++++++++++------------------- 5 files changed, 57 insertions(+), 127 deletions(-) delete mode 100644 broadcast_test.go diff --git a/api.go b/api.go index 09f0531d5..cae91b20f 100644 --- a/api.go +++ b/api.go @@ -149,6 +149,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return resp, nil } +func (api *API) Holder() *Holder { + return api.server.Holder() +} + // readColumnAttrSets returns a list of column attribute objects by id. func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { if index == nil { diff --git a/broadcast.go b/broadcast.go index 1da6ca57f..00835db64 100644 --- a/broadcast.go +++ b/broadcast.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "reflect" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -78,48 +77,13 @@ const ( messageTypeNodeStatus ) -// MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(m proto.Message) ([]byte, error) { - var typ uint8 - switch obj := m.(type) { - case *internal.CreateShardMessage: - typ = messageTypeCreateShard - case *internal.CreateIndexMessage: - typ = messageTypeCreateIndex - case *internal.DeleteIndexMessage: - typ = messageTypeDeleteIndex - case *internal.CreateFieldMessage: - typ = messageTypeCreateField - case *internal.DeleteFieldMessage: - typ = messageTypeDeleteField - case *internal.CreateViewMessage: - typ = messageTypeCreateView - case *internal.DeleteViewMessage: - typ = messageTypeDeleteView - case *internal.ClusterStatus: - typ = messageTypeClusterStatus - case *internal.ResizeInstruction: - typ = messageTypeResizeInstruction - case *internal.ResizeInstructionComplete: - typ = messageTypeResizeInstructionComplete - case *internal.SetCoordinatorMessage: - typ = messageTypeSetCoordinator - case *internal.UpdateCoordinatorMessage: - typ = messageTypeUpdateCoordinator - case *internal.NodeStateMessage: - typ = messageTypeNodeState - case *internal.RecalculateCaches: - typ = messageTypeRecalculateCaches - case *internal.NodeEventMessage: - typ = messageTypeNodeEvent - case *internal.NodeStatus: - typ = messageTypeNodeStatus - default: - return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) - } - buf, err := proto.Marshal(m) +// MarshalInternalMessage serializes the pilosa message and adds pilosa internal +// type info which is used by the internal messaging stuff. +func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) { + typ := getMessageType(m) + buf, err := s.Marshal(m) if err != nil { - return nil, errors.Wrap(err, "marshalling") + return nil, errors.Wrap(err, "marshaling") } return append([]byte{typ}, buf...), nil } diff --git a/broadcast_test.go b/broadcast_test.go deleted file mode 100644 index 415228718..000000000 --- a/broadcast_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "reflect" - "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" -) - -// Ensure a message can be marshaled and unmarshaled. -func TestMessage_Marshal(t *testing.T) { - - testMessageMarshal(t, &internal.CreateShardMessage{ - Index: "i", - Shard: 8, - }) - - testMessageMarshal(t, &internal.DeleteIndexMessage{ - Index: "i", - }) -} - -func testMessageMarshal(t *testing.T, m proto.Message) { - marshalled, err := pilosa.MarshalMessage(m) - if err != nil { - t.Fatal(err) - } - unmarshalled, err := pilosa.UnmarshalMessage(marshalled) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(unmarshalled, m) { - t.Fatalf("unexpected message marshalling: %s", unmarshalled) - } -} diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 1c83ba4ea..10cc13dc4 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -153,6 +153,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeNodeStatus(msg, mt) return nil + case *pilosa.Node: + msg := &internal.Node{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling Node") + } + decodeNode(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -192,6 +200,8 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeNodeEventMessage(mt) case *pilosa.NodeStatus: return encodeNodeStatus(mt) + case *pilosa.Node: + return encodeNode(mt) } return nil } @@ -199,8 +209,8 @@ func encodeToProto(m pilosa.Message) proto.Message { func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { return &internal.ResizeInstruction{ JobID: m.JobID, - Node: EncodeNode(m.Node), - Coordinator: EncodeNode(m.Coordinator), + Node: encodeNode(m.Node), + Coordinator: encodeNode(m.Coordinator), Sources: encodeResizeSources(m.Sources), Schema: encodeSchema(m.Schema), ClusterStatus: encodeClusterStatus(m.ClusterStatus), @@ -217,7 +227,7 @@ func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource { func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource { return &internal.ResizeSource{ - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), Index: m.Index, Field: m.Field, View: m.View, @@ -286,13 +296,13 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { func EncodeNodes(a []*pilosa.Node) []*internal.Node { other := make([]*internal.Node, len(a)) for i := range a { - other[i] = EncodeNode(a[i]) + other[i] = encodeNode(a[i]) } return other } -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *pilosa.Node) *internal.Node { +// encodeNode converts a Node into its internal representation. +func encodeNode(n *pilosa.Node) *internal.Node { return &internal.Node{ ID: n.ID, URI: n.URI.Encode(), @@ -368,20 +378,20 @@ func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMe func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete { return &internal.ResizeInstructionComplete{ JobID: m.JobID, - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), Error: m.Error, } } func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage { return &internal.SetCoordinatorMessage{ - New: EncodeNode(m.New), + New: encodeNode(m.New), } } func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { return &internal.UpdateCoordinatorMessage{ - New: EncodeNode(m.New), + New: encodeNode(m.New), } } @@ -395,13 +405,13 @@ func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessa func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage { return &internal.NodeEventMessage{ Event: uint32(m.Event), - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), } } func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus { return &internal.NodeStatus{ - Node: EncodeNode(m.Node), + Node: encodeNode(m.Node), MaxShards: &internal.MaxShards{Standard: m.MaxShards}, Schema: encodeSchema(m.Schema), } diff --git a/gossip/gossip.go b/gossip/gossip.go index cd1d9ad0b..251851077 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -26,10 +26,9 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -44,8 +43,9 @@ type GossipMemberSet struct { broadcasts *memberlist.TransmitLimitedQueue - papi *pilosa.API - config *gossipConfig + papi *pilosa.API + serializer pilosa.Serializer + config *gossipConfig Logger pilosa.Logger @@ -150,8 +150,9 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { host := api.Node().URI.GetHost() g := &GossipMemberSet{ - papi: api, - Logger: pilosa.NopLogger, + papi: api, + serializer: proto.Serializer{}, + Logger: pilosa.NopLogger, } // options @@ -222,7 +223,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := proto.Marshal(pilosa.EncodeNode(g.papi.Node())) + buf, err := g.serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} @@ -248,14 +249,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. func (g *GossipMemberSet) LocalState(join bool) []byte { - pb := &internal.NodeStatus{ - Node: pilosa.EncodeNode(g.papi.Node()), - MaxShards: &internal.MaxShards{Standard: g.papi.MaxShards(context.Background())}, - Schema: &internal.Schema{Indexes: pilosa.EncodeIndexes(g.papi.Schema(context.Background()))}, + m := &pilosa.NodeStatus{ + Node: g.papi.Node(), + MaxShards: g.papi.MaxShards(context.Background()), + Schema: &pilosa.Schema{Indexes: g.papi.Holder().Schema()}, } // Marshal nodestate data to bytes. - buf, err := pilosa.MarshalMessage(pb) + buf, err := pilosa.MarshalInternalMessage(m, g.serializer) if err != nil { g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} @@ -278,8 +279,9 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type gossipEventReceiver struct { - ch chan memberlist.NodeEvent - papi *pilosa.API + ch chan memberlist.NodeEvent + papi *pilosa.API + serializer pilosa.Serializer logger *log.Logger } @@ -287,9 +289,10 @@ type gossipEventReceiver struct { // newGossipEventReceiver returns a new instance of GossipEventReceiver. func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { ger := &gossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - logger: logger, - papi: papi, + ch: make(chan memberlist.NodeEvent, 1), + logger: logger, + papi: papi, + serializer: proto.Serializer{}, } go ger.listen() return ger @@ -323,16 +326,16 @@ func (g *gossipEventReceiver) listen() { } // Get the node from the event.Node meta data. - var n internal.Node - if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { - panic("failed to unmarshal event node meta data") + var n pilosa.Node + if err := g.serializer.Unmarshal(e.Node.Meta, &n); err != nil { + panic("failed to unmarshal event node meta into node") } - ne := &internal.NodeEventMessage{ - Event: uint32(nodeEventType), + ne := &pilosa.NodeEvent{ + Event: nodeEventType, Node: &n, } - buf, err := pilosa.MarshalMessage(ne) + buf, err := pilosa.MarshalInternalMessage(ne, g.serializer) if err != nil { panic(err) } From 59e80f9692aa2c687cd9365e61af6802bf8fccb6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 13:35:14 -0500 Subject: [PATCH 008/166] put Serializer on API, add QueryRequest/Response to serializer --- api.go | 3 + encoding/proto/proto.go | 288 ++++++++++++++++++++++++++++++++++++++++ gossip/gossip.go | 31 ++--- http/handler.go | 13 +- 4 files changed, 310 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index cae91b20f..a9bd02f2d 100644 --- a/api.go +++ b/api.go @@ -38,6 +38,8 @@ type API struct { holder *Holder cluster *cluster server *Server + + Serializer Serializer } // APIOption is a functional option type for pilosa.API @@ -48,6 +50,7 @@ func OptAPIServer(s *Server) APIOption { a.server = s a.holder = s.holder a.cluster = s.cluster + a.Serializer = s.serializer return nil } } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 10cc13dc4..0f2574f1b 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -2,6 +2,7 @@ package proto import ( "fmt" + "sort" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" @@ -161,6 +162,23 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeNode(msg, mt) return nil + case *pilosa.QueryRequest: + msg := &internal.QueryRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryRequest") + } + decodeQueryRequest(msg, mt) + return nil + case *pilosa.QueryResponse: + msg := &internal.QueryResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryResponse") + } + decodeQueryResponse(msg, mt) + return nil + default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -202,10 +220,62 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeNodeStatus(mt) case *pilosa.Node: return encodeNode(mt) + case *pilosa.QueryRequest: + return encodeQueryRequest(mt) + case *pilosa.QueryResponse: + return encodeQueryResponse(mt) } return nil } +func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { + return &internal.QueryRequest{ + Query: m.Query, + Shards: m.Shards, + ColumnAttrs: m.ColumnAttrs, + Remote: m.Remote, + ExcludeRowAttrs: m.ExcludeRowAttrs, + ExcludeColumns: m.ExcludeColumns, + } +} + +func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(m.Results)), + ColumnAttrSets: EncodeColumnAttrSets(m.ColumnAttrSets), + } + + for i := range m.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := m.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = queryResultTypeRow + pb.Results[i].Row = EncodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = queryResultTypePairs + pb.Results[i].Pairs = EncodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = queryResultTypeValCount + pb.Results[i].ValCount = EncodeValCount(result) + case uint64: + pb.Results[i].Type = queryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = queryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = queryResultTypeNil + } + } + + if m.Err != nil { + pb.Err = m.Err.Error() + } + + return pb +} + func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { return &internal.ResizeInstruction{ JobID: m.JobID, @@ -602,3 +672,221 @@ func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { } func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {} + +func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { + m.Query = pb.Query + m.Shards = pb.Shards + m.ColumnAttrs = pb.ColumnAttrs + m.Remote = pb.Remote + m.ExcludeRowAttrs = pb.ExcludeRowAttrs + m.ExcludeColumns = pb.ExcludeColumns +} + +func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { + m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) + decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) + m.Err = errors.New(pb.Err) + m.Results = make([]interface{}, len(pb.Results)) + decodeQueryResults(pb.Results, m.Results) + +} + +func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { + for i := range pb { + decodeColumnAttrSet(pb[i], m[i]) + } +} + +func decodeColumnAttrSet(pb *internal.ColumnAttrSet, m *pilosa.ColumnAttrSet) { + m.ID = pb.ID + m.Key = pb.Key + m.Attrs = decodeAttrs(pb.Attrs) +} + +func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) { + for i := range pb { + m[i] = decodeQueryResult(pb[i]) + } +} + +// QueryResult types. +const ( + queryResultTypeNil uint32 = iota + queryResultTypeRow + queryResultTypePairs + queryResultTypeValCount + queryResultTypeUint64 + queryResultTypeBool +) + +func decodeQueryResult(pb *internal.QueryResult) interface{} { + switch pb.Type { + case queryResultTypeRow: + return decodeRow(pb.Row) + case queryResultTypePairs: + return decodePairs(pb.Pairs) + case queryResultTypeValCount: + return decodeValCount(pb.ValCount) + case queryResultTypeUint64: + return pb.N + case queryResultTypeBool: + return pb.Changed + case queryResultTypeNil: + return nil + } + panic(fmt.Sprintf("unknown type: %d", pb.Type)) +} + +// DecodeRow converts r from its internal representation. +func decodeRow(pr *internal.Row) *pilosa.Row { + if pr == nil { + return nil + } + + r := pilosa.NewRow() + r.Attrs = decodeAttrs(pr.Attrs) + for _, v := range pr.Columns { + r.SetBit(v) + } + return r +} + +func decodeAttrs(pb []*internal.Attr) map[string]interface{} { + m := make(map[string]interface{}, len(pb)) + for i := range pb { + key, value := decodeAttr(pb[i]) + m[key] = value + } + return m +} + +const ( + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 +) + +func decodeAttr(attr *internal.Attr) (key string, value interface{}) { + switch attr.Type { + case attrTypeString: + return attr.Key, attr.StringValue + case attrTypeInt: + return attr.Key, attr.IntValue + case attrTypeBool: + return attr.Key, attr.BoolValue + case attrTypeFloat: + return attr.Key, attr.FloatValue + default: + return attr.Key, nil + } +} + +func decodePairs(a []*internal.Pair) []pilosa.Pair { + other := make([]pilosa.Pair, len(a)) + for i := range a { + other[i] = decodePair(a[i]) + } + return other +} + +func decodePair(pb *internal.Pair) pilosa.Pair { + return pilosa.Pair{ + ID: pb.ID, + Key: pb.Key, + Count: pb.Count, + } +} + +func decodeValCount(pb *internal.ValCount) pilosa.ValCount { + return pilosa.ValCount{ + Val: pb.Val, + Count: pb.Count, + } +} + +func EncodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { + other := make([]*internal.ColumnAttrSet, len(a)) + for i := range a { + other[i] = EncodeColumnAttrSet(a[i]) + } + return other +} + +func EncodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { + return &internal.ColumnAttrSet{ + ID: set.ID, + Attrs: encodeAttrs(set.Attrs), + } +} + +func EncodeRow(r *pilosa.Row) *internal.Row { + if r == nil { + return nil + } + + return &internal.Row{ + Columns: r.Columns(), + Attrs: encodeAttrs(r.Attrs), + } +} + +func EncodePairs(a pilosa.Pairs) []*internal.Pair { + other := make([]*internal.Pair, len(a)) + for i := range a { + other[i] = encodePair(a[i]) + } + return other +} + +func encodePair(p pilosa.Pair) *internal.Pair { + return &internal.Pair{ + ID: p.ID, + Key: p.Key, + Count: p.Count, + } +} + +func EncodeValCount(vc pilosa.ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} + +func encodeAttrs(m map[string]interface{}) []*internal.Attr { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + a := make([]*internal.Attr, len(keys)) + for i := range keys { + a[i] = encodeAttr(keys[i], m[keys[i]]) + } + return a +} + +// encodeAttr converts a key/value pair into an Attr internal representation. +func encodeAttr(key string, value interface{}) *internal.Attr { + pb := &internal.Attr{Key: key} + switch value := value.(type) { + case string: + pb.Type = attrTypeString + pb.StringValue = value + case float64: + pb.Type = attrTypeFloat + pb.FloatValue = value + case uint64: + pb.Type = attrTypeInt + pb.IntValue = int64(value) + case int64: + pb.Type = attrTypeInt + pb.IntValue = value + case bool: + pb.Type = attrTypeBool + pb.BoolValue = value + } + return pb +} diff --git a/gossip/gossip.go b/gossip/gossip.go index 251851077..28b3ac690 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -28,7 +28,6 @@ import ( "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -43,9 +42,8 @@ type GossipMemberSet struct { broadcasts *memberlist.TransmitLimitedQueue - papi *pilosa.API - serializer pilosa.Serializer - config *gossipConfig + papi *pilosa.API + config *gossipConfig Logger pilosa.Logger @@ -150,9 +148,8 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { host := api.Node().URI.GetHost() g := &GossipMemberSet{ - papi: api, - serializer: proto.Serializer{}, - Logger: pilosa.NopLogger, + papi: api, + Logger: pilosa.NopLogger, } // options @@ -223,7 +220,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := g.serializer.Marshal(g.papi.Node()) + buf, err := g.papi.Serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} @@ -256,7 +253,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte { } // Marshal nodestate data to bytes. - buf, err := pilosa.MarshalInternalMessage(m, g.serializer) + buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer) if err != nil { g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} @@ -279,9 +276,8 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type gossipEventReceiver struct { - ch chan memberlist.NodeEvent - papi *pilosa.API - serializer pilosa.Serializer + ch chan memberlist.NodeEvent + papi *pilosa.API logger *log.Logger } @@ -289,10 +285,9 @@ type gossipEventReceiver struct { // newGossipEventReceiver returns a new instance of GossipEventReceiver. func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { ger := &gossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - logger: logger, - papi: papi, - serializer: proto.Serializer{}, + ch: make(chan memberlist.NodeEvent, 1), + logger: logger, + papi: papi, } go ger.listen() return ger @@ -327,7 +322,7 @@ func (g *gossipEventReceiver) listen() { // Get the node from the event.Node meta data. var n pilosa.Node - if err := g.serializer.Unmarshal(e.Node.Meta, &n); err != nil { + if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil { panic("failed to unmarshal event node meta into node") } @@ -335,7 +330,7 @@ func (g *gossipEventReceiver) listen() { Event: nodeEventType, Node: &n, } - buf, err := pilosa.MarshalInternalMessage(ne, g.serializer) + buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer) if err != nil { panic(err) } diff --git a/http/handler.go b/http/handler.go index 9195f05e9..84832b1d4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -815,13 +815,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques return nil, errors.Wrap(err, "reading") } - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") + qreq := &pilosa.QueryRequest{} + err = h.API.Serializer.Unmarshal(body, qreq) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling query request") } - - return decodeQueryRequest(&req), nil + return qreq, nil } // readURLQueryRequest parses query parameters from URL parameters from r. @@ -860,7 +859,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res // writeProtobufQueryResponse writes the response from the executor to w as protobuf. func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { + if buf, err := h.API.Serializer.Marshal(resp); err != nil { return errors.Wrap(err, "marshalling") } else if _, err := w.Write(buf); err != nil { return errors.Wrap(err, "writing") From 36926bdc47bd9b965f858d125cfc1bb8cff6ed70 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 5 Jul 2018 14:52:30 -0500 Subject: [PATCH 009/166] WIP syntax and API updates --- docs/administration.md | 40 +++--- docs/api-reference.md | 71 ++++------ docs/data-model.md | 36 +++--- docs/examples.md | 45 ++++--- docs/getting-started.md | 30 ++--- docs/glossary.md | 22 ++-- docs/query-language.md | 277 ++++++++++++++++++---------------------- docs/webui.md | 10 +- 8 files changed, 244 insertions(+), 287 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index accbc46de..6764823e9 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -24,19 +24,19 @@ Pilosa holds all row/column bitmap data in main memory. While this data is compr #### CPUs -Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [slice](../data-model/#slice), so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload. +Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [shard](../data-model/#shard), so a single query will only use a number of cores up to the number of shards stored on that host. Multiple queries can still take advantage of multiple cores as well, so tuning in this area is dependent upon the expected workload. #### Disk -Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs—especially if you have a write heavy application. +Even though the main dataset is in memory Pilosa backs up to disk frequently. We recommend SSDs—especially if you have a write-heavy application. #### Network -Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there already should be a system of record, or ability to rebuild a cluster quickly from backups. +Pilosa is designed to be a distributed application, with data replication replicated across the cluster. As such, every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all nodes exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions is not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there should already be a system of record, or ability to rebuild a cluster quickly from backups. #### Overview -While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. +While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines, as the internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. ### Open File Limits @@ -56,23 +56,23 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -##### Importing Field Values +##### Importing Integer Values -If you are using [BSI Range-Encoding](../data-model/#bsi-range-encoding) field values, you can import field values for a single frame and single field using `--field`. The CSV file should be in the format `Column,Value`. +If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`. ``` -pilosa import -i project -f stargazer --field star_count project-stargazer-counts.csv +pilosa import -i project -f stargazer-counts project-stargazer-counts.csv ```
-

Note that you must first create a frame and a field. View Create Frame for more details.

+

Note that you must first create a field. View Create Field for more details. The `-e` flag can create the necessary schema when using a field of type "set".

#### Exporting -Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the frame. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column. +Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a field. The data will be in csv format `Row,Column` and sorted by column. ```request -curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0" \ +curl "http://localhost:10101/export?index=repository&field=stargazer&slice=0" \ --header "Accept: text/csv" ``` ```response @@ -132,8 +132,8 @@ Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topo **Application changes**: 1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported. -2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed. -3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release. +2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-field, as index-level time-quantums have been removed. +3. Inverse fields have been deprecated, removed from docs, and will be unsupported in the next release. ### Resizing the Cluster @@ -211,7 +211,7 @@ curl localhost:10101/cluster/resize/set-coordinator \ ### Backup/restore -Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. +Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node. @@ -230,11 +230,11 @@ Note: This will only work when the replication factor is >= 2 - To accomplish this you will first need: - List of all indexes on your cluster - - List of all frames in your indexes + - List of all fields in your indexes - Max slice per index, listed in the `/slices/max` endpoint - With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each slice - Using the list of slices owned by this node you will then need to manually: - - setup a directory structure similar to the other nodes with a path for each Index/Frame + - setup a directory structure similar to the other nodes with a path for each Index/Field - copy each owned slice for an existing node to this new node - Modify the cluster config file to replace the previous node address with the new node address. - Restart the cluster @@ -249,10 +249,10 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **Cluster:** List of nodes in the cluster. - **NumNodes:** Number of nodes in the cluster. - **NumCPU:** Number of cores per node -- **BSIEnabled:** Bit Slice Index Frames in use. -- **TimeQuantumEnabled:** Time Quantum Frames in use. +- **BSIEnabled:** Bit Sliced Index Fields in use. +- **TimeQuantumEnabled:** Time Quantum Fields in use. - **NumIndexes:** Number of indexes in the Cluster. -- **NumFrames:** Number of frames in the Cluster. +- **NumFields:** Number of fields in the Cluster. - **NumSlices:** Number of slices in the Cluster. - **NumViews:** Number of views in the Cluster. - **OpenFiles:** Open file handle count. @@ -274,7 +274,7 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - NodeID - Index -- Frame +- Field - View - Slice @@ -282,7 +282,7 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: We currently track the following events - **Index:** The creation of a new index. -- **Frame:** The creation of a new frame. +- **Field:** The creation of a new field. - **MaxSlice:** The creation of a new Slice. - **SetBit:** Count of set bits. - **ClearBit:** Count of cleared bits. diff --git a/docs/api-reference.md b/docs/api-reference.md index f94cc5009..41e0288ce 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -17,7 +17,7 @@ Returns the schema of all indexes in JSON. curl -XGET localhost:10101/index ``` ``` response -{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} +{"indexes":[{"name":"user","fields":[{"name":"collab"}]}]} ``` ### List index schema @@ -30,7 +30,7 @@ Returns the schema of the specified index in JSON. curl -XGET localhost:10101/index/user ``` ``` response -{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} +{"name":"user", "fields":[{"name":"collab"}]} ``` ### Create index @@ -43,7 +43,7 @@ Creates an index with the given name. curl -XPOST localhost:10101/index/user ``` ``` response -{} +{"success":true} ``` ### Remove index @@ -56,7 +56,7 @@ Removes the given index. curl -XDELETE localhost:10101/index/user ``` ``` response -{} +{"success":true} ``` ### Query index @@ -68,10 +68,10 @@ Sends a [query](../query-language/) to the Pilosa server with the given index. T ``` request curl localhost:10101/index/user/query \ -X POST \ - -d 'Bitmap(frame="language", row=5)' + -d 'Row(language=5)' ``` ``` response -{"results":[{"attrs":{},"bits":[100]}]} +{"results":[{"attrs":{},"columns":[100]}]} ``` In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`. @@ -83,87 +83,66 @@ The query is executed for all [slices](../data-model/#slice) by default. To use ``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ - -d 'Bitmap(frame="language", row=5)' + -d 'Row(language=5)' ``` ``` response { - "results":[{"attrs":{},"bits":[100]}], + "results":[{"attrs":{},"columns":[100]}], "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] } ``` -By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`. +By default, all bits and attributes (*for `Row` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`. -### Create frame +### Create field -`POST /index//frame/` +`POST /index//field/` -Creates a frame in the given index with the given name. +Creates a field in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: -* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. -* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field. +* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: * `name` (string): Field name. -* `type` (string): Field type, currently only "int" is supported. +* `type` (string): Field type, "set", "int" or "time". * `min` (int): Minimum value allowed for this field. * `max` (int): Maximum value allowed for this field. Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. ``` request -curl localhost:10101/index/user/frame/language -X POST +curl localhost:10101/index/user/field/language -X POST ``` ``` response -{} +{"success":true} ``` ``` request -curl localhost:10101/index/repository/frame/stats \ +curl localhost:10101/index/repository/field/stats \ -X POST \ -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` ``` response -{} +{"success":true} ``` -### Remove frame +### Remove field -`DELETE /index//frame/` +`DELETE /index//field/` -Removes the given frame. +Removes the given field. ``` request -curl -XDELETE localhost:10101/index/user/frame/language +curl -XDELETE localhost:10101/index/user/field/language ``` ``` response -{} -``` - -### Create Field - -`POST /index//frame//field/` - -Creates a new field to store integer values in the given frame. - -The request payload is JSON, and it must contain the fields `type`, `min`, `max`. - -* `type` (string): Field type, currently only "int" is supported. -* `min` (int): Minimum value allowed for this field. -* `max` (int): Maximum value allowed for this field. - -``` request -curl localhost:10101/index/repository/frame/stats/field/pullrequests \ - -X POST \ - -d '{"type": "int", "min": 0, "max": 1000000}' -``` -``` response -{} +{"success":true} ``` ### Get version @@ -191,7 +170,7 @@ in a multi-node cluster, the cache is only recalculated on the node that receives the request. ``` request -curl -XGET localhost:10101/recalculate-caches +curl -XPOST localhost:10101/recalculate-caches ``` Response: `204 No Content` diff --git a/docs/data-model.md b/docs/data-model.md index ad6a9ea9d..bdcec4861 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -6,7 +6,7 @@ nav = [ "Index", "Column", "Row", - "Frame", + "Field", "Time Quantum", "Attribute", "Slice", @@ -22,7 +22,7 @@ The central component of Pilosa's data model is a boolean matrix. Each cell in t Rows and columns can represent anything (they could even represent the same set of things - a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix. -Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of bits set in each row. +Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row. Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 263 on a single-node cluster, for example, will not work well due to memory limitations. @@ -43,12 +43,14 @@ Row ids are sequential increasing integers namespaced to each Field within an In ### Field -Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, a field with BSI values could represent all possible integer values of a relational field. +Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, an integer field could represent all possible integer values of a relational field. #### Relational Analogy The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here. +TODO diagram showing a few rows of a relational table and corresponding pilosa index + Entities: Relational | Pilosa @@ -64,9 +66,9 @@ Simple queries: Relational | Pilosa ---------------------------------------------|------------------------------------ - `select ID from People where Name = 'Bob'` | `Bitmap(frame=Name, row=[Bob])` - `select ID from People where Age > 30` | `Range(frame=Default, Age > 30)` - `select ID from People where Member = true` | `Bitmap(frame=Member, row=[true])` + `select ID from People where Name = 'Bob'` | `Row(Name="Bob")` + `select ID from People where Age > 30` | `Range(Age > 30)` + `select ID from People where Member = true` | `Row(Member=0)` # TODO this is unfortunate In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: @@ -80,7 +82,7 @@ where c.Make = 'Ford' can be accomplished with a Pilosa query like this (note that [Sum](../query-language/#sum) returns a json object containing both the sum and count, from which the average is easily computed): ```pql -Sum(Bitmap(frame="Car-Make", row=[Ford]), frame=Default, field=Age) +Sum(Row(Car-Make="Ford"), field=Age) ``` This is one major component of Pilosa's ability to combine relationships from multiple data stores. @@ -125,11 +127,11 @@ The standard View contains the same Row/Column format as the input data. #### Time Quantums -If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the diagram below: +If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below: ``` -SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00") -SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") +Set(3, A=8, 2017-05-18T00:00) +Set(3, A=8, 2017-05-19T00:00) ``` ![time quantum field diagram](/img/docs/field-time-quantum.svg) @@ -141,15 +143,15 @@ Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-b Internally Pilosa stores each BSI (TODO!!!!!) `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. -For example, the following `SetFieldValue()` queries will result in the data described in the diagram below: +For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below: ``` -SetFieldValue(col=1, frame="A", field0=1) -SetFieldValue(col=2, frame="A", field0=2) -SetFieldValue(col=3, frame="A", field0=3) -SetFieldValue(col=4, frame="A", field0=7) -SetFieldValue(col=2, frame="A", field1=1) -SetFieldValue(col=3, frame="A", field1=6) +Set(1, A=1) +Set(2, A=2) +Set(3, A=3) +Set(4, A=7) +Set(2, B=1) +Set(3, B=6) ``` ![BSI field diagram](/img/docs/field-bsi.svg) diff --git a/docs/examples.md b/docs/examples.md index 6c4f70cc1..1660e8592 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -33,9 +33,9 @@ The NYC taxi data is comprised of a number of csv files listed here: http://www. * Dropoff time: timestamp * Pickup time: timestamp -We import these fields, creating one or more Pilosa frames from each of them: +We import these fields, creating one or more Pilosa fields from each of them: -frame |mapping +field |mapping ------------|--------------------- cab_type |direct map of enum int → row ID dist_miles |round(dist) → row ID @@ -52,24 +52,24 @@ pickup_month |month(timestamp) → row ID pickup_day |day(timestamp) → row ID pickup_time |time of day mapped to one of 48 half-hour buckets → row ID -We also created two extra frames that represent the duration and average speed of each ride: +We also created two extra fields that represent the duration and average speed of each ride: -frame |mapping +field |mapping --------------------|------------- duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID #### Mapping -Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. +Each column that we want to use must be mapped to a combination of fields and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. -##### 0 columns → 1 frame +##### 0 columns → 1 field -**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. +**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this field. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this field are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type field is constant. -##### 1 column → 1 frame +##### 1 column → 1 field -The following three frames are mapped in a simple direct way from single columns of the original data. +The following three fields are mapped in a simple direct way from single columns of the original data. **dist_miles:** each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. @@ -84,7 +84,7 @@ lfm := pdk.LinearFloatMapper{ `Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. -This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). +This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Frame`). TODO update so this makes sense ```go pdk.BitMapper{ Frame: "dist_miles", @@ -129,27 +129,27 @@ Here, we define a list of Mappers, each including a name, which we use to refer **passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. -##### 1 column → multiple frames +##### 1 column → multiple fields When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. -We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". +We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day". -We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". +We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day". -We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. +We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total fields for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. -##### Multiple columns → 1 frame +##### Multiple columns → 1 field The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. -We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. +We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two fields for two locations: pickup_grid_id, drop_grid_id. Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. ##### Complex mappings -We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: +We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the field `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the field `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: ```go durm := pdk.CustomMapper{ Func: func(fields ...interface{}) interface{} { @@ -172,7 +172,7 @@ Now we can run some example queries. Count per cab type can be retrieved, sorted, with a single PQL call. ```request -TopN(frame=cab_type) +TopN(cab_type) ``` ```response {"results":[[{"id":1,"count":1992943},{"id":0,"count":7057}]]} @@ -181,7 +181,7 @@ TopN(frame=cab_type) High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. ```request -TopN(frame=pickup_grid_id) +TopN(pickup_grid_id) ``` ```response {"results":[[{"id":5060,"count":40620},{"id":4861,"count":38145},{"id":4962,"count":35268},...]]} @@ -193,7 +193,7 @@ Average of `total_amount` per `passenger_count` can be computed with some postpr queries = '' pcounts = range(10) for i in pcounts: - queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i + queries += "TopN(Row(passenger_count=%d), total_amount_dollars)" % i resp = requests.post(qurl, data=queries) average_amounts = [] @@ -209,6 +209,8 @@ Note that the BSI-powered @@ -326,3 +328,6 @@ python benchmarks.py -id 6223 As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster + + +--> diff --git a/docs/getting-started.md b/docs/getting-started.md index 7f5759844..6f935f735 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -47,7 +47,7 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term #### Create the Schema Note: -The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: +If at any time you want to verify the data structure, you can request the schema as follows: ``` request curl localhost:10101/schema @@ -61,7 +61,7 @@ Before we can import data or run queries, we need to create our indexes and the curl localhost:10101/index/repository -X POST ``` ``` response -{} +{"success":true} ``` Let's create the `stargazer` field which has user IDs of stargazers as its rows: @@ -71,10 +71,10 @@ curl localhost:10101/index/repository/field/stargazer \ -d '{"options": {"type": "time", "timeQuantum": "YMD"}}' ``` ``` response -{} +{"success":true} ``` -Since our data contains time stamps for the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. +Since our data contains time stamps whcih represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` field, which will contain IDs for programming languages: ``` request @@ -82,7 +82,7 @@ curl localhost:10101/index/repository/field/language \ -X POST ``` ``` response -{} +{"success":true} ``` The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options. @@ -119,7 +119,7 @@ Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Bitmap(field="stargazer", row=14)' + -d 'Row(stargazer=14)' ``` ``` response { @@ -136,7 +136,7 @@ What are the top 5 languages in the sample data: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'TopN(field="language", n=5)' + -d 'TopN(language, n=5)' ``` ``` response { @@ -157,8 +157,8 @@ Which repositories were starred by user 14 and 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(field="stargazer", row=14), - Bitmap(field="stargazer", row=19) + Row(stargazer=14), + Row(stargazer=19) )' ``` ``` response @@ -177,8 +177,8 @@ Which repositories were starred by user 14 or 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Union( - Bitmap(field="stargazer", row=14), - Bitmap(field="stargazer", row=19) + Row(stargazer=14), + Row(stargazer=19) )' ``` ``` response @@ -197,9 +197,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(field="stargazer", row=14), - Bitmap(field="stargazer", row=19), - Bitmap(field="language", row=1) + Row(stargazer=14), + Row(stargazer=19), + Row(language=1) )' ``` ``` response @@ -217,7 +217,7 @@ Set user 99999 as a stargazer for repository 77777: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(field="stargazer", col=77777, row=99999)' + -d 'Set(77777, stargazer=99999)' ``` ``` response {"results":[true]} diff --git a/docs/glossary.md b/docs/glossary.md index 78ca7c1fe..eb5dd780e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -12,19 +12,17 @@ nav = [] [Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column). -[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. +[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). [BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries. -Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. +Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. [Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index). -[Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries. - Fragment: A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index). -[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. +[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of three types: set, [int](#bsi), and time. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). [Frame](../data-model/#field): Prior to Pilosa 1.0, fields were known as frames. @@ -34,11 +32,11 @@ nav = [] [Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). +[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in an [integer](#bsi) [field](#field). -MaxShard: The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. +MaxShard: The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. MaxShard is zero-indexed, so if an index contains six shards, its MaxShard will be 5. -[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). +[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in an [integer](#bsi) [field](#field). Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). @@ -64,14 +62,12 @@ nav = [] ShardWidth: This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. -[Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field). +[Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field). -[Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). - -[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for time [Range](#range) queries. +[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for [Range](#range) queries on time [fields](#field). [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). -[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [field](#field). +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of rows, sorted by the count of [columns](#column) set in the [row](#row), within a specified [field](#field). [View](../data-model/#view): Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. diff --git a/docs/query-language.md b/docs/query-language.md index a598bf351..62c912ba4 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -1,4 +1,4 @@ -+++ +v+++ title = "Query Language" weight = 6 nav = [ @@ -29,13 +29,13 @@ There will be one item in the `results` array for each PQL query in the request. ##### Examples -Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data. +Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index and fields, and to populate them with some data. -The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would: +The examples just show the PQL quer(ies) needed - to run the query `Set(10, stargazer=1)` against a server using curl, you would: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", col=10, row=1)' + -d 'Set(10, stargazer=1)' ``` ``` response {"results":[true]} @@ -43,28 +43,27 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types -* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. -* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") +* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04) * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. -* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` +* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations -#### SetBit +#### Set **Spec:** ``` -SetBit(, , , - [timestamp=TIMESTAMP]) +Set(, field=, [TIMESTAMP]) ``` **Description:** -`SetBit` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column. +`Set` assigns a value of 1 to a bit in the binary matrix, thus associating the given row (the `` value) in the given field with the given column. **Result Type:** boolean @@ -77,17 +76,17 @@ A return value of `false` indicates that the bit was already set to 1 and nothin Set the bit at row 1, column 10: ```request -SetBit(frame="stargazer", col=10, row=1) +Set(10, stargazer=1) ``` ```response {"results":[true]} ``` -This sets a bit in the stargazer frame, representing that the user with id=1 has starred the repository with id=10. +This sets a bit in the stargazer field, representing that the user with id=1 has starred the repository with id=10. -SetBit also supports providing a timestamp. To write the date that a user starred a repository: +Set also supports providing a timestamp. To write the date that a user starred a repository: ```request -SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") +Set(10, stargazer=1, 2016-01-01T00:00) ``` ```response {"results":[true]} @@ -95,24 +94,32 @@ SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") Set multiple bits in a single request: ```request -SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2) +Set(1, stargazer=10) Set(2, stargazer=10) Set(1, stargazer=20) Set(2, stargazer=30) ``` ```response {"results":[false,true,true,true]} ``` +Set the field "pullrequests" to integer value 2 at column 10: +```request +Set(10, pullrequests=2) +``` +```response +{"results":[true]} +``` + #### SetRowAttrs **Spec:** ``` -SetRowAttrs(, , +SetRowAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` **Description:** -`SetRowAttrs` associates arbitrary key/value pairs with a row in a frame. Setting a value of `null`, without quotes, deletes an attribute. +`SetRowAttrs` associates arbitrary key/value pairs with a row in a field. Setting a value of `null`, without quotes, deletes an attribute. **Result Type:** null @@ -122,17 +129,17 @@ SetRowAttrs queries always return `null` upon success. Set attributes `username` and `active` on row 10: ```request -SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true) +SetRowAttrs(stargazer, 10, username="mrpi", active=true) ``` ```response {"results":[null]} ``` -Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", row=10)`. +Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Row](../query-language/#row) query like so `Row(stargazer=10)`. Delete attribute `username` on row 10: ```request -SetRowAttrs(frame="stargazer", row=10, username=null) +SetRowAttrs(stargazer, 10, username=null) ``` ```response {"results":[null]} @@ -143,7 +150,7 @@ SetRowAttrs(frame="stargazer", row=10, username=null) **Spec:** ``` -SetColumnAttrs(, , +SetColumnAttrs(, , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -154,13 +161,13 @@ SetColumnAttrs(, , **Result Type:** null -SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. To avoid confusion, `frame` cannot be used as an attribute name. +SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. **Examples:** Set attributes `stars`, `url`, and `active` on column 10: ```request -SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true) +SetColumnAttrs(10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` ```response {"results":[null]} @@ -170,13 +177,13 @@ Set url value and active status for project 10. These are arbitrary key/value pa ColumnAttrs can be requested by adding the URL parameter `columnAttrs=true` to a query. For example: ```request -curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Bitmap(frame="stargazer", row=1)Bitmap(frame="stargazer", row=2)' +curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Row(stargazer=1) Row(stargazer=2)' ``` ```response { "results":[ - {"attrs":{},"bits":[10,20]}, - {"attrs":{},"bits":[10,30]} + {"attrs":{},"cols":[10,20]}, + {"attrs":{},"cols":[10,30]} ], "columnAttrs":[ {"id":10,"attrs":{"active":true,"stars":123,"url":"http://projects.pilosa.com/10"}}, @@ -189,7 +196,7 @@ In this example, ColumnAttrs have been set on columns 10 and 20, but not column Delete the `url` attribute on column 10: ```request -SetColumnAttrs(col=10, url=null) +SetColumnAttrs(10, url=null) ``` ```response {"results":[null]} @@ -200,14 +207,14 @@ SetColumnAttrs(col=10, url=null) **Spec:** ``` -ClearBit(, , ) +Clear(, field=) ``` **Description:** -`ClearBit` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column. +`Clear` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column. -Note that clearing bits from time views is not supported. +Note that clearing a column on a time field will remove all data for that column. **Result Type:** boolean @@ -217,9 +224,9 @@ A return value of `false` indicates that the bit was already set to 0 and nothin **Examples:** -Clear the bit at row 1 and column 10 in the stargazer frame: +Clear the bit at row 1 and column 10 in the stargazer field: ```request -ClearBit(frame="stargazer", col=10, row=1) +Clear(10, stargazer=1) ``` ```response {"results":[true]} @@ -227,79 +234,48 @@ ClearBit(frame="stargazer", col=10, row=1) This represents removing the relationship between the user with id=1 and the repository with id=10. -#### SetFieldValue - -**Spec:** - -``` -SetFieldValue(, , ) -``` - -**Description:** - -`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`. - -**Result Type:** null - -SetFieldValue returns `null` upon success. - -**Examples:** - -Set the field value `pullrequest` to the value 2, on column 10 in frame `stats`: -```request -SetFieldValue(col=10, frame="stats", pullrequests=2) -``` -```response -{"results":[null]} -``` - -This represents setting the number of pull requests of repository 10 to 2. - -This example assumes the existence of the frame `stats` and the field `pullrequests`. See [frame creation](../api-reference/#create-frame) and [field creation](../api-reference/#create-field) for more information. - - ### Read Operations -#### Bitmap +#### Row **Spec:** ``` -Bitmap(, ( | =UINT)) +Row(field=) ``` **Description:** -`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column. +`Row` retrieves the indices of all the columns in a row. It also retrieves any attributes set on that row. -**Result Type:** object with attrs and bits. +**Result Type:** object with attrs and columns. -e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` +e.g. `{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}` **Examples:** -Query all columns with a bit set in row 1 of the frame `stargazer` (repositories that are starred by user 1): +Query all columns with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1): ```request -Bitmap(frame="stargazer", row=1) +Row(stargazer=1) ``` ```response -{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]} +{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]} ``` * attrs are the attributes for user 1 -* bits are the repositories which user 1 has starred. +* columns are the repositories which user 1 has starred. #### Union **Spec:** ``` -Union([BITMAP_CALL ...]) +Union([ROW_CALL ...]) ``` **Description:** -Union performs a logical OR on the results of all `BITMAP_CALL` queries passed to it. +Union performs a logical OR on the results of all `ROW_CALL` queries passed to it. **Result Type:** object with attrs and bits @@ -309,28 +285,27 @@ attrs will always be empty Query columns with a bit set in either of two rows (repositories that are starred by either of two users): ```request -Union(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2)) +Union(Row(stargazer=1), Row(stargazer=2)) ``` ```response -{"attrs":{},"bits":[10, 20, 30]} +{"attrs":{},"columns":[10, 20, 30]} ``` -* bits are repositories that were starred by user 1 OR user 2 +* columns are repositories that were starred by user 1 OR user 2 #### Intersect - **Spec:** ``` -Intersect(, [BITMAP_CALL ...]) +Intersect(, [ROW_CALL ...]) ``` **Description:** -Intersect performs a logical AND on the results of all `BITMAP_CALL` queries passed to it. +Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it. -**Result Type:** object with attrs and bits +**Result Type:** object with attrs and columns attrs will always be empty @@ -339,27 +314,27 @@ attrs will always be empty Query columns with a bit set in both of two rows (repositories that are starred by both of two users): ```request -Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) +Intersect(Row(stargazer=1), Row(stargazer=2)) ``` ```response -{"attrs":{},"bits":[10]} +{"attrs":{},"columns":[10]} ``` -* bits are repositories that were starred by user 1 AND user 2 +* columns are repositories that were starred by user 1 AND user 2 #### Difference **Spec:** ``` -Difference(, [BITMAP_CALL ...]) +Difference(, [ROW_CALL ...]) ``` **Description:** -Difference returns all of the bits from the first `BITMAP_CALL` argument passed to it, without the bits from each subsequent `BITMAP_CALL`. +Difference returns all of the bits from the first `ROW_CALL` argument passed to it, without the bits from each subsequent `ROW_CALL`. -**Result Type:** object with attrs and bits +**Result Type:** object with attrs and columns attrs will always be empty @@ -367,37 +342,37 @@ attrs will always be empty Query columns with a bit set in one row and not another (repositories that are starred by one user and not another): ```request -Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2)) +Difference(Row(stargazer=1), Row(stargazer=2)) ``` ```response -{"results":[{"attrs":{},"bits":[20]}]} +{"results":[{"attrs":{},"columns":[20]}]} ``` -* bits are repositories that were starred by user 1 BUT NOT user 2 +* columns are repositories that were starred by user 1 BUT NOT user 2 Query for the opposite difference: ```request -Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1)) +Difference(Row(stargazer=2), Row(stargazer=1)) ``` ```response -{"attrs":{},"bits":[30]} +{"attrs":{},"columns":[30]} ``` -* Bits are repositories that were starred by user 2 BUT NOT user 1 +* columnss are repositories that were starred by user 2 BUT NOT user 1 #### Xor **Spec:** ``` -Xor(, [BITMAP_CALL ...]) +Xor(, [ROW_CALL ...]) ``` **Description:** -Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it. +Xor performs a logical XOR on the results of each `ROW_CALL` query passed to it. -**Result Type:** object with attrs and bits +**Result Type:** object with attrs and columns attrs will always be empty @@ -406,24 +381,24 @@ attrs will always be empty Query columns with a bit set in exactly one of two rows (repositories that are starred by only one of two users): ```request -Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) +Xor(Row(stargazer=2), Row(stargazer=1)) ``` ```response -{"results":[{"attrs":{},"bits":[10,20,30]}]} +{"results":[{"attrs":{},"columns":[10,20,30]}]} ``` -* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) +* columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) #### Count **Spec:** ``` -Count() +Count() ``` **Description:** -Returns the number of set bits in the `BITMAP_CALL` passed in. +Returns the number of set bits in the `ROW_CALL` passed in. **Result Type:** int @@ -431,7 +406,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in. Query the number of bits set in a row (the number of repositories a user has starred): ```request -Count(Bitmap(frame="stargazer", row=1)) +Count(Row(stargazer=1)) ``` ```response {"results":[1]} @@ -444,34 +419,34 @@ Count(Bitmap(frame="stargazer", row=1)) **Spec:** ``` -TopN([BITMAP_CALL], , [n=UINT], - [, ]) +TopN([ROW_CALL], , [n=UINT], + [attrName=, attrValues=<[]ATTR_VALUE>]) ``` **Description:** -Return the id and count of the top `n` bitmaps (by count of bits) in the frame. -The `field` and `filters` arguments work together to only return Bitmaps which -have the attribute specified by `field` with one of the values specified in -`filters`. +Return the id and count of the top `n` bitmaps (by count of bits) in the field. +The `attrName` and `attrValues` arguments work together to only return rows which +have the attribute specified by `attrName` with one of the values specified in +`attrValues`. **Result Type:** array of key/count objects **Caveats:** -* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order. -* Frames with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of frame will return bitmaps sorted in order of most recently set bit. -* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. -* Once full, the cache will truncate the set of bitmaps according to the frame option CacheSize. Bitmaps that straddle the limit and have the same count will be truncated in no particular order. +* Performing a TopN() query on a field with cache type ranked will return the top bitmaps sorted by count in descending order. +* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return bitmaps sorted in order of most recently set bit. +* The field's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. +* Once full, the cache will truncate the set of bitmaps according to the field option CacheSize. Bitmaps that straddle the limit and have the same count will be truncated in no particular order. * The TopN query's attribute filter is applied to the existing sorted cache of bitmaps. Bitmaps that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored. -See [frame creation](../api-reference/#create-frame) for more information about the cache. +See [field creation](../api-reference/#create-field) for more information about the cache. **Examples:** Basic TopN query: ```request -TopN(frame="stargazer") +TopN(stargazer) ``` ```response {"results":[[{"id":1240,"count":102},{"id":4734,"count":100},{"id":12709,"count":93},...]]} @@ -479,11 +454,11 @@ TopN(frame="stargazer") * `id` is a row ID (user ID) * `count` is a count of columns (repositories) -* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer frame. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository. +* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer field. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository. Limit the number of results: ```request -TopN(frame="stargazer", n=2) +TopN(stargazer, n=2) ``` ```response {"results":[[{"id":1240,"count":102},{"id":4734,"count":100}]]} @@ -493,17 +468,17 @@ TopN(frame="stargazer", n=2) Filter based on an existing Bitmap: ```request -TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2) +TopN(Row(language=1), stargazer, n=2) ``` ```response {"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]} ``` -* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language frame (repositories that they've starred which are written in language 1). +* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language field (repositories that they've starred which are written in language 1). Filter based on attributes: ```request -TopN(frame="stargazer", n=2, field=active, filters=[true]) +TopN(stargazer, n=2, attrName=active, attrValues=[true]) ``` ```response {"results":[[{"id":10,"count":1},{"id":13,"count":1}]]} @@ -516,31 +491,30 @@ TopN(frame="stargazer", n=2, field=active, filters=[true]) **Spec:** ``` -Range(, , - , ) +Range(field=, , ) ``` **Description:** -Similar to `Bitmap`, but only returns bits which were set with timestamps -between the given `start` and `end` timestamps. +Similar to `Row`, but only returns bits which were set with timestamps +between the given `start` (first) and `end` (second) timestamps. **Result Type:** object with attrs and bits **Examples:** -Query all columns with a bit set in row 1 of a frame (repositories that a user has starred), within a date range: +Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: ```request -Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00") +Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00) ``` ```response -{{"attrs":{},"bits":[10]} +{{"attrs":{},"columns":[10]} ``` This example assumes timestamps have been set on some bits. -* bits are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. +* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. #### Range (BSI) @@ -548,16 +522,15 @@ This example assumes timestamps have been set on some bits. **Spec:** ``` -Range(, ) +Range([ ] ) ``` **Description:** -The `Range` query is overloaded to work on `field` values as well as `timestamp` values. +The `Range` query is overloaded to work on `integer` values as well as `timestamp` values. Returns bits that are true for the comparison operator. -**Result Type:** object with attrs and bits - +**Result Type:** object with attrs and columns **Examples:** @@ -565,13 +538,13 @@ In our source data, commitactivity was counted over the last year. The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits): ```request -Range(frame="stats", commitactivity > 100) +Range(commitactivity > 100) ``` ```response -{{"attrs":{},"bits":[10]} +{{"attrs":{},"columns":[10]} ``` -* bits are repositories which had at least 100 commits in the last year. +* columns are repositories which had at least 100 commits in the last year. BSI range queries support the following operators: @@ -583,35 +556,37 @@ BSI range queries support the following operators: `>=` | greater-than-or-equal-to, GTE | integer `==` | equal-to, EQ | integer `!=` | not-equal-to, NEQ | integer or `null` - `><` | between, BETWEEN | [integer, integer] -The `BETWEEN` form specifies an interval with both bounds, using the `><` operator, and a two-element list containing the lower and upper bounds of the interval: +`<`, and `<=` can be chained together to represent a bounded interval. For example: -```pql -Range(frame="stats", commitactivity >< [100, 200]) +```request +Range(50 < commitactivity < 150) +``` +```response +{{"attrs":{},"columns":[10]} ``` -This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. +As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported. #### Min **Spec:** ``` -Min([BITMAP_CALL], , ) +Min([ROW_CALL], field=) ``` **Description:** -Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. +Returns the minimum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered. **Result Type:** object with the min and count of columns containing the min value. **Examples:** -Query the minimum value of all fields in a frame (minimum size of all repositories): +Query the minimum value of a field (minimum size of all repositories): ```request -Min(frame="stats", field="diskusage") +Min(field="diskusage") ``` ```response {"value":4,"count":2} @@ -624,20 +599,20 @@ Min(frame="stats", field="diskusage") **Spec:** ``` -Max([BITMAP_CALL], , ) +Max([ROW_CALL], field=) ``` **Description:** -Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. +Returns the maximum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered. **Result Type:** object with the max and count of columns containing the max value. **Examples:** -Query the maximum value of all fields in a frame (maximum size of all repositories): +Query the maximum value of a field (maximum size of all repositories): ```request -Max(frame="stats", field="diskusage") +Max(field="diskusage") ``` ```response {"value":88,"count":13} @@ -650,12 +625,12 @@ Max(frame="stats", field="diskusage") **Spec:** ``` -Sum([BITMAP_CALL], , ) +Sum([ROW_CALL], field=) ``` **Description:** -Returns the count and computed sum of all BSI integer values in the `field` and `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. +Returns the count and computed sum of all BSI integer values in the `field`. If the optional `Row` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. **Result Type:** object with the computed sum and count of the bitmap field. @@ -663,7 +638,7 @@ Returns the count and computed sum of all BSI integer values in the `field` and Query the size of all repositories. ```request -Sum(frame="stats", field="diskusage") +Sum(field="diskusage") ``` ```response {"value":10,"count":3} diff --git a/docs/webui.md b/docs/webui.md index 94377fbe8..a07132852 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -43,14 +43,14 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create index ` - `:delete index ` - `:use ` -- `:create frame ` -- `:delete frame ` +- `:create field ` +- `:delete field ` -Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). +Field creation also supports options like `timeQuantum`. When creating a new field, add options by using the keys documented in [API reference](../api-reference/#create-field). -- `:create frame cacheSize=10000` +- `:create field cacheSize=10000` ### Cluster Admin -Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. +Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Fields. From e76a90e69b5241d40ea20b56d80e82554428dc91 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 15:02:33 -0500 Subject: [PATCH 010/166] change Query and QueryNode to use pilosa.* Query structs --- cache.go | 8 ------- client.go | 14 +++++------- encoding/proto/proto.go | 6 ++++- executor.go | 47 ++++++-------------------------------- executor_test.go | 2 +- fragment.go | 2 +- http/client.go | 17 +++++++------- http/client_test.go | 9 ++++---- http/handler.go | 50 ----------------------------------------- 9 files changed, 32 insertions(+), 123 deletions(-) diff --git a/cache.go b/cache.go index 192f38cc7..4c826f5d7 100644 --- a/cache.go +++ b/cache.go @@ -409,14 +409,6 @@ func (p Pairs) String() string { return buf.String() } -func EncodePairs(a Pairs) []*internal.Pair { - other := make([]*internal.Pair, len(a)) - for i := range a { - other[i] = encodePair(a[i]) - } - return other -} - func decodePairs(a []*internal.Pair) []Pair { other := make([]Pair, len(a)) for i := range a { diff --git a/client.go b/client.go index 01e0b847a..5c51ae63f 100644 --- a/client.go +++ b/client.go @@ -3,8 +3,6 @@ package pilosa import ( "context" "io" - - "github.com/pilosa/pilosa/internal" ) // Bit represents the intersection of a row and a column. It can be specifed by @@ -35,8 +33,8 @@ type InternalClient interface { Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) - Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error @@ -55,12 +53,12 @@ type InternalClient interface { //=============== type InternalQueryClient interface { - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) } type NopInternalQueryClient struct{} -func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } @@ -90,10 +88,10 @@ func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt In func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { return nil, nil } -func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 0f2574f1b..341ad46f7 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -685,7 +685,11 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) - m.Err = errors.New(pb.Err) + if pb.Err == "" { + m.Err = nil + } else { + m.Err = errors.New(pb.Err) + } m.Results = make([]interface{}, len(pb.Results)) decodeQueryResults(pb.Results, m.Results) diff --git a/executor.go b/executor.go index 1a81e92bd..aa2c2fb88 100644 --- a/executor.go +++ b/executor.go @@ -337,7 +337,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrap(err, "map reduce") } // Attach attributes for Row() calls. @@ -1392,7 +1392,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p // exec executes a PQL query remotely for a set of shards on a node. func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) { // Encode request object. - pbreq := &internal.QueryRequest{ + pbreq := &QueryRequest{ Query: q.String(), Shards: shards, Remote: true, @@ -1403,40 +1403,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q * return nil, err } - // Return an error, if specified on response. - if err := decodeError(pb.Err); err != nil { - return nil, err - } - - // Return appropriate data for the query. - results = make([]interface{}, len(q.Calls)) - for i, call := range q.Calls { - var v interface{} - var err error - - switch call.Name { - case "Average", "Sum": - v, err = decodeValCount(pb.Results[i].GetValCount()), nil - case "TopN": - v, err = decodePairs(pb.Results[i].GetPairs()), nil - case "Count": - v, err = pb.Results[i].N, nil - case "Set": - v, err = pb.Results[i].Changed, nil - case "Clear": - v, err = pb.Results[i].Changed, nil - case "SetRowAttrs": - case "SetColumnAttrs": - default: - v, err = DecodeRow(pb.Results[i].GetRow()), nil - } - if err != nil { - return nil, err - } - - results[i] = v - } - return results, nil + return pb.Results, pb.Err } // shardsByNode returns a mapping of nodes to shards. @@ -1490,7 +1457,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, for { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, errors.Wrap(ctx.Err(), "context done") case resp := <-ch: // On error retry against remaining nodes. If an error returns then // the context will cancel and cause all open goroutines to return. @@ -1500,10 +1467,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { return nil, resp.err } else if err != nil { - return nil, err + return nil, errors.Wrap(err, "calling mapper") } continue } @@ -1524,7 +1491,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return err + return errors.Wrap(err, "shards by node") } // Execute each node in a separate goroutine. diff --git a/executor_test.go b/executor_test.go index f54fb7e33..fc72b00f3 100644 --- a/executor_test.go +++ b/executor_test.go @@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) { hldr.SetBit("i", "f", 1, 0) if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { - t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err)) + t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err) } } diff --git a/fragment.go b/fragment.go index b502e1597..1ae3c5bb3 100644 --- a/fragment.go +++ b/fragment.go @@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { } // Execute query. - queryRequest := &internal.QueryRequest{ + queryRequest := &QueryRequest{ Query: buffers[k].String(), Remote: true, } diff --git a/http/client.go b/http/client.go index 41602b778..d8fb7f7ef 100644 --- a/http/client.go +++ b/http/client.go @@ -220,22 +220,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard } // Query executes query against the index. -func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { if index == "" { return nil, pilosa.ErrIndexRequired } else if queryRequest.Query == "" { return nil, pilosa.ErrQueryRequired } - // Encode request object. - buf, err := proto.Marshal(queryRequest) + buf, err := c.serializer.Marshal(queryRequest) if err != nil { - return nil, errors.Wrap(err, "marshaling") + return nil, errors.Wrap(err, "marshaling queryRequest") } // Create HTTP request. @@ -265,11 +264,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s return nil, errors.New(string(body)) } - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { + qresp := &pilosa.QueryResponse{} + if err := c.serializer.Unmarshal(body, qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) + } else if qresp.Err != nil { + return nil, qresp.Err } return qresp, nil diff --git a/http/client_test.go b/http/client_test.go index fb1105a27..38a677673 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -24,7 +24,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" @@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) { client[2] = MustNewClient(c[2].URL(), defaultClient) topN := 4 - queryRequest := &internal.QueryRequest{ + queryRequest := &pilosa.QueryRequest{ Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), Remote: false, } @@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) { } // Test must return exactly N results. - if len(result.Results[0].Pairs) != topN { + if len(result.Results[0].([]pilosa.Pair)) != topN { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } - p := []*internal.Pair{ + p := []pilosa.Pair{ {ID: 100, Count: 12}, {ID: 22, Count: 10}, {ID: 98, Count: 8}, {ID: 99, Count: 7}} // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.Results[0].Pairs, p) { + if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } diff --git a/http/handler.go b/http/handler.go index 84832b1d4..33085879c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1102,56 +1102,6 @@ const ( QueryResultTypeBool ) -func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { - req := &pilosa.QueryRequest{ - Query: pb.Query, - Shards: pb.Shards, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - -func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *pilosa.Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = pilosa.EncodeRow(result) - case []pilosa.Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = pilosa.EncodePairs(result) - case pilosa.ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = pilosa.EncodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb -} - // parseUint64Slice returns a slice of uint64s from a comma-delimited string. func parseUint64Slice(s string) ([]uint64, error) { var a []uint64 From da4cd8482067f7e1c6cc599d3191f1affc3e56b2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 15:31:07 -0500 Subject: [PATCH 011/166] Remove some dead code --- http/client.go | 10 ---------- http/handler.go | 4 ---- pql/ast.go | 33 --------------------------------- server/config.go | 7 ------- test/holder.go | 9 --------- 5 files changed, 63 deletions(-) diff --git a/http/client.go b/http/client.go index 27597a5cb..7c88bd8d3 100644 --- a/http/client.go +++ b/http/client.go @@ -27,19 +27,12 @@ import ( "sort" "strconv" - "crypto/tls" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) -// ClientOptions represents the configuration for a InternalHTTPClient -type ClientOptions struct { - TLS *tls.Config -} - // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { defaultURI *pilosa.URI @@ -70,9 +63,6 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) } } -// Host returns the host the client was initialized with. -func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI } - // MaxShardByIndex returns the number of shards on a server by index. func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { return c.maxShardByIndex(ctx) diff --git a/http/handler.go b/http/handler.go index 8b2015094..da9a84d9f 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1329,10 +1329,6 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } } -func (h *Handler) GetAPI() *pilosa.API { - return h.API -} - type defaultClusterMessageResponse struct{} func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { diff --git a/pql/ast.go b/pql/ast.go index 0bcc582d4..344292ed4 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -220,23 +220,6 @@ func (q *Query) WriteCallN() int { return n } -// HasKeys returns true if any call in the query uses keys and requires translation to ids. -func (q *Query) HasKeys() bool { - for _, call := range q.Calls { - if call.Args["col"] != nil { - if _, ok := call.Args["col"].(string); ok { - return true - } - } - if call.Args["row"] != nil { - if _, ok := call.Args["row"].(string); ok { - return true - } - } - } - return false -} - // String returns a string representation of the query. func (q *Query) String() string { a := make([]string, len(q.Calls)) @@ -309,22 +292,6 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } -// StringArg is for reading the value at key from call.Args as a string. If the -// key is not in Call.Args, the value of the returned bool will be false, and -// the error will be nil. An error is returned if the value is not a string. -func (c *Call) StringArg(key string) (string, bool, error) { - val, ok := c.Args[key] - if !ok { - return "", false, nil - } - switch tval := val.(type) { - case string: - return tval, true, nil - default: - return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval) - } -} - // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/server/config.go b/server/config.go index 55da45768..73f0b289e 100644 --- a/server/config.go +++ b/server/config.go @@ -21,13 +21,6 @@ import ( "github.com/pilosa/pilosa/toml" ) -// Cluster types. -const ( - ClusterNone = "" - ClusterStatic = "static" - ClusterGossip = "gossip" -) - // TLSConfig contains TLS configuration type TLSConfig struct { // CertificatePath contains the path to the certificate (.crt or .pem file) diff --git a/test/holder.go b/test/holder.go index 9cc96fe14..ff87ae02c 100644 --- a/test/holder.go +++ b/test/holder.go @@ -81,15 +81,6 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption return &Index{Index: idx} } -// MustCreateFieldIfNotExists returns a given field. Panic on error. -func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) - if err != nil { - panic(err) - } - return f -} - // Row returns a Row for a given field. func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) From bbb93abd57cd1f0a7195d504f5257c391e247b78 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 15:35:32 -0500 Subject: [PATCH 012/166] remove lots of unused code --- api.go | 4 +- broadcast.go | 87 --------- cluster.go | 410 ---------------------------------------- encoding/proto/proto.go | 10 +- executor.go | 15 -- field.go | 34 ---- holder.go | 15 -- index.go | 24 --- row.go | 27 --- uri.go | 29 --- 10 files changed, 11 insertions(+), 644 deletions(-) diff --git a/api.go b/api.go index a9bd02f2d..1abe4502c 100644 --- a/api.go +++ b/api.go @@ -770,8 +770,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // Send the set-coordinator message to new node. err = api.server.SendTo( newNode, - &internal.SetCoordinatorMessage{ - New: EncodeNode(newNode), + &SetCoordinatorMessage{ + New: newNode, }) if err != nil { return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) diff --git a/broadcast.go b/broadcast.go index 00835db64..a3ea01a4f 100644 --- a/broadcast.go +++ b/broadcast.go @@ -17,8 +17,6 @@ package pilosa import ( "fmt" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -165,88 +163,3 @@ func getMessageType(m Message) byte { panic(fmt.Sprintf("don't have type for message %#v", m)) } } - -// UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (proto.Message, error) { - typ, buf := buf[0], buf[1:] - var m proto.Message - switch typ { - case messageTypeCreateShard: - m = &internal.CreateShardMessage{} - case messageTypeCreateIndex: - m = &internal.CreateIndexMessage{} - case messageTypeDeleteIndex: - m = &internal.DeleteIndexMessage{} - case messageTypeCreateField: - m = &internal.CreateFieldMessage{} - case messageTypeDeleteField: - m = &internal.DeleteFieldMessage{} - case messageTypeCreateView: - m = &internal.CreateViewMessage{} - case messageTypeDeleteView: - m = &internal.DeleteViewMessage{} - case messageTypeClusterStatus: - m = &internal.ClusterStatus{} - case messageTypeResizeInstruction: - m = &internal.ResizeInstruction{} - case messageTypeResizeInstructionComplete: - m = &internal.ResizeInstructionComplete{} - case messageTypeSetCoordinator: - m = &internal.SetCoordinatorMessage{} - case messageTypeUpdateCoordinator: - m = &internal.UpdateCoordinatorMessage{} - case messageTypeNodeState: - m = &internal.NodeStateMessage{} - case messageTypeRecalculateCaches: - m = &internal.RecalculateCaches{} - case messageTypeNodeEvent: - m = &internal.NodeEventMessage{} - case messageTypeNodeStatus: - m = &internal.NodeStatus{} - default: - return nil, fmt.Errorf("invalid message type: %d", typ) - } - - if err := proto.Unmarshal(buf, m); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - return m, nil -} - -func decode(m proto.Message) Message { - switch mt := m.(type) { - case *internal.CreateShardMessage: - return decodeCreateShardMessage(mt) - case *internal.CreateIndexMessage: - return decodeCreateIndexMessage(mt) - case *internal.DeleteIndexMessage: - return decodeDeleteIndexMessage(mt) - case *internal.CreateFieldMessage: - return decodeCreateFieldMessage(mt) - case *internal.DeleteFieldMessage: - return decodeDeleteFieldMessage(mt) - case *internal.CreateViewMessage: - return decodeCreateViewMessage(mt) - case *internal.DeleteViewMessage: - return decodeDeleteViewMessage(mt) - case *internal.ClusterStatus: - return decodeClusterStatus(mt) - case *internal.ResizeInstruction: - return decodeResizeInstruction(mt) - case *internal.ResizeInstructionComplete: - return decodeResizeInstructionComplete(mt) - case *internal.SetCoordinatorMessage: - return decodeSetCoordinatorMessage(mt) - case *internal.UpdateCoordinatorMessage: - return decodeUpdateCoordinatorMessage(mt) - case *internal.NodeStateMessage: - return decodeNodeStateMessage(mt) - case *internal.RecalculateCaches: - return decodeRecalculateCaches(mt) - case *internal.NodeEventMessage: - return decodeNodeEventMessage(mt) - case *internal.NodeStatus: - return decodeNodeStatus(mt) - } - return nil -} diff --git a/cluster.go b/cluster.go index e2f29fde5..541c83642 100644 --- a/cluster.go +++ b/cluster.go @@ -1757,28 +1757,6 @@ type ResizeInstruction struct { ClusterStatus *ClusterStatus } -func decodeResizeInstruction(ri *internal.ResizeInstruction) *ResizeInstruction { - return &ResizeInstruction{ - JobID: ri.JobID, - Node: DecodeNode(ri.Node), - Coordinator: DecodeNode(ri.Coordinator), - Sources: decodeResizeSources(ri.Sources), - Schema: decodeSchema(ri.Schema), - ClusterStatus: decodeClusterStatus(ri.ClusterStatus), - } -} - -func encodeResizeInstruction(m *ResizeInstruction) *internal.ResizeInstruction { - return &internal.ResizeInstruction{ - JobID: m.JobID, - Node: EncodeNode(m.Node), - Coordinator: EncodeNode(m.Coordinator), - Sources: encodeResizeSources(m.Sources), - Schema: encodeSchema(m.Schema), - ClusterStatus: encodeClusterStatus(m.ClusterStatus), - } -} - type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -1787,192 +1765,11 @@ type ResizeSource struct { Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func decodeResizeSources(srcs []*internal.ResizeSource) []*ResizeSource { - new := make([]*ResizeSource, 0, len(srcs)) - for _, src := range srcs { - new = append(new, decodeResizeSource(src)) - } - return new -} - -func encodeResizeSources(srcs []*ResizeSource) []*internal.ResizeSource { - new := make([]*internal.ResizeSource, 0, len(srcs)) - for _, src := range srcs { - new = append(new, encodeResizeSource(src)) - } - return new -} - -func decodeResizeSource(rs *internal.ResizeSource) *ResizeSource { - return &ResizeSource{ - Node: DecodeNode(rs.Node), - Index: rs.Index, - Field: rs.Field, - View: rs.View, - Shard: rs.Shard, - } -} - -func encodeResizeSource(m *ResizeSource) *internal.ResizeSource { - return &internal.ResizeSource{ - Node: EncodeNode(m.Node), - Index: m.Index, - Field: m.Field, - View: m.View, - Shard: m.Shard, - } -} - // Schema is a schema type Schema struct { Indexes []*IndexInfo } -func decodeSchema(s *internal.Schema) *Schema { - return &Schema{ - Indexes: decodeIndexes(s.Indexes), - } -} - -func encodeSchema(m *Schema) *internal.Schema { - return &internal.Schema{ - Indexes: encodeIndexInfos(m.Indexes), - } -} - -func decodeIndexes(idxs []*internal.Index) []*IndexInfo { - new := make([]*IndexInfo, 0, len(idxs)) - for _, idx := range idxs { - new = append(new, decodeIndex(idx)) - } - return new -} - -func encodeIndexInfos(idxs []*IndexInfo) []*internal.Index { - new := make([]*internal.Index, 0, len(idxs)) - for _, idx := range idxs { - new = append(new, encodeIndexInfo(idx)) - } - return new -} - -func decodeIndex(idx *internal.Index) *IndexInfo { - return &IndexInfo{ - Name: idx.Name, - Fields: decodeFields(idx.Fields), - } -} - -func encodeIndexInfo(idx *IndexInfo) *internal.Index { - return &internal.Index{ - Name: idx.Name, - Fields: encodeFieldInfos(idx.Fields), - } -} - -func decodeFields(fs []*internal.Field) []*FieldInfo { - new := make([]*FieldInfo, 0, len(fs)) - for _, f := range fs { - new = append(new, decodeField(f)) - } - return new -} - -func encodeFieldInfos(fs []*FieldInfo) []*internal.Field { - new := make([]*internal.Field, 0, len(fs)) - for _, f := range fs { - new = append(new, encodeFieldInfo(f)) - } - return new -} - -func decodeField(f *internal.Field) *FieldInfo { - fi := &FieldInfo{ - Name: f.Name, - Options: *decodeFieldOptions(f.Meta), - Views: make([]*ViewInfo, 0, len(f.Views)), - } - for _, viewname := range f.Views { - fi.Views = append(fi.Views, &ViewInfo{Name: viewname}) - } - return fi -} - -func encodeFieldInfo(f *FieldInfo) *internal.Field { - ifield := &internal.Field{ - Name: f.Name, - Meta: encodeFieldOptions(&f.Options), - Views: make([]string, 0, len(f.Views)), - } - - for _, viewinfo := range f.Views { - ifield.Views = append(ifield.Views, viewinfo.Name) - } - return ifield -} - -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*Node) []*internal.Node { - other := make([]*internal.Node, len(a)) - for i := range a { - other[i] = EncodeNode(a[i]) - } - return other -} - -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *Node) *internal.Node { - return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), - IsCoordinator: n.IsCoordinator, - } -} - -// DecodeNodes converts a proto message into a slice of Nodes. -func DecodeNodes(a []*internal.Node) []*Node { - if len(a) == 0 { - return nil - } - other := make([]*Node, len(a)) - for i := range a { - other[i] = DecodeNode(a[i]) - } - return other -} - -func decodeClusterStatus(cs *internal.ClusterStatus) *ClusterStatus { - return &ClusterStatus{ - State: cs.State, - ClusterID: cs.ClusterID, - Nodes: DecodeNodes(cs.Nodes), - } -} - -func encodeClusterStatus(m *ClusterStatus) *internal.ClusterStatus { - return &internal.ClusterStatus{ - State: m.State, - ClusterID: m.ClusterID, - Nodes: EncodeNodes(m.Nodes), - } -} - -// DecodeNode converts a proto message into a Node. -func DecodeNode(node *internal.Node) *Node { - return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), - IsCoordinator: node.IsCoordinator, - } -} - -func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { - return &NodeEvent{ - Event: NodeEventType(ne.Event), - Node: DecodeNode(ne.Node), - } -} - func encodeTopology(topology *Topology) *internal.Topology { if topology == nil { return nil @@ -2004,267 +1801,60 @@ type CreateShardMessage struct { Shard uint64 } -func encodeCreateShardMessage(m *CreateShardMessage) *internal.CreateShardMessage { - return &internal.CreateShardMessage{ - Index: m.Index, - Shard: m.Shard, - } -} - -func decodeCreateShardMessage(pb *internal.CreateShardMessage) *CreateShardMessage { - return &CreateShardMessage{ - Index: pb.Index, - Shard: pb.Shard, - } -} - type CreateIndexMessage struct { Index string Meta *IndexOptions } -func encodeCreateIndexMessage(m *CreateIndexMessage) *internal.CreateIndexMessage { - return &internal.CreateIndexMessage{ - Index: m.Index, - Meta: encodeIndexMeta(m.Meta), - } -} - -func decodeCreateIndexMessage(pb *internal.CreateIndexMessage) *CreateIndexMessage { - return &CreateIndexMessage{ - Index: pb.Index, - Meta: decodeIndexMeta(pb.Meta), - } -} - -func encodeIndexMeta(m *IndexOptions) *internal.IndexMeta { - return &internal.IndexMeta{ - Keys: m.Keys, - } -} - -func decodeIndexMeta(pb *internal.IndexMeta) *IndexOptions { - return &IndexOptions{ - Keys: pb.Keys, - } -} - type DeleteIndexMessage struct { Index string } -func encodeDeleteIndexMessage(m *DeleteIndexMessage) *internal.DeleteIndexMessage { - return &internal.DeleteIndexMessage{ - Index: m.Index, - } -} - -func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage) *DeleteIndexMessage { - return &DeleteIndexMessage{ - Index: pb.Index, - } -} - type CreateFieldMessage struct { Index string Field string Meta *FieldOptions } -func encodeCreateFieldMessage(m *CreateFieldMessage) *internal.CreateFieldMessage { - return &internal.CreateFieldMessage{ - Index: m.Index, - Field: m.Field, - Meta: encodeFieldOptions(m.Meta), - } -} - -func decodeCreateFieldMessage(pb *internal.CreateFieldMessage) *CreateFieldMessage { - return &CreateFieldMessage{ - Index: pb.Index, - Field: pb.Field, - Meta: decodeFieldOptions(pb.Meta), - } -} - type DeleteFieldMessage struct { Index string Field string } -func encodeDeleteFieldMessage(m *DeleteFieldMessage) *internal.DeleteFieldMessage { - return &internal.DeleteFieldMessage{ - Index: m.Index, - Field: m.Field, - } -} - -func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage) *DeleteFieldMessage { - return &DeleteFieldMessage{ - Index: pb.Index, - Field: pb.Field, - } -} - type CreateViewMessage struct { Index string Field string View string } - -func encodeCreateViewMessage(m *CreateViewMessage) *internal.CreateViewMessage { - return &internal.CreateViewMessage{ - Index: m.Index, - Field: m.Field, - View: m.View, - } -} - -func decodeCreateViewMessage(pb *internal.CreateViewMessage) *CreateViewMessage { - return &CreateViewMessage{ - Index: pb.Index, - Field: pb.Field, - View: pb.View, - } -} - type DeleteViewMessage struct { Index string Field string View string } -func encodeDeleteViewMessage(m *DeleteViewMessage) *internal.DeleteViewMessage { - return &internal.DeleteViewMessage{ - Index: m.Index, - Field: m.Field, - View: m.View, - } -} - -func decodeDeleteViewMessage(pb *internal.DeleteViewMessage) *DeleteViewMessage { - return &DeleteViewMessage{ - Index: pb.Index, - Field: pb.Field, - View: pb.View, - } -} - type ResizeInstructionComplete struct { JobID int64 Node *Node Error string } -func encodeResizeInstructionComplete(m *ResizeInstructionComplete) *internal.ResizeInstructionComplete { - return &internal.ResizeInstructionComplete{ - JobID: m.JobID, - Node: EncodeNode(m.Node), - Error: m.Error, - } -} - -func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete) *ResizeInstructionComplete { - return &ResizeInstructionComplete{ - JobID: pb.JobID, - Node: DecodeNode(pb.Node), - Error: pb.Error, - } -} - type SetCoordinatorMessage struct { New *Node } -func encodeSetCoordinatorMessage(m *SetCoordinatorMessage) *internal.SetCoordinatorMessage { - return &internal.SetCoordinatorMessage{ - New: EncodeNode(m.New), - } -} - -func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage) *SetCoordinatorMessage { - return &SetCoordinatorMessage{ - New: DecodeNode(pb.New), - } -} - type UpdateCoordinatorMessage struct { New *Node } -func encodeUpdateCoordinatorMessage(m *UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { - return &internal.UpdateCoordinatorMessage{ - New: EncodeNode(m.New), - } -} - -func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage) *UpdateCoordinatorMessage { - return &UpdateCoordinatorMessage{ - New: DecodeNode(pb.New), - } -} - type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } -func encodeNodeStateMessage(m *NodeStateMessage) *internal.NodeStateMessage { - return &internal.NodeStateMessage{ - NodeID: m.NodeID, - State: m.State, - } -} - -func decodeNodeStateMessage(pb *internal.NodeStateMessage) *NodeStateMessage { - return &NodeStateMessage{ - NodeID: pb.NodeID, - State: pb.State, - } -} - -func encodeNodeEventMessage(m *NodeEvent) *internal.NodeEventMessage { - return &internal.NodeEventMessage{ - Event: uint32(m.Event), - Node: EncodeNode(m.Node), - } -} - -func decodeNodeEventMessage(pb *internal.NodeEventMessage) *NodeEvent { - return &NodeEvent{ - Event: NodeEventType(pb.Event), - Node: DecodeNode(pb.Node), - } -} - type NodeStatus struct { Node *Node MaxShards map[string]uint64 Schema *Schema } -func encodeNodeStatus(m *NodeStatus) *internal.NodeStatus { - return &internal.NodeStatus{ - Node: EncodeNode(m.Node), - MaxShards: &internal.MaxShards{Standard: m.MaxShards}, - Schema: encodeSchema(m.Schema), - } -} - -func decodeNodeStatus(pb *internal.NodeStatus) *NodeStatus { - return &NodeStatus{ - Node: DecodeNode(pb.Node), - MaxShards: pb.MaxShards.Standard, - Schema: decodeSchema(pb.Schema), - } -} - type RecalculateCaches struct{} - -func decodeRecalculateCaches(pb *internal.RecalculateCaches) *RecalculateCaches { - return &RecalculateCaches{} -} - -func encodeRecalculateCaches(*RecalculateCaches) *internal.RecalculateCaches { - return &internal.RecalculateCaches{} -} diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 341ad46f7..003e9cd94 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -375,11 +375,19 @@ func EncodeNodes(a []*pilosa.Node) []*internal.Node { func encodeNode(n *pilosa.Node) *internal.Node { return &internal.Node{ ID: n.ID, - URI: n.URI.Encode(), + URI: encodeURI(n.URI), IsCoordinator: n.IsCoordinator, } } +func encodeURI(u pilosa.URI) *internal.URI { + return &internal.URI{ + Scheme: u.Scheme, + Host: u.Host, + Port: uint32(u.Port), + } +} + func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { return &internal.ClusterStatus{ State: m.State, diff --git a/executor.go b/executor.go index aa2c2fb88..399da685d 100644 --- a/executor.go +++ b/executor.go @@ -20,7 +20,6 @@ import ( "sort" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -1738,20 +1737,6 @@ func (vc *ValCount) Add(other ValCount) ValCount { } } -func EncodeValCount(vc ValCount) *internal.ValCount { - return &internal.ValCount{ - Val: vc.Val, - Count: vc.Count, - } -} - -func decodeValCount(pb *internal.ValCount) ValCount { - return ValCount{ - Val: pb.Val, - Count: pb.Count, - } -} - // Smaller returns the smaller of the two ValCounts. func (vc *ValCount) Smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { diff --git a/field.go b/field.go index fb7e9af4f..0ef5630e3 100644 --- a/field.go +++ b/field.go @@ -1088,25 +1088,6 @@ func (f *Field) MarshalJSON() ([]byte, error) { return json.Marshal(thing) } -// encodeFields converts a into its internal representation. -func encodeFields(a []*Field) []*internal.Field { - other := make([]*internal.Field, len(a)) - for i := range a { - other[i] = encodeField(a[i]) - } - return other -} - -// encodeField converts f into its internal representation. -func encodeField(f *Field) *internal.Field { - fo := f.options - return &internal.Field{ - Name: f.name, - Meta: fo.Encode(), - Views: f.viewNames(), - } -} - type fieldSlice []*Field func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -1170,21 +1151,6 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { } } -func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { - if options == nil { - return nil - } - return &FieldOptions{ - Type: options.Type, - CacheType: options.CacheType, - CacheSize: options.CacheSize, - Min: options.Min, - Max: options.Max, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Keys: options.Keys, - } -} - func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { case FieldTypeSet: diff --git a/holder.go b/holder.go index 10e147098..cb067db92 100644 --- a/holder.go +++ b/holder.go @@ -27,7 +27,6 @@ import ( "syscall" "time" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -256,20 +255,6 @@ func (h *Holder) applySchema(schema *Schema) error { return nil } -// encodeMaxShards creates and internal representation of max shards. -func (h *Holder) encodeMaxShards() *internal.MaxShards { - return &internal.MaxShards{ - Standard: h.maxShards(), - } -} - -// encodeSchema creates an internal representation of schema. -func (h *Holder) encodeSchema() *internal.Schema { - return &internal.Schema{ - Indexes: EncodeIndexes(h.Indexes()), - } -} - // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } diff --git a/index.go b/index.go index 98f50eced..7451030cd 100644 --- a/index.go +++ b/index.go @@ -403,35 +403,11 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// EncodeIndexes converts a into its internal representation. -func EncodeIndexes(a []*Index) []*internal.Index { - other := make([]*internal.Index, len(a)) - for i := range a { - other[i] = encodeIndex(a[i]) - } - return other -} - -// encodeIndex converts d into its internal representation. -func encodeIndex(d *Index) *internal.Index { - return &internal.Index{ - Name: d.name, - Fields: encodeFields(d.Fields()), - } -} - // IndexOptions represents options to set when initializing an index. type IndexOptions struct { Keys bool `json:"keys"` } -// Encode converts i into its internal representation. -func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{ - Keys: i.Keys, - } -} - // hasTime returns true if a contains a non-nil time. func hasTime(a []*time.Time) bool { for _, t := range a { diff --git a/row.go b/row.go index cbfa6b270..2134026e9 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,6 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" ) @@ -271,32 +270,6 @@ func (r *Row) Columns() []uint64 { return a } -// EncodeRow converts r into its internal representation. -func EncodeRow(r *Row) *internal.Row { - if r == nil { - return nil - } - - return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), - } -} - -// DecodeRow converts r from its internal representation. -func DecodeRow(pr *internal.Row) *Row { - if pr == nil { - return nil - } - - r := NewRow() - r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Columns { - r.SetBit(v) - } - return r -} - // Union performs a union on a slice of rows. func Union(rows []*Row) *Row { other := rows[0] diff --git a/uri.go b/uri.go index 2058f70fa..b58678381 100644 --- a/uri.go +++ b/uri.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -206,34 +205,6 @@ func parseAddress(address string) (uri *URI, err error) { return uri, nil } -// Encode converts o into its internal representation. -func (u URI) Encode() *internal.URI { - return encodeURI(u) -} - -func encodeURI(u URI) *internal.URI { - return &internal.URI{ - Scheme: u.Scheme, - Host: u.Host, - Port: uint32(u.Port), - } -} - -func DecodeURI(i *internal.URI) URI { - return decodeURI(i) -} - -func decodeURI(i *internal.URI) URI { - if i == nil { - return URI{} - } - return URI{ - Scheme: i.Scheme, - Host: i.Host, - Port: uint16(i.Port), - } -} - // MarshalJSON marshals URI into a JSON-encoded byte slice. func (u *URI) MarshalJSON() ([]byte, error) { var output struct { From db2a53223d591a396c5d50964d3e675069616af9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:02:18 -0500 Subject: [PATCH 013/166] remove internal references from api and http/* --- api.go | 14 ++--- encoding/proto/proto.go | 134 +++++++++++++++++++++++++++++++++++++++- handler.go | 37 +++++++++++ http/client.go | 34 +++++----- http/handler.go | 12 ++-- 5 files changed, 197 insertions(+), 34 deletions(-) diff --git a/api.go b/api.go index 1abe4502c..181d1b3fb 100644 --- a/api.go +++ b/api.go @@ -26,8 +26,6 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -437,8 +435,8 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, if err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read body error")) } - var req internal.BlockDataRequest - if err := proto.Unmarshal(reqBytes, &req); err != nil { + var req BlockDataRequest + if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error")) } @@ -448,11 +446,11 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, return nil, ErrFragmentNotFound } - var resp = internal.BlockDataResponse{} + var resp = BlockDataResponse{} resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. - buf, err := proto.Marshal(&resp) + buf, err := api.Serializer.Marshal(&resp) if err != nil { return nil, errors.Wrap(err, "merge block response encoding error") } @@ -657,7 +655,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s } // Import bulk imports data into a particular index,field,shard. -func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { +func (api *API) Import(ctx context.Context, req *ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } @@ -686,7 +684,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { } // ImportValue bulk imports values into a particular field. -func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { +func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { return errors.Wrap(err, "validating api method") } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 003e9cd94..ed751247e 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -178,7 +178,46 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeQueryResponse(msg, mt) return nil - + case *pilosa.ImportRequest: + msg := &internal.ImportRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportRequest") + } + decodeImportRequest(msg, mt) + return nil + case *pilosa.ImportValueRequest: + msg := &internal.ImportValueRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportValueRequest") + } + decodeImportValueRequest(msg, mt) + return nil + case *pilosa.ImportResponse: + msg := &internal.ImportResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportResponse") + } + decodeImportResponse(msg, mt) + return nil + case *pilosa.BlockDataRequest: + msg := &internal.BlockDataRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataRequest") + } + decodeBlockDataRequest(msg, mt) + return nil + case *pilosa.BlockDataResponse: + msg := &internal.BlockDataResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataResponse") + } + decodeBlockDataResponse(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -224,10 +263,66 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeQueryRequest(mt) case *pilosa.QueryResponse: return encodeQueryResponse(mt) + case *pilosa.ImportRequest: + return encodeImportRequest(mt) + case *pilosa.ImportValueRequest: + return encodeImportValueRequest(mt) + case *pilosa.ImportResponse: + return encodeImportResponse(mt) + case *pilosa.BlockDataRequest: + return encodeBlockDataRequest(mt) + case *pilosa.BlockDataResponse: + return encodeBlockDataResponse(mt) } return nil } +func encodeBlockDataRequest(m *pilosa.BlockDataRequest) *internal.BlockDataRequest { + return &internal.BlockDataRequest{ + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + Block: m.Block, + } +} +func encodeBlockDataResponse(m *pilosa.BlockDataResponse) *internal.BlockDataResponse { + return &internal.BlockDataResponse{ + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + } +} + +func encodeImportResponse(m *pilosa.ImportResponse) *internal.ImportResponse { + return &internal.ImportResponse{ + Err: m.Err, + } +} + +func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { + return &internal.ImportRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, + } +} + +func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { + return &internal.ImportValueRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + } +} + func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { return &internal.QueryRequest{ Query: m.Query, @@ -690,6 +785,43 @@ func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { m.ExcludeColumns = pb.ExcludeColumns } +func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs + m.RowKeys = pb.RowKeys + m.ColumnKeys = pb.ColumnKeys + m.Timestamps = pb.Timestamps +} + +func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.ColumnIDs = pb.ColumnIDs + m.ColumnKeys = pb.ColumnKeys + m.Values = pb.Values +} + +func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) { + m.Err = pb.Err +} + +func decodeBlockDataRequest(pb *internal.BlockDataRequest, m *pilosa.BlockDataRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View + m.Shard = pb.Shard + m.Block = pb.Block +} + +func decodeBlockDataResponse(pb *internal.BlockDataResponse, m *pilosa.BlockDataResponse) { + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs +} + func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) diff --git a/handler.go b/handler.go index 1f3d04300..9fc3af368 100644 --- a/handler.go +++ b/handler.go @@ -75,3 +75,40 @@ func (n nopHandler) Close() error { } var NopHandler Handler = nopHandler{} + +type ImportValueRequest struct { + Index string + Field string + Shard uint64 + ColumnIDs []uint64 + ColumnKeys []string + Values []int64 +} + +type ImportRequest struct { + Index string + Field string + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +type ImportResponse struct { + Err string +} + +type BlockDataRequest struct { + Index string + Field string + View string + Shard uint64 + Block uint64 +} + +type BlockDataResponse struct { + RowIDs []uint64 + ColumnIDs []uint64 +} diff --git a/http/client.go b/http/client.go index d8fb7f7ef..19bf82815 100644 --- a/http/client.go +++ b/http/client.go @@ -29,10 +29,8 @@ import ( "crypto/tls" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - pilosaproto "github.com/pilosa/pilosa/encoding/proto" - "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pkg/errors" ) @@ -68,7 +66,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { return &InternalClient{ defaultURI: defaultURI, - serializer: pilosaproto.Serializer{}, + serializer: proto.Serializer{}, HTTPClient: remoteClient, } } @@ -282,7 +280,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard return pilosa.ErrFieldRequired } - buf, err := marshalImportPayload(index, field, shard, bits) + buf, err := c.marshalImportPayload(index, field, shard, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -311,7 +309,7 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, colum return pilosa.ErrFieldRequired } - buf, err := marshalImportPayloadK(index, field, columns) + buf, err := c.marshalImportPayloadK(index, field, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -345,14 +343,14 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { +func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ Index: index, Field: field, Shard: shard, @@ -367,14 +365,14 @@ func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) } // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { +func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowKeys := Bits(bits).RowKeys() columnKeys := Bits(bits).ColumnKeys() timestamps := Bits(bits).Timestamps() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ Index: index, Field: field, RowKeys: rowKeys, @@ -416,8 +414,8 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde return errors.New(string(body)) } - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { + var isresp pilosa.ImportResponse + if err := c.serializer.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) } else if s := isresp.Err; s != "" { return errors.New(s) @@ -434,7 +432,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s return pilosa.ErrFieldRequired } - buf, err := marshalImportValuePayload(index, field, shard, vals) + buf, err := c.marshalImportValuePayload(index, field, shard, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -456,13 +454,13 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { +func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{ Index: index, Field: field, Shard: shard, @@ -685,7 +683,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, if uri == nil { panic("need to pass a URI to BlockData") } - buf, err := proto.Marshal(&internal.BlockDataRequest{ + buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{ Index: index, Field: field, Shard: shard, @@ -721,10 +719,10 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, } // Decode response object. - var rsp internal.BlockDataResponse + var rsp pilosa.BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { + } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { return nil, nil, errors.Wrap(err, "unmarshalling") } return rsp.RowIDs, rsp.ColumnIDs, nil diff --git a/http/handler.go b/http/handler.go index 33085879c..2b3d3ebd8 100644 --- a/http/handler.go +++ b/http/handler.go @@ -33,11 +33,9 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -911,8 +909,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if field.Type() == pilosa.FieldTypeInt { // Field type: Int // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { + req := &pilosa.ImportValueRequest{} + if err := h.API.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -929,8 +927,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } else { // Field type: Set, Time // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { + req := &pilosa.ImportRequest{} + if err := h.API.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -947,7 +945,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: ""}) + buf, e := h.API.Serializer.Marshal(&pilosa.ImportResponse{Err: ""}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) return From a164233c92f890c45bab13fa837073abe3d96feb Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:21:27 -0500 Subject: [PATCH 014/166] fix handler tests not to use internal and fix bug --- encoding/proto/proto.go | 1 + server/handler_test.go | 80 +++++++++++++++++++---------------------- 2 files changed, 37 insertions(+), 44 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index ed751247e..0020260ab 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -837,6 +837,7 @@ func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { for i := range pb { + m[i] = &pilosa.ColumnAttrSet{} decodeColumnAttrSet(pb[i], m[i]) } } diff --git a/server/handler_test.go b/server/handler_test.go index e49b1eb0f..f0eeb18c8 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -27,10 +27,8 @@ import ( gohttp "net/http" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -144,7 +142,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Shards args protobuf", func(t *testing.T) { // Generate request body. - reqBody, err := proto.Marshal(&internal.QueryRequest{ + reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ Query: "Count(Row(f0=30))", Shards: []uint64{0, 1}, }) @@ -196,13 +194,11 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if n := resp.Results[0].N; n != 3 { - t.Fatalf("unexpected n: %d", n) + } else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) } }) @@ -244,27 +240,25 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + } else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) } }) t.Run("Row columnattrs protobuf", func(t *testing.T) { // Encode request body. - buf, err := proto.Marshal(&internal.QueryRequest{ + buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ Query: "Row(f0=30)", ColumnAttrs: true, }) @@ -281,22 +275,22 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + } else if _, ok := resp.Results[0].(*pilosa.Row); !ok { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) } if a := resp.ColumnAttrSets; len(a) != 2 { @@ -305,8 +299,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected column attr length: %d", len(a)) - } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if a[0].Attrs["x"] != "y" { + t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"]) } }) @@ -329,12 +323,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if a := resp.Results[0].GetPairs(); len(a) != 2 { + } else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } }) @@ -358,10 +350,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if s := resp.Err; s != `executing: field not found` { + } else if s := resp.Err.Error(); s != `executing: field not found` { t.Fatalf("unexpected error: %s", s) } }) From b7a583d6a8fc8ca687611d40885b1c59f7a5f0d6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 16:24:40 -0500 Subject: [PATCH 015/166] use Set() instead of SetValue() for integer fields --- executor.go | 123 ++++++++++++++++++++--------------------------- executor_test.go | 58 +++++++++++----------- pql/ast.go | 20 ++++++++ 3 files changed, 102 insertions(+), 99 deletions(-) diff --git a/executor.go b/executor.go index 1a81e92bd..96e327f43 100644 --- a/executor.go +++ b/executor.go @@ -185,8 +185,6 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeCount(ctx, index, c, shards, opt) case "Set": return e.executeSetBit(ctx, index, c, opt) - case "SetValue": - return nil, e.executeSetValue(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -1077,14 +1075,7 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, ErrFieldNotFound } - // Read fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Set() row: %v", err) - } else if !ok { - return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) - } - + // Read colID using labels. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { return false, fmt.Errorf("reading Set() column: %v", err) @@ -1092,20 +1083,40 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, fmt.Errorf("Set() column argument '%v' required", columnLabel) } - var timestamp *time.Time - sTimestamp, ok := c.Args["_timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) + if f.Type() == FieldTypeInt { + // Read remaining fields using labels. + rowVal, ok, err := c.IntArg(fieldName) if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) } - timestamp = &t - } - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) + } else { + // Read remaining fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + } } -// executeSetBitField executes a Set() call for a specific view. +// executeSetBitField executes a Set() call for a specific field. func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { shard := colID / ShardWidth ret := false @@ -1137,64 +1148,36 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. return ret, nil } -// executeSetValue executes a SetValue() call. -func (e *executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { - // Parse labels. - columnID, ok, err := c.UintArg(columnLabel) - if err != nil { - return fmt.Errorf("reading SetValue() column: %v", err) - } else if !ok { - return fmt.Errorf("SetValue() column field '%v' required", columnLabel) - } +// executeSetValueField executes a Set() call for a specific int field. +func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { + shard := colID / ShardWidth + ret := false - // Copy args and remove reserved fields. - args := pql.CopyArgs(c.Args) - // While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. - // Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility. - delete(args, columnLabel) - - // Set values. - for name, value := range args { - // Retrieve field. - field := e.Holder.Field(index, name) - if field == nil { - return ErrFieldNotFound - } - - switch value := value.(type) { - case int64: - if _, err := field.SetValue(columnID, value); err != nil { - return err + for _, node := range e.Cluster.shardNodes(index, shard) { + // Update locally if host matches. + if node.ID == e.Node.ID { + val, err := f.SetValue(colID, value) + if err != nil { + return false, err + } else if val { + ret = true } - default: - return ErrInvalidBSIGroupValueType + continue } - field.Stats.Count("SetValue", 1, 1.0) - } - // Do not forward call if this is already being forwarded. - if opt.Remote { - return nil - } + // Do not forward call if this is already being forwarded. + if opt.Remote { + continue + } - // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) - resp := make(chan error, len(nodes)) - for _, node := range nodes { - go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) - resp <- err - }(node) - } - - // Return first error. - for range nodes { - if err := <-resp; err != nil { - return err + // Forward call to remote node otherwise. + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + return false, err + } else { + ret = res[0].(bool) } } - - return nil + return ret, nil } // executeSetRowAttrs executes a SetRowAttrs() call. diff --git a/executor_test.go b/executor_test.go index c5e2f24a8..1bbeafddd 100644 --- a/executor_test.go +++ b/executor_test.go @@ -405,9 +405,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil { + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil { t.Fatal(err) } @@ -440,19 +440,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) @@ -748,14 +748,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) { Set(1, x=1) Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) - SetValue(col=0, f=20) - SetValue(col=1, f=-5) - SetValue(col=2, f=-5) - SetValue(col=3, f=10) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60) + Set(0, f=20) + Set(1, f=-5) + Set(2, f=-5) + Set(3, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) `}); err != nil { t.Fatal(err) } @@ -844,13 +844,13 @@ func TestExecutor_Execute_Sum(t *testing.T) { Set(0, x=0) Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - SetValue(col=0, foo=20) - SetValue(col=0, bar=2000) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) - SetValue(col=0, other=1000) + Set(0, foo=20) + Set(0, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) `}); err != nil { t.Fatal(err) } @@ -959,15 +959,15 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) - SetValue(col=50, foo=20) - SetValue(col=50, bar=2000) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) - SetValue(col=0, other=1000) - SetValue(col=0, edge=100) - SetValue(col=1, edge=-100) + Set(50, foo=20) + Set(50, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + Set(0, edge=100) + Set(1, edge=-100) `}); err != nil { t.Fatal(err) } diff --git a/pql/ast.go b/pql/ast.go index 0bcc582d4..4539bc95f 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -285,6 +285,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } } +// IntArg is for reading the value at key from call.Args as a uint64. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a uint64 or an int64 and +// then cast to a uint64. An error is returned if the value is not an int64 or +// uint64. +func (c *Call) IntArg(key string) (int64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return tval, true, nil + case uint64: + return int64(tval), true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.IntArg", tval, tval) + } +} + // UintSliceArg reads the value at key from call.Args as a slice of uint64. If // the key is not in Call.Args, the value of the returned bool will be false, // and the error will be nil. If the value is a slice of int64 it will convert From def3be0f17df00ccdfde76d829930f16aaf2e021 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:25:04 -0500 Subject: [PATCH 016/166] remove more dead code --- cache.go | 25 ------------------------- pilosa.go | 19 ------------------- 2 files changed, 44 deletions(-) diff --git a/cache.go b/cache.go index 4c826f5d7..044d85f3c 100644 --- a/cache.go +++ b/cache.go @@ -22,7 +22,6 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/lru" ) @@ -318,22 +317,6 @@ type Pair struct { Count uint64 `json:"count"` } -func encodePair(p Pair) *internal.Pair { - return &internal.Pair{ - ID: p.ID, - Key: p.Key, - Count: p.Count, - } -} - -func decodePair(pb *internal.Pair) Pair { - return Pair{ - ID: pb.ID, - Key: pb.Key, - Count: pb.Count, - } -} - // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -409,14 +392,6 @@ func (p Pairs) String() string { return buf.String() } -func decodePairs(a []*internal.Pair) []Pair { - other := make([]Pair, len(a)) - for i := range a { - other[i] = decodePair(a[i]) - } - return other -} - // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/pilosa.go b/pilosa.go index ebc2be438..9615e88b8 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,8 +17,6 @@ package pilosa import ( "errors" "regexp" - - "github.com/pilosa/pilosa/internal" ) // System errors. @@ -122,23 +120,6 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } -// EncodeColumnAttrSets converts a into its internal representation. -func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { - other := make([]*internal.ColumnAttrSet, len(a)) - for i := range a { - other[i] = EncodeColumnAttrSet(a[i]) - } - return other -} - -// EncodeColumnAttrSet converts set into its internal representation. -func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { - return &internal.ColumnAttrSet{ - ID: set.ID, - Attrs: encodeAttrs(set.Attrs), - } -} - // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" From 462b27d9a9f8b1e8d50fab7ccd105e25a3bfbf22 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 16:31:06 -0500 Subject: [PATCH 017/166] change executeSetBit to executeSet --- executor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 96e327f43..8ebc1df90 100644 --- a/executor.go +++ b/executor.go @@ -184,7 +184,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) case "Set": - return e.executeSetBit(ctx, index, c, opt) + return e.executeSet(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -1058,8 +1058,8 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq return ret, nil } -// executeSetBit executes a Set() call. -func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { +// executeSet executes a Set() call. +func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Set() argument required: field") From 5717e32310912583c31f5e74ac7152b3f850ad02 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 16:35:27 -0500 Subject: [PATCH 018/166] fix comment for IntArg --- pql/ast.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 4539bc95f..dc16f4cdf 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -285,10 +285,10 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } } -// IntArg is for reading the value at key from call.Args as a uint64. If the +// IntArg is for reading the value at key from call.Args as an int64. If the // key is not in Call.Args, the value of the returned bool will be false, and -// the error will be nil. The value is assumed to be a uint64 or an int64 and -// then cast to a uint64. An error is returned if the value is not an int64 or +// the error will be nil. The value is assumed to be a unt64 or an int64 and +// then cast to an int64. An error is returned if the value is not an int64 or // uint64. func (c *Call) IntArg(key string) (int64, bool, error) { val, ok := c.Args[key] @@ -301,7 +301,7 @@ func (c *Call) IntArg(key string) (int64, bool, error) { case uint64: return int64(tval), true, nil default: - return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.IntArg", tval, tval) + return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval) } } From d0485a3a1985a7eebd119c3f5b13240077207f44 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 16:51:23 -0500 Subject: [PATCH 019/166] remove unused code in row.go --- row.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/row.go b/row.go index 4d9ba3662..d6a0037cc 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,6 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" ) @@ -252,41 +251,6 @@ func (r *Row) Columns() []uint64 { return a } -// Union performs a union on a slice of rows. -func Union(rows []*Row) *Row { - other := rows[0] - for _, r := range rows[1:] { - other = other.Union(r) - } - return other -} - -// EncodeRow converts r into its internal representation. -func EncodeRow(r *Row) *internal.Row { - if r == nil { - return nil - } - - return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), - } -} - -// DecodeRow converts r from its internal representation. -func DecodeRow(pr *internal.Row) *Row { - if pr == nil { - return nil - } - - r := NewRow() - r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Columns { - r.SetBit(v) - } - return r -} - // RowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The // width of the segment will always match the shard width. From e33682cfd27a3d1efdaf6fb8888e3cc9d66f5bea Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 17:38:49 -0500 Subject: [PATCH 020/166] address feedback --- cluster.go | 4 ++-- utils_internal_test.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 6d8cf38b4..f61173c3b 100644 --- a/cluster.go +++ b/cluster.go @@ -459,7 +459,7 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { return nil } -// Status returns the the cluster's status including what nodes it contains, it's ID, and current state. +// Status returns the the cluster's status including what nodes it contains, its ID, and current state. func (c *cluster) Status() *ClusterStatus { return &ClusterStatus{ ClusterID: c.id, @@ -1764,7 +1764,7 @@ type ResizeSource struct { Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } -// Schema is a schema +// Schema contains information about indexes and their configuration. type Schema struct { Indexes []*IndexInfo } diff --git a/utils_internal_test.go b/utils_internal_test.go index e0a42bdc1..df1b7d2e0 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -364,8 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error } for _, src := range instr.Sources { - srcNode := src.Node - srcCluster := t.clusterByID(srcNode.ID) + srcCluster := t.clusterByID(src.Node.ID) srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) From 21cf6b6e57b1cd18c26a8a51609f452cab49f482 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 17:59:18 -0500 Subject: [PATCH 021/166] remove URI getters since the fields were exported for serialization --- gossip/gossip.go | 6 +++--- http/client.go | 4 ++-- server/server.go | 12 ++++++------ uri.go | 15 --------------- uri_internal_test.go | 14 +++++++------- 5 files changed, 18 insertions(+), 33 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 28b3ac690..7a11d01d4 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -146,7 +146,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - host := api.Node().URI.GetHost() + host := api.Node().URI.Host g := &GossipMemberSet{ papi: api, Logger: pilosa.NopLogger, @@ -191,10 +191,10 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO conf := memberlist.DefaultWANConfig() conf.Transport = g.transport.Net conf.Name = api.Node().ID - conf.BindAddr = api.Node().URI.GetHost() + conf.BindAddr = api.Node().URI.Host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = hostToIP(api.Node().URI.GetHost()) + conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult diff --git a/http/client.go b/http/client.go index 19bf82815..de6eb89f6 100644 --- a/http/client.go +++ b/http/client.go @@ -993,7 +993,7 @@ func pos(rowID, columnID uint64) uint64 { func uriPathToURL(uri *pilosa.URI, path string) url.URL { return url.URL{ - Scheme: uri.GetScheme(), + Scheme: uri.Scheme, Host: uri.HostPort(), Path: path, } @@ -1001,7 +1001,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL { func nodePathToURL(node *pilosa.Node, path string) url.URL { return url.URL{ - Scheme: node.URI.GetScheme(), + Scheme: node.URI.Scheme, Host: node.URI.HostPort(), Path: path, } diff --git a/server/server.go b/server/server.go index f46f6a64a..401da09e8 100644 --- a/server/server.go +++ b/server/server.go @@ -203,7 +203,7 @@ func (m *Command) SetupServer() error { // Setup TLS var TLSConfig *tls.Config - if uri.GetScheme() == "https" { + if uri.Scheme == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") } @@ -236,7 +236,7 @@ func (m *Command) SetupServer() error { } // If port is 0, get auto-allocated port from listener - if uri.GetPort() == 0 { + if uri.Port == 0 { uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) } @@ -311,7 +311,7 @@ func (m *Command) SetupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.API.Node().URI.GetHost() + gossipHost := m.API.Node().URI.Host m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return errors.Wrap(err, "getting transport") @@ -368,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { // getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS - if uri.GetScheme() == "https" && tlsconf != nil { + if uri.Scheme == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) if err != nil { return nil, errors.Wrap(err, "tls.Listener") } - } else if uri.GetScheme() == "http" { + } else if uri.Scheme == "http" { // Open HTTP listener to determine port (if specified as :0). ln, err = net.Listen("tcp", uri.HostPort()) if err != nil { return nil, errors.Wrap(err, "net.Listen") } } else { - return nil, errors.Errorf("unsupported scheme: %s", uri.GetScheme()) + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme) } return ln, nil diff --git a/uri.go b/uri.go index e01bf8bcc..8577c8238 100644 --- a/uri.go +++ b/uri.go @@ -82,11 +82,6 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// GetScheme returns the scheme of this URI. -func (u *URI) GetScheme() string { - return u.Scheme -} - // SetScheme sets the scheme of this URI. func (u *URI) SetScheme(scheme string) error { m := schemeRegexp.FindStringSubmatch(scheme) @@ -97,11 +92,6 @@ func (u *URI) SetScheme(scheme string) error { return nil } -// GetHost returns the host of this URI. -func (u *URI) GetHost() string { - return u.Host -} - // SetHost sets the host of this URI. func (u *URI) SetHost(host string) error { m := hostRegexp.FindStringSubmatch(host) @@ -112,11 +102,6 @@ func (u *URI) SetHost(host string) error { return nil } -// GetPort returns the port of this URI. -func (u *URI) GetPort() uint16 { - return u.Port -} - // SetPort sets the port of this URI. func (u *URI) SetPort(port uint16) { u.Port = port diff --git a/uri_internal_test.go b/uri_internal_test.go index 2587223f8..3c9631661 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -83,8 +83,8 @@ func TestSetScheme(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.GetScheme() != target { - t.Fatalf("%s != %s", uri.GetScheme(), target) + if uri.Scheme != target { + t.Fatalf("%s != %s", uri.Scheme, target) } } @@ -95,7 +95,7 @@ func TestSetHost(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.GetHost() != target { + if uri.Host != target { t.Fatalf("%s != %s", uri.Host, target) } } @@ -104,7 +104,7 @@ func TestSetPort(t *testing.T) { uri := DefaultURI() target := uint16(9999) uri.SetPort(target) - if uri.GetPort() != target { + if uri.Port != target { t.Fatalf("%d != %d", uri.Port, target) } } @@ -137,13 +137,13 @@ func TestHostPort(t *testing.T) { } func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { - if uri.GetScheme() != scheme { + if uri.Scheme != scheme { t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme) } - if uri.GetHost() != host { + if uri.Host != host { t.Fatalf("Host does not match: %s != %s", uri.Host, host) } - if uri.GetPort() != port { + if uri.Port != port { t.Fatalf("Port does not match: %d != %d", uri.Port, port) } } From 1f65dbcdf27763f1e31702f7bc958e756b08ab72 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 5 Jul 2018 18:12:30 -0500 Subject: [PATCH 022/166] More terminology updates --- docs/administration.md | 22 +++++++++++----------- docs/api-reference.md | 4 ++-- docs/client-libraries.md | 6 +++--- docs/configuration.md | 2 +- docs/data-model.md | 16 ++++++++-------- docs/examples.md | 1 - docs/glossary.md | 2 +- docs/pdk.md | 36 ++++++++++++++++++------------------ docs/query-language.md | 26 +++++++++++++------------- 9 files changed, 57 insertions(+), 58 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 6764823e9..71ab54672 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -70,9 +70,9 @@ pilosa import -i project -f stargazer-counts project-stargazer-counts.csv #### Exporting -Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a field. The data will be in csv format `Row,Column` and sorted by column. +Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the shard number, but the `pilosa export` sub command will export all shards within a field. The data will be in csv format `Row,Column` and sorted by column. ```request -curl "http://localhost:10101/export?index=repository&field=stargazer&slice=0" \ +curl "http://localhost:10101/export?index=repository&field=stargazer&shard=0" \ --header "Accept: text/csv" ``` ```response @@ -122,7 +122,7 @@ Pilosa v0.9 introduces a few compatibility changes that need to be addressed. Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called [`topology-generator`](https://github.com/pilosa/upgrade-utils/tree/master/v0.9/topology-generator): -1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. +1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. 2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`. 3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`. 4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position. @@ -211,7 +211,7 @@ curl localhost:10101/cluster/resize/set-coordinator \ ### Backup/restore -Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. +Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered shard files. These data files can be routinely backed up to restore nodes in a cluster. Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node. @@ -231,11 +231,11 @@ Note: This will only work when the replication factor is >= 2 - To accomplish this you will first need: - List of all indexes on your cluster - List of all fields in your indexes - - Max slice per index, listed in the `/slices/max` endpoint -- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each slice -- Using the list of slices owned by this node you will then need to manually: + - Max shard per index, listed in the `/internal/shards/max` endpoint +- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each shard +- Using the list of shards owned by this node you will then need to manually: - setup a directory structure similar to the other nodes with a path for each Index/Field - - copy each owned slice for an existing node to this new node + - copy each owned shard for an existing node to this new node - Modify the cluster config file to replace the previous node address with the new node address. - Restart the cluster - Wait for the first sync (10 minutes) to validate Index connections @@ -253,7 +253,7 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **TimeQuantumEnabled:** Time Quantum Fields in use. - **NumIndexes:** Number of indexes in the Cluster. - **NumFields:** Number of fields in the Cluster. -- **NumSlices:** Number of slices in the Cluster. +- **NumShards:** Number of shards in the Cluster. - **NumViews:** Number of views in the Cluster. - **OpenFiles:** Open file handle count. - **GoRoutines:** Go routine count. @@ -276,14 +276,14 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - Index - Field - View -- Slice +- Shard #### Events We currently track the following events - **Index:** The creation of a new index. - **Field:** The creation of a new field. -- **MaxSlice:** The creation of a new Slice. +- **MaxShard:** The creation of a new Shard. - **SetBit:** Count of set bits. - **ClearBit:** Count of cleared bits. - **ImportBit:** During a bulk data import this represents the count of bits created. diff --git a/docs/api-reference.md b/docs/api-reference.md index 41e0288ce..c3a0144df 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -78,10 +78,10 @@ In order to send protobuf binaries in the request and response, set `Content-Typ The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`. -The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. +The query is executed for all [shards](../data-model/#shard) by default. To use specified shards only, set the `shards` query argument to a comma-separated list of slice indices. ``` request -curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ +curl "localhost:10101/index/user/query?columnAttrs=true&shards=0,1" \ -X POST \ -d 'Row(language=5)' ``` diff --git a/docs/client-libraries.md b/docs/client-libraries.md index a492cf9b5..a277141ec 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -90,7 +90,7 @@ func main() { fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns) // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.SetBit(99999, 77777)) + client.Query(stargazer.Set(99999, 77777)) } ``` @@ -174,7 +174,7 @@ mutually_starred = client.query(query).result.row.columns print("User 14 or 19 starred, written in language 1:", mutually_starred) # Set user 99999 as a stargazer for repository 77777 -client.query(stargazer.setbit(99999, 77777)) +client.query(stargazer.set(99999, 77777)) ``` Running the above program should produce output like this: @@ -275,7 +275,7 @@ public class StarTrace { System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); // Set user 99999 as a stargazer for repository 77777: - client.query(stargazer.setBit(99999, 77777)); + client.query(stargazer.set(99999, 77777)); } } ``` diff --git a/docs/configuration.md b/docs/configuration.md index 200c91449..22a67ddcf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,7 +106,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Max Writes Per Request -* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue. +* Description: Maximum number of mutating commands allowed per request. This includes Set, Clear, SetRowAttrs, and SetColumnAttrs. * Flag: `--max-writes-per-request=5000` * Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000` * Config: diff --git a/docs/data-model.md b/docs/data-model.md index bdcec4861..e453d9245 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -9,7 +9,7 @@ nav = [ "Field", "Time Quantum", "Attribute", - "Slice", + "Shard", "View", ] +++ @@ -64,13 +64,13 @@ Entities: Simple queries: - Relational | Pilosa ----------------------------------------------|------------------------------------ - `select ID from People where Name = 'Bob'` | `Row(Name="Bob")` - `select ID from People where Age > 30` | `Range(Age > 30)` - `select ID from People where Member = true` | `Row(Member=0)` # TODO this is unfortunate + Relational | Pilosa +-----------------------------------------------|------------------------------------ + `select ID from People where Name = 'Bob'` | `Row(Name="Bob")` + `select ID from People where Age > 30` | `Range(Age > 30)` + `select ID from People where Member = true` | `Row(Member=0)` -In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: +Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: ```sql select AVG(p.Age) from People p @@ -141,7 +141,7 @@ Set(3, A=8, 2017-05-19T00:00) Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. -Internally Pilosa stores each BSI (TODO!!!!!) `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below: diff --git a/docs/examples.md b/docs/examples.md index 1660e8592..692cd7e39 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -3,7 +3,6 @@ title = "Examples" weight = 4 nav = [ "Transportation", - "Chemical similarity search", ] +++ diff --git a/docs/glossary.md b/docs/glossary.md index eb5dd780e..81323674d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -56,7 +56,7 @@ nav = [] [Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap). -[Slice](../data-model/#slice): Prior to Pilosa 1.0, shards were known as slices. +[Slice](../data-model/#shard): Prior to Pilosa 1.0, shards were known as slices. [Shard](../data-model/#shard): [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). diff --git a/docs/pdk.md b/docs/pdk.md index 2ca4b6ee8..dd6ddaf61 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -18,7 +18,7 @@ Running `pdk -h` will give the most up to date list of all the tools and example `pdk kafka` reads either JSON or Avro encoded records from Kafka (using the Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each record from Kafka is assigned a Pilosa column, and each value in a record is -assigned a row or field. Frame and field names are built from the "path" through +assigned a row or field. Pilosa field names are built from the "path" through the record to arrive at that field. For example: ```json @@ -38,30 +38,30 @@ the record to arrive at that field. For example: This JSON object would result in the following Pilosa schema: -| Name | Field | Type | Min | Max | Size | -|----------------|-----------|--------|-----|------------|--------| -| name | | ranked | | | 100000 | -| favorite_foods | | ranked | | | 100000 | -| default | | ranked | | | 100000 | -| | age | int | 0 | 2147483647 | | -| location | | ranked | | | 1000 | -| | latitude | int | 0 | 2147483647 | | -| | longitude | int | 0 | 2147483647 | | -| location-city | | ranked | | | 100000 | -| location-state | | ranked | | | 100000 | +| Field | Type | Min | Max | Size | +|----------------|--------|-----|------------|--------| +| name | ranked | | | 100000 | +| favorite_foods | ranked | | | 100000 | +| default | ranked | | | 100000 | +| age | int | 0 | 2147483647 | | +| location | ranked | | | 1000 | +| latitude | int | 0 | 2147483647 | | +| longitude | int | 0 | 2147483647 | | +| location-city | ranked | | | 100000 | +| location-state | ranked | | | 100000 | -All frames are created as ranked frames by default, with the cache size listed above. Fields are created with -a minimum size of zero and a fixed maximum of 2147483647. Fields at the top level -are created in the default frame. Frames are a dash-separated concatenation of -all key values in the path - you can see this with frames like location-city. +All set fields are created as ranked fields by default, with the cache size +listed above. Integer fields are created with a minimum size of zero and a +fixed maximum of 2147483647. Field names are a dash-separated concatenation of +all key values in the path - you can see this with fields like location-city. Most of the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts, kafka topics, kafka group, etc.), but there are a few options that give some control over the way data is indexed, and ingestion performance. -* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. -* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state". +* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per field*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. +* `--framer.collapse`: This is a list of strings which will be removed from the field names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be fields named "city" and "state" rather than "location-city" and "location-state". * `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it. * `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used. * `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb. diff --git a/docs/query-language.md b/docs/query-language.md index 62c912ba4..2ea85a06e 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -58,7 +58,7 @@ curl localhost:10101/index/repository/query \ **Spec:** ``` -Set(, field=, [TIMESTAMP]) +Set(, =, [TIMESTAMP]) ``` **Description:** @@ -202,12 +202,12 @@ SetColumnAttrs(10, url=null) {"results":[null]} ``` -#### ClearBit +#### Clear **Spec:** ``` -Clear(, field=) +Clear(, =) ``` **Description:** @@ -241,7 +241,7 @@ This represents removing the relationship between the user with id=1 and the rep **Spec:** ``` -Row(field=) +Row(=) ``` **Description:** @@ -425,7 +425,7 @@ TopN([ROW_CALL], , [n=UINT], **Description:** -Return the id and count of the top `n` bitmaps (by count of bits) in the field. +Return the id and count of the top `n` rows (by count of bits) in the field. The `attrName` and `attrValues` arguments work together to only return rows which have the attribute specified by `attrName` with one of the values specified in `attrValues`. @@ -434,11 +434,11 @@ have the attribute specified by `attrName` with one of the values specified in **Caveats:** -* Performing a TopN() query on a field with cache type ranked will return the top bitmaps sorted by count in descending order. -* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return bitmaps sorted in order of most recently set bit. -* The field's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. -* Once full, the cache will truncate the set of bitmaps according to the field option CacheSize. Bitmaps that straddle the limit and have the same count will be truncated in no particular order. -* The TopN query's attribute filter is applied to the existing sorted cache of bitmaps. Bitmaps that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored. +* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order. +* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit. +* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. +* Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order. +* The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored. See [field creation](../api-reference/#create-field) for more information about the cache. @@ -466,7 +466,7 @@ TopN(stargazer, n=2) * Results are the top two rows (users) sorted by number of bits set (repositories they've starred) in descending order. -Filter based on an existing Bitmap: +Filter based on an existing row: ```request TopN(Row(language=1), stargazer, n=2) ``` @@ -491,7 +491,7 @@ TopN(stargazer, n=2, attrName=active, attrValues=[true]) **Spec:** ``` -Range(field=, , ) +Range(=, , ) ``` **Description:** @@ -632,7 +632,7 @@ Sum([ROW_CALL], field=) Returns the count and computed sum of all BSI integer values in the `field`. If the optional `Row` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. -**Result Type:** object with the computed sum and count of the bitmap field. +**Result Type:** object with the computed sum and count of the values in the integer field. **Examples:** From 7873126720ff6b125796b56e44cc1c40cb74a831 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 18:39:25 -0500 Subject: [PATCH 023/166] handle errors a bit better in handlePostQuery --- http/handler.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/http/handler.go b/http/handler.go index 7786eb2f3..8cc52a9b9 100644 --- a/http/handler.go +++ b/http/handler.go @@ -412,18 +412,25 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.API.Query(r.Context(), req) if err != nil { - w.WriteHeader(http.StatusBadRequest) + switch errors.Cause(resp.Err) { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusBadRequest) + } h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) return } - // Set appropriate status code, if there is an error. + // Set appropriate status code, if there is an error. It doesn't appear that + // resp.Err could ever be set in API.Query, so this code block is probably + // doing nothing right now. if resp.Err != nil { - switch resp.Err { + switch errors.Cause(resp.Err) { case pilosa.ErrTooManyWrites: w.WriteHeader(http.StatusRequestEntityTooLarge) default: - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(http.StatusBadRequest) } } From 9305712237935229f562af2c034a9c192b5486df Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 18:02:31 -0500 Subject: [PATCH 024/166] add options.keys and lowercase names to json output --- field.go | 14 ++++++++++---- index.go | 15 +++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/field.go b/field.go index e1eed501d..4cf1861df 100644 --- a/field.go +++ b/field.go @@ -1094,9 +1094,9 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { func (f *Field) MarshalJSON() ([]byte, error) { thing := struct { - Name string - Options FieldOptions - Views []*ViewInfo + Name string `json:"name"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views"` }{ Name: f.Name(), Options: f.Options(), @@ -1134,7 +1134,7 @@ type FieldOptions struct { Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Keys bool `json:"keys,omitempty"` + Keys bool `json:"keys"` } // applyDefaultOptions returns a new FieldOptions object @@ -1177,28 +1177,34 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { Type string `json:"type"` CacheType string `json:"cacheType"` CacheSize uint32 `json:"cacheSize"` + Keys bool `json:"keys"` }{ o.Type, o.CacheType, o.CacheSize, + o.Keys, }) case FieldTypeInt: return json.Marshal(struct { Type string `json:"type"` Min int64 `json:"min"` Max int64 `json:"max"` + Keys bool `json:"keys"` }{ o.Type, o.Min, o.Max, + o.Keys, }) case FieldTypeTime: return json.Marshal(struct { Type string `json:"type"` TimeQuantum TimeQuantum `json:"timeQuantum"` + Keys bool `json:"keys"` }{ o.Type, o.TimeQuantum, + o.Keys, }) } return nil, errors.New("invalid field type") diff --git a/index.go b/index.go index 9d27f4174..0f6d222be 100644 --- a/index.go +++ b/index.go @@ -83,11 +83,13 @@ func (i *Index) MarshalJSON() ([]byte, error) { fields = append(fields, f) } thing := struct { - Name string - Fields []*Field + Name string `json:"name"` + Options IndexOptions `json:"options"` + Fields []*Field `json:"fields"` }{ - Name: i.name, - Fields: fields, + Name: i.name, + Options: i.Options(), + Fields: fields, } return json.Marshal(thing) } @@ -422,8 +424,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { - Name string `json:"name"` - Fields []*FieldInfo `json:"fields"` + Name string `json:"name"` + Options IndexOptions `json:"options"` + Fields []*FieldInfo `json:"fields"` } type indexInfoSlice []*IndexInfo From c10cdc9d222f1f045f822108386bb86d6cbfb147 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 19:15:34 -0500 Subject: [PATCH 025/166] exclude views from http schema output --- api.go | 6 +++--- holder.go | 16 ++++++++++++++++ http/handler.go | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/api.go b/api.go index 7a1a6576a..8fe028675 100644 --- a/api.go +++ b/api.go @@ -482,9 +482,9 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { } // Schema returns information about each index in Pilosa including which fields -// and views they contain. -func (api *API) Schema(ctx context.Context) []*Index { - return api.holder.Indexes() +// they contain. +func (api *API) Schema(ctx context.Context) []*IndexInfo { + return api.holder.limitedSchema() } // Views returns the views in the given field. diff --git a/holder.go b/holder.go index 2481f4768..192c83a46 100644 --- a/holder.go +++ b/holder.go @@ -228,6 +228,22 @@ func (h *Holder) Schema() []*IndexInfo { return a } +// limitedSchema returns schema information for all indexes and fields. +func (h *Holder) limitedSchema() []*IndexInfo { + var a []*IndexInfo + for _, index := range h.Indexes() { + di := &IndexInfo{Name: index.Name()} + for _, field := range index.Fields() { + fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + di.Fields = append(di.Fields, fi) + } + sort.Sort(fieldInfoSlice(di.Fields)) + a = append(a, di) + } + sort.Sort(indexInfoSlice(a)) + return a +} + // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. diff --git a/http/handler.go b/http/handler.go index 7786eb2f3..6dce53bc2 100644 --- a/http/handler.go +++ b/http/handler.go @@ -463,7 +463,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { } indexName := mux.Vars(r)["index"] for _, idx := range h.API.Schema(r.Context()) { - if idx.Name() == indexName { + if idx.Name == indexName { if err := json.NewEncoder(w).Encode(idx); err != nil { h.Logger.Printf("write response error: %s", err) } From bf2b4e9284d3e9724d8c1f79fff05f4f1b40b9f4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 20:50:00 -0500 Subject: [PATCH 026/166] remove index and field MarshalJSON --- field.go | 15 --------------- index.go | 19 ------------------- 2 files changed, 34 deletions(-) diff --git a/field.go b/field.go index 4cf1861df..bcdb800f8 100644 --- a/field.go +++ b/field.go @@ -1092,21 +1092,6 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { return nil } -func (f *Field) MarshalJSON() ([]byte, error) { - thing := struct { - Name string `json:"name"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views"` - }{ - Name: f.Name(), - Options: f.Options(), - } - for _, viewname := range f.viewNames() { - thing.Views = append(thing.Views, &ViewInfo{Name: viewname}) - } - return json.Marshal(thing) -} - type fieldSlice []*Field func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/index.go b/index.go index 0f6d222be..7217bb659 100644 --- a/index.go +++ b/index.go @@ -15,7 +15,6 @@ package pilosa import ( - "encoding/json" "fmt" "io/ioutil" "os" @@ -76,24 +75,6 @@ func NewIndex(path, name string) (*Index, error) { }, nil } -func (i *Index) MarshalJSON() ([]byte, error) { - fields := make([]*Field, 0, len(i.fields)) - for _, f := range i.fields { - - fields = append(fields, f) - } - thing := struct { - Name string `json:"name"` - Options IndexOptions `json:"options"` - Fields []*Field `json:"fields"` - }{ - Name: i.name, - Options: i.Options(), - Fields: fields, - } - return json.Marshal(thing) -} - // Name returns name of the index. func (i *Index) Name() string { return i.name } From 21bfa5df773e0d65a440a89e8f1ef8affbef5245 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 5 Jul 2018 20:58:25 -0500 Subject: [PATCH 027/166] remove Holder from API --- api.go | 4 ---- gossip/gossip.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/api.go b/api.go index 8fe028675..679ab46de 100644 --- a/api.go +++ b/api.go @@ -150,10 +150,6 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return resp, nil } -func (api *API) Holder() *Holder { - return api.server.Holder() -} - // readColumnAttrSets returns a list of column attribute objects by id. func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { if index == nil { diff --git a/gossip/gossip.go b/gossip/gossip.go index 7a11d01d4..849b33d01 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -249,7 +249,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte { m := &pilosa.NodeStatus{ Node: g.papi.Node(), MaxShards: g.papi.MaxShards(context.Background()), - Schema: &pilosa.Schema{Indexes: g.papi.Holder().Schema()}, + Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } // Marshal nodestate data to bytes. From 6c3d8da35b360ff5e34d8143e351ae99cab0f9f8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 5 Jul 2018 21:52:07 -0500 Subject: [PATCH 028/166] update the tutorials for 1.0 --- docs/tutorials.md | 166 +++++++++++++++++++++++++++------------------- 1 file changed, 97 insertions(+), 69 deletions(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 06cdab286..e55a2094a 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -407,37 +407,65 @@ curl localhost:10101/index/patients \ -X POST ``` ``` response -{} +{"success":true} ``` -In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. +In addition to storing rows of bits, a frame can also contain fields that store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. ``` request -curl localhost:10101/index/patients/frame/measurements \ +curl localhost:10101/index/patients/field/age \ -X POST \ - -d '{"options":{ - "fields": [ - {"name": "age", "type": "int", "min": 0, "max": 120}, - {"name": "weight", "type": "int", "min": 0, "max": 500}, - {"name": "tcells", "type": "int", "min": 0, "max": 2000} - ] - }}' + -d '{"options":{"type": "int", "min": 0, "max": 120}}' ``` ``` response -{} +{"success":true} ``` -If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). +``` request +curl localhost:10101/index/patients/field/weight \ + -X POST \ + -d '{"options":{"type": "int", "min": 0, "max": 500}}' +``` +``` response +{"success":true} +``` + +``` request +curl localhost:10101/index/patients/field/tcells \ + -X POST \ + -d '{"options":{"type": "int", "min": 0, "max": 2000}}' +``` +``` response +{"success":true} +``` Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. -This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: +The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)' + -d 'Set(1, age=34)' ``` ``` response -{"results":[null]} +{"results":[true]} +``` + +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Set(1, weight=128)' +``` +``` response +{"results":[true]} +``` + +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Set(1, tcells=1145)' +``` +``` response +{"results":[true]} ``` In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. @@ -454,7 +482,7 @@ Assuming we have a file called `ages.csv` that is structured like this: 8,33 9,63 ``` -where the first column of the CSV represents the patient `ID` and the second column represents the patient's`age`, then we can import the data into our `age` field by running this command: +where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command: ``` pilosa import -i patients -f measurements --field age ages.csv ``` @@ -465,10 +493,10 @@ In order to find all patients over the age of 40, then simply run a `Range` quer ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Range(frame="measurements", age > 40)' + -d 'Range(age > 40)' ``` ``` response -{"results":[{"attrs":{},"bits":[2,6,9]}]} +{"results":[{"attrs":{},"columns":[2,6,9]}]} ``` You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. @@ -477,21 +505,21 @@ To find the average age of all patients, run a `Sum` query: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Sum(frame="measurements", field="age")' + -d 'Sum(field="age")' ``` ``` response -{"results":[{"sum":377,"count":9}]} +{"results":[{"value":377,"count":9}]} ``` -The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. +The results you get from the `Sum` query contain the sum of all values as well as the `count` of columns with a value. To get the average you can just divide `value` by `count`. You can also provide a filter to the `Sum()` function to find the average age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' + -d 'Sum(Range(age > 40), field="age")' ``` ``` response -{"results":[{"sum":191,"count":3}]} +{"results":[{"value":191,"count":3}]} ``` Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. @@ -499,42 +527,42 @@ To find the minimum age of all patients, run a `Min` query: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Min(frame="measurements", field="age")' + -d 'Min(field="age")' ``` ``` response -{"results":[{"min":19,"count":1}]} +{"results":[{"value":19,"count":1}]} ``` -The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value. +The results you get from the `Min` query contain the minimum `value` of all values as well as the `count` of columns with that value. You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")' + -d 'Min(Range(age > 40), field="age")' ``` ``` response -{"results":[{"min":57,"count":1}]} +{"results":[{"value":57,"count":1}]} ``` To find the maximum age of all patients, run a `Max` query: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Max(frame="measurements", field="age")' + -d 'Max(field="age")' ``` ``` response -{"results":[{"max":71,"count":1}]} +{"results":[{"value":71,"count":1}]} ``` -The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value. +The results you get from the `Max` query contain the maximum `value` of all values as well as the `count` of columns with that value. You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")' + -d 'Max(Range(age < 40), field="age")' ``` ``` response -{"results":[{"max":34,"count":1}]} +{"results":[{"value":34,"count":1}]} ``` ### Storing Row and Column Attributes @@ -549,28 +577,28 @@ curl localhost:10101/index/books \ -X POST ``` ``` response -{} +{"success":true} ``` -Next, create a frame in the `books` index called `members` which will represent library members who have read books. +Next, create a field in the `books` index called `members` which will represent library members who have read books. ``` request -curl localhost:10101/index/books/frame/members \ +curl localhost:10101/index/books/field/members \ -X POST \ -d '{}' ``` ``` response -{} +{"success":true} ``` Now, let's add some books to our index. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960) - SetColumnAttrs(col=2, name="No Name in the Street", year=1972) - SetColumnAttrs(col=3, name="The Tipping Point", year=2000) - SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003) - SetColumnAttrs(col=5, name="The Forever War", year=2008)' + -d 'SetColumnAttrs(1, name="To Kill a Mockingbird", year=1960) + SetColumnAttrs(2, name="No Name in the Street", year=1972) + SetColumnAttrs(3, name="The Tipping Point", year=2000) + SetColumnAttrs(4, name="Out Stealing Horses", year=2003) + SetColumnAttrs(5, name="The Forever War", year=2008)' ``` ``` response {"results":[null,null,null,null,null]} @@ -580,11 +608,11 @@ And add some members. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith") - SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins") - SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks") - SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez") - SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")' + -d 'SetRowAttrs(members, 10001, fullName="John Smith") + SetRowAttrs(members, 10002, fullName="Sue Perkins") + SetRowAttrs(members, 10003, fullName="Jennifer Hawks") + SetRowAttrs(members, 10004, fullName="Pedro Vazquez") + SetRowAttrs(members, 10005, fullName="Pat Washington")' ``` ``` response {"results":[null,null,null,null,null]} @@ -594,29 +622,29 @@ At this point we can query one of the `member` records by querying that row. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", row=10002)' + -d 'Row(members=10002)' ``` ``` response -{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} +{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[]}]} ``` Now let's add some data to the matrix such that each pair represents a member who has read that book. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetBit(frame="members", row=10001, col=3) - SetBit(frame="members", row=10001, col=5) - SetBit(frame="members", row=10002, col=1) - SetBit(frame="members", row=10002, col=2) - SetBit(frame="members", row=10002, col=4) - SetBit(frame="members", row=10003, col=3) - SetBit(frame="members", row=10004, col=4) - SetBit(frame="members", row=10004, col=5) - SetBit(frame="members", row=10005, col=1) - SetBit(frame="members", row=10005, col=2) - SetBit(frame="members", row=10005, col=3) - SetBit(frame="members", row=10005, col=4) - SetBit(frame="members", row=10005, col=5)' + -d 'Set(3, members=10001) + Set(5, members=10001) + Set(1, members=10002) + Set(2, members=10002) + Set(4, members=10002) + Set(3, members=10003) + Set(4, members=10004) + Set(5, members=10004) + Set(1, members=10005) + Set(2, members=10005) + Set(3, members=10005) + Set(4, members=10005) + Set(5, members=10005)' ``` ``` response {"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} @@ -626,22 +654,22 @@ Now pull the record for `Sue Perkins` again. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", row=10002)' + -d 'Row(members=10002)' ``` ``` response -{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} +{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}]} ``` -Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. +Notice that the result set now contains a list of integers in the `columns` attribute. These integers match the column IDs of the books that Sue has read. In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Bitmap(frame="members", row=10002)' + -d 'Row(members=10002)' ``` ``` response { - "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], + "results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}], "columnAttrs":[ {"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}}, {"id":2,"attrs":{"name":"No Name in the Street","year":1972}}, @@ -655,11 +683,11 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`, ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))' + -d 'Intersect(Row(members=10002), Row(members=10004))' ``` ``` response { - "results":[{"attrs":{},"bits":[4]}], + "results":[{"attrs":{},"columns":[4]}], "columnAttrs":[ {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}} ] From b4011778dd03aaf91c43e057928d4531368cc987 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:27:59 -0500 Subject: [PATCH 029/166] Unexport APIOption, b.BTCIterator, API.Holder, API.Serializer, APIOption, http.Handler.API --- api.go | 8 ++-- enterprise/b/containers_btree.go | 8 ++-- http/handler.go | 76 ++++++++++++++++---------------- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/api.go b/api.go index 679ab46de..2a20c6234 100644 --- a/api.go +++ b/api.go @@ -40,10 +40,10 @@ type API struct { Serializer Serializer } -// APIOption is a functional option type for pilosa.API -type APIOption func(*API) error +// apiOption is a functional option type for pilosa.API +type apiOption func(*API) error -func OptAPIServer(s *Server) APIOption { +func OptAPIServer(s *Server) apiOption { return func(a *API) error { a.server = s a.holder = s.holder @@ -54,7 +54,7 @@ func OptAPIServer(s *Server) APIOption { } // NewAPI returns a new API instance. -func NewAPI(opts ...APIOption) (*API, error) { +func NewAPI(opts ...apiOption) (*API, error) { api := &API{} for _, opt := range opts { diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 7c208b1db..71727700f 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -172,18 +172,18 @@ func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterato found = true } - return &BTCIterator{ + return &btcIterator{ e: e, }, found } -type BTCIterator struct { +type btcIterator struct { e *Enumerator key uint64 val *roaring.Container } -func (i *BTCIterator) Next() bool { +func (i *btcIterator) Next() bool { k, v, err := i.e.Next() if err == io.EOF { @@ -194,7 +194,7 @@ func (i *BTCIterator) Next() bool { return true } -func (i *BTCIterator) Value() (uint64, *roaring.Container) { +func (i *btcIterator) Value() (uint64, *roaring.Container) { if i.val == nil { return 0, nil } diff --git a/http/handler.go b/http/handler.go index a221b1445..0693a4d17 100644 --- a/http/handler.go +++ b/http/handler.go @@ -49,7 +49,7 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec - API *pilosa.API + api *pilosa.API AllowedOrigins []string @@ -90,7 +90,7 @@ func OptHandlerAllowedOrigins(origins []string) HandlerOption { func OptHandlerAPI(api *pilosa.API) HandlerOption { return func(h *Handler) error { - h.API = api + h.api = api return nil } } @@ -124,7 +124,7 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) { } } - if handler.API == nil { + if handler.api == nil { return nil, errors.New("must pass OptHandlerAPI") } @@ -252,7 +252,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Calculate per request StatsD metrics when the handler is fully configured. statsTags := make([]string, 0, 3) - longQueryTime := h.API.LongQueryTime() + longQueryTime := h.api.LongQueryTime() if longQueryTime > 0 && dif > longQueryTime { h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) statsTags = append(statsTags, "slow_query") @@ -267,7 +267,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // useragent tag identifies internal/external endpoints statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) + stats := h.api.StatsWithTags(statsTags) if stats != nil { stats.Histogram("http."+endpointName, float64(dif), 0.1) } @@ -355,7 +355,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { return } - schema := h.API.Schema(r.Context()) + schema := h.api.Schema(r.Context()) if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { h.Logger.Printf("write schema response error: %s", err) } @@ -368,9 +368,9 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { return } status := getStatusResponse{ - State: h.API.State(), - Nodes: h.API.Hosts(r.Context()), - LocalID: h.API.Node().ID, + State: h.api.State(), + Nodes: h.api.Hosts(r.Context()), + LocalID: h.api.Node().ID, } if err := json.NewEncoder(w).Encode(status); err != nil { h.Logger.Printf("write status response error: %s", err) @@ -382,7 +382,7 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - info := h.API.Info() + info := h.api.Info() if err := json.NewEncoder(w).Encode(info); err != nil { h.Logger.Printf("write info response error: %s", err) } @@ -410,7 +410,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // TODO: Remove req.Index = mux.Vars(r)["index"] - resp, err := h.API.Query(r.Context(), req) + resp, err := h.api.Query(r.Context(), req) if err != nil { switch errors.Cause(resp.Err) { case pilosa.ErrTooManyWrites: @@ -447,7 +447,7 @@ func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) { return } if err := json.NewEncoder(w).Encode(getShardsMaxResponse{ - Standard: h.API.MaxShards(r.Context()), + Standard: h.api.MaxShards(r.Context()), }); err != nil { h.Logger.Printf("write shards-max response error: %s", err) } @@ -469,7 +469,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { return } indexName := mux.Vars(r)["index"] - for _, idx := range h.API.Schema(r.Context()) { + for _, idx := range h.api.Schema(r.Context()) { if idx.Name == indexName { if err := json.NewEncoder(w).Encode(idx); err != nil { h.Logger.Printf("write response error: %s", err) @@ -563,7 +563,7 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] resp := successResponse{} - err := h.API.DeleteIndex(r.Context(), indexName) + err := h.api.DeleteIndex(r.Context(), indexName) resp.write(w, err) } @@ -584,7 +584,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp.write(w, err) return } - _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) + _, err = h.api.CreateIndex(r.Context(), indexName, req.Options) resp.write(w, err) } @@ -604,7 +604,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request return } - attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) + attrs, err := h.api.IndexAttrDiff(r.Context(), indexName, req.Blocks) if err != nil { if errors.Cause(err) == pilosa.ErrIndexNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -673,7 +673,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { } } - _, err = h.API.CreateField(r.Context(), indexName, fieldName, fos...) + _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) resp.write(w, err) } @@ -760,7 +760,7 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { fieldName := mux.Vars(r)["field"] resp := successResponse{} - err := h.API.DeleteField(r.Context(), indexName, fieldName) + err := h.api.DeleteField(r.Context(), indexName, fieldName) resp.write(w, err) } @@ -780,7 +780,7 @@ func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request return } - attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) + attrs, err := h.api.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) if err != nil { switch errors.Cause(err) { case pilosa.ErrFragmentNotFound: @@ -826,7 +826,7 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques } qreq := &pilosa.QueryRequest{} - err = h.API.Serializer.Unmarshal(body, qreq) + err = h.api.Serializer.Unmarshal(body, qreq) if err != nil { return nil, errors.Wrap(err, "unmarshalling query request") } @@ -869,7 +869,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res // writeProtobufQueryResponse writes the response from the executor to w as protobuf. func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { - if buf, err := h.API.Serializer.Marshal(resp); err != nil { + if buf, err := h.api.Serializer.Marshal(resp); err != nil { return errors.Wrap(err, "marshalling") } else if _, err := w.Write(buf); err != nil { return errors.Wrap(err, "writing") @@ -897,7 +897,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Get index and field type to determine how to handle the // import data. - field, err := h.API.Field(r.Context(), indexName, fieldName) + field, err := h.api.Field(r.Context(), indexName, fieldName) if err != nil { switch errors.Cause(err) { case pilosa.ErrIndexNotFound: @@ -922,12 +922,12 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Field type: Int // Marshal into request object. req := &pilosa.ImportValueRequest{} - if err := h.API.Serializer.Unmarshal(body, req); err != nil { + if err := h.api.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - if err := h.API.ImportValue(r.Context(), req); err != nil { + if err := h.api.ImportValue(r.Context(), req); err != nil { switch errors.Cause(err) { case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) @@ -940,12 +940,12 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Field type: Set, Time // Marshal into request object. req := &pilosa.ImportRequest{} - if err := h.API.Serializer.Unmarshal(body, req); err != nil { + if err := h.api.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - if err := h.API.Import(r.Context(), req); err != nil { + if err := h.api.Import(r.Context(), req); err != nil { switch errors.Cause(err) { case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) @@ -957,7 +957,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Marshal response object. - buf, e := h.API.Serializer.Marshal(&pilosa.ImportResponse{Err: ""}) + buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: ""}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) return @@ -988,7 +988,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - if err = h.API.ExportCSV(r.Context(), index, field, shard, w); err != nil { + if err = h.api.ExportCSV(r.Context(), index, field, shard, w); err != nil { switch errors.Cause(err) { case pilosa.ErrFragmentNotFound: break @@ -1018,7 +1018,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes, err := h.API.ShardNodes(r.Context(), index, shard) + nodes, err := h.api.ShardNodes(r.Context(), index, shard) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1032,7 +1032,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) // handleGetFragmentBlockData handles GET /internal/fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - buf, err := h.API.FragmentBlockData(r.Context(), r.Body) + buf, err := h.api.FragmentBlockData(r.Context(), r.Body) if err != nil { if _, ok := err.(pilosa.BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1064,7 +1064,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard) + blocks, err := h.api.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard) if err != nil { if errors.Cause(err) == pilosa.ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -1095,7 +1095,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { err := json.NewEncoder(w).Encode(struct { Version string `json:"version"` }{ - Version: h.API.Version(), + Version: h.api.Version(), }) if err != nil { h.Logger.Printf("write version response error: %s", err) @@ -1152,7 +1152,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r return } - oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + oldNode, newNode, err := h.api.SetCoordinator(r.Context(), req.ID) if err != nil { if errors.Cause(err) == pilosa.ErrNodeIDNotExists { http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) @@ -1193,7 +1193,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht return } - removeNode, err := h.API.RemoveNode(req.ID) + removeNode, err := h.api.RemoveNode(req.ID) if err != nil { if errors.Cause(err) == pilosa.ErrNodeIDNotExists { http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) @@ -1225,7 +1225,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - err := h.API.ResizeAbort() + err := h.api.ResizeAbort() var msg string if err != nil { switch errors.Cause(err) { @@ -1252,7 +1252,7 @@ type clusterResizeAbortResponse struct { } func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.API.RecalculateCaches(r.Context()) + err := h.api.RecalculateCaches(r.Context()) if err != nil { http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) return @@ -1272,7 +1272,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - err := h.API.ClusterMessage(r.Context(), r.Body) + err := h.api.ClusterMessage(r.Context(), r.Body) if err != nil { // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) @@ -1291,7 +1291,7 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) pipeR, pipeW := io.Pipe() - err := h.API.GetTranslateData(r.Context(), pipeW, offset) + err := h.api.GetTranslateData(r.Context(), pipeW, offset) if err != nil { if errors.Cause(err) == pilosa.ErrNotImplemented { From 71000ee6d0b5c090c6da26052a799b2c4b3f8114 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:38:55 -0500 Subject: [PATCH 030/166] Unexport ApiMethodNotAllowedError --- pilosa.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pilosa.go b/pilosa.go index 9615e88b8..9b5918ee2 100644 --- a/pilosa.go +++ b/pilosa.go @@ -63,15 +63,15 @@ var ( ErrNotImplemented = errors.New("not implemented") ) -// ApiMethodNotAllowedError wraps an error value indicating that a particular +// apiMethodNotAllowedError wraps an error value indicating that a particular // API method is not allowed in the current cluster state. -type ApiMethodNotAllowedError struct { +type apiMethodNotAllowedError struct { error } // NewApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. -func NewApiMethodNotAllowedError(err error) ApiMethodNotAllowedError { - return ApiMethodNotAllowedError{err} +func NewApiMethodNotAllowedError(err error) apiMethodNotAllowedError { + return apiMethodNotAllowedError{err} } // BadRequestError wraps an error value to signify that a request could not be From 7559382115899c1338e78530790de704a32549b4 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:39:01 -0500 Subject: [PATCH 031/166] Unexport AttrBlocks --- api.go | 4 ++-- attr.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index 2a20c6234..0b70397e7 100644 --- a/api.go +++ b/api.go @@ -554,7 +554,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At // Read all attributes from all mismatched blocks. attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + for _, blockID := range attrBlocks(localBlocks).Diff(blocks) { // Retrieve block data. m, err := index.ColumnAttrStore().BlockData(blockID) if err != nil { @@ -588,7 +588,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s // Read all attributes from all mismatched blocks. attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + for _, blockID := range attrBlocks(localBlocks).Diff(blocks) { // Retrieve block data. m, err := f.RowAttrStore().BlockData(blockID) if err != nil { diff --git a/attr.go b/attr.go index 628613172..0f553be95 100644 --- a/attr.go +++ b/attr.go @@ -82,12 +82,12 @@ type AttrBlock struct { Checksum []byte `json:"checksum"` } -// AttrBlocks represents a list of blocks. -type AttrBlocks []AttrBlock +// attrBlocks represents a list of blocks. +type attrBlocks []AttrBlock // Diff returns a list of block ids that are different or are new in other. // Block lists must be in sorted order. -func (a AttrBlocks) Diff(other []AttrBlock) []uint64 { +func (a attrBlocks) Diff(other []AttrBlock) []uint64 { var ids []uint64 for { // Read next block from each list. From 9fd8cdb00719397a598f04da15a995e7e095b941 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:42:06 -0500 Subject: [PATCH 032/166] Unexport DefaultMapSize --- translate.go | 2 +- translate_mapsize_all64bitsystems.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.go b/translate.go index 660269ea0..d58586fc6 100644 --- a/translate.go +++ b/translate.go @@ -84,7 +84,7 @@ func NewTranslateFile() *TranslateFile { cols: make(map[string]*index), rows: make(map[frameKey]*index), - MapSize: DefaultMapSize, + MapSize: defaultMapSize, ReplicationRetryInterval: defaultReplicationRetryInterval, } diff --git a/translate_mapsize_all64bitsystems.go b/translate_mapsize_all64bitsystems.go index 5275c6ec5..605d6270a 100644 --- a/translate_mapsize_all64bitsystems.go +++ b/translate_mapsize_all64bitsystems.go @@ -2,7 +2,7 @@ package pilosa -// DefaultMapSize is the default size of mapped memory for the translate store. +// defaultMapSize is the default size of mapped memory for the translate store. // It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on // 64bit systems. -const DefaultMapSize = 10 * (1 << 30) // 10GB +const defaultMapSize = 10 * (1 << 30) // 10GB From c102143b501fefa19e5cc8be58c12ff1cc653445 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:42:12 -0500 Subject: [PATCH 033/166] Unexport DefaultPartitionN --- cluster.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster.go b/cluster.go index f61173c3b..45bbb835d 100644 --- a/cluster.go +++ b/cluster.go @@ -36,8 +36,8 @@ import ( ) const ( - // DefaultPartitionN is the default number of partitions in a cluster. - DefaultPartitionN = 256 + // defaultPartitionN is the default number of partitions in a cluster. + defaultPartitionN = 256 // ClusterState represents the state returned in the /status endpoint. ClusterStateStarting = "STARTING" @@ -219,7 +219,7 @@ type cluster struct { func newCluster() *cluster { return &cluster{ Hasher: &jmphasher{}, - partitionN: DefaultPartitionN, + partitionN: defaultPartitionN, ReplicaN: 1, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel From 761f6878fb950091b5803093fe192a6c0b69c5fd Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:42:18 -0500 Subject: [PATCH 034/166] Unexport DefaultURI --- pilosa.go | 2 +- uri.go | 6 +++--- uri_internal_test.go | 12 ++++++------ utils_internal_test.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pilosa.go b/pilosa.go index 9b5918ee2..a63e64f76 100644 --- a/pilosa.go +++ b/pilosa.go @@ -159,7 +159,7 @@ func stringSlicesAreEqual(a, b []string) bool { // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { if addr == "" { - return DefaultURI(), nil + return defaultURI(), nil } else { return NewURIFromAddress(addr) } diff --git a/uri.go b/uri.go index 8577c8238..332166b4e 100644 --- a/uri.go +++ b/uri.go @@ -47,8 +47,8 @@ type URI struct { Port uint16 `json:"port"` } -// DefaultURI creates and returns the default URI. -func DefaultURI() *URI { +// defaultURI creates and returns the default URI. +func defaultURI() *URI { return &URI{ Scheme: "http", Host: "localhost", @@ -68,7 +68,7 @@ func (u URIs) HostPortStrings() []string { // NewURIFromHostPort returns a URI with specified host and port. func NewURIFromHostPort(host string, port uint16) (*URI, error) { - uri := DefaultURI() + uri := defaultURI() err := uri.SetHost(host) if err != nil { return nil, errors.Wrap(err, "setting uri host") diff --git a/uri_internal_test.go b/uri_internal_test.go index 3c9631661..64db3fd8e 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -17,7 +17,7 @@ package pilosa import "testing" func TestDefaultURI(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() compare(t, uri, "http", "localhost", 10101) } @@ -77,7 +77,7 @@ func TestURIPath(t *testing.T) { } func TestSetScheme(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() target := "fun" err := uri.SetScheme(target) if err != nil { @@ -89,7 +89,7 @@ func TestSetScheme(t *testing.T) { } func TestSetHost(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() target := "10.20.30.40" err := uri.SetHost(target) if err != nil { @@ -101,7 +101,7 @@ func TestSetHost(t *testing.T) { } func TestSetPort(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() target := uint16(9999) uri.SetPort(target) if uri.Port != target { @@ -110,7 +110,7 @@ func TestSetPort(t *testing.T) { } func TestSetInvalidScheme(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() err := uri.SetScheme("?invalid") if err == nil { t.Fatalf("Should have failed") @@ -118,7 +118,7 @@ func TestSetInvalidScheme(t *testing.T) { } func TestSetInvalidHost(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() err := uri.SetHost("index?.pilosa.com") if err == nil { t.Fatalf("Should have failed") diff --git a/utils_internal_test.go b/utils_internal_test.go index df1b7d2e0..7adb110e8 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -55,7 +55,7 @@ func NewTestCluster(n int) *cluster { // NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { - uri := DefaultURI() + uri := defaultURI() uri.SetScheme(scheme) uri.SetHost(host) uri.SetPort(port) @@ -63,7 +63,7 @@ func NewTestURI(scheme, host string, port uint16) URI { } func NewTestURIFromHostPort(host string, port uint16) URI { - uri := DefaultURI() + uri := defaultURI() uri.SetHost(host) uri.SetPort(port) return *uri From 7245a936768c99f8607f0b91145b717005d341fb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:43:21 -0500 Subject: [PATCH 035/166] Unexport DiagnosticsCollector --- diagnostics.go | 28 ++++++++++++++-------------- server.go | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 6ad6e16b1..8bdcdef3c 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -37,8 +37,8 @@ type versionResponse struct { Message string `json:"message"` } -// DiagnosticsCollector represents a collector/sender of diagnostics data. -type DiagnosticsCollector struct { +// diagnosticsCollector represents a collector/sender of diagnostics data. +type diagnosticsCollector struct { mu sync.Mutex host string VersionURL string @@ -57,8 +57,8 @@ type DiagnosticsCollector struct { } // NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". -func NewDiagnosticsCollector(host string) *DiagnosticsCollector { - return &DiagnosticsCollector{ +func NewDiagnosticsCollector(host string) *diagnosticsCollector { + return &diagnosticsCollector{ host: host, VersionURL: defaultVersionCheckURL, startTime: time.Now().Unix(), @@ -70,13 +70,13 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector { } // SetVersion of locally running Pilosa Cluster to check against master. -func (d *DiagnosticsCollector) SetVersion(v string) { +func (d *diagnosticsCollector) SetVersion(v string) { d.version = v d.Set("Version", v) } // Flush sends the current metrics. -func (d *DiagnosticsCollector) Flush() error { +func (d *diagnosticsCollector) Flush() error { d.mu.Lock() defer d.mu.Unlock() d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) @@ -99,7 +99,7 @@ func (d *DiagnosticsCollector) Flush() error { } // CheckVersion of the local build against Pilosa master. -func (d *DiagnosticsCollector) CheckVersion() error { +func (d *diagnosticsCollector) CheckVersion() error { var rsp versionResponse req, err := http.NewRequest("GET", d.VersionURL, nil) if err != nil { @@ -131,7 +131,7 @@ func (d *DiagnosticsCollector) CheckVersion() error { } // compareVersion check version strings. -func (d *DiagnosticsCollector) compareVersion(value string) error { +func (d *diagnosticsCollector) compareVersion(value string) error { currentVersion := versionSegments(value) localVersion := versionSegments(d.version) @@ -147,12 +147,12 @@ func (d *DiagnosticsCollector) compareVersion(value string) error { } // Encode metrics maps into the json message format. -func (d *DiagnosticsCollector) encode() ([]byte, error) { +func (d *diagnosticsCollector) encode() ([]byte, error) { return json.Marshal(d.metrics) } // Set adds a key value metric. -func (d *DiagnosticsCollector) Set(name string, value interface{}) { +func (d *diagnosticsCollector) Set(name string, value interface{}) { switch v := value.(type) { case string: if v == "" { @@ -166,7 +166,7 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) { } // logErr logs the error and returns true if an error exists -func (d *DiagnosticsCollector) logErr(err error) bool { +func (d *diagnosticsCollector) logErr(err error) bool { if err != nil { d.Logger.Printf("%v", err) return true @@ -175,7 +175,7 @@ func (d *DiagnosticsCollector) logErr(err error) bool { } // EnrichWithOSInfo adds OS information to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithOSInfo() { +func (d *diagnosticsCollector) EnrichWithOSInfo() { uptime, err := d.server.systemInfo.Uptime() if !d.logErr(err) { d.Set("HostUptime", uptime) @@ -199,7 +199,7 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() { } // EnrichWithMemoryInfo adds memory information to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { +func (d *diagnosticsCollector) EnrichWithMemoryInfo() { memFree, err := d.server.systemInfo.MemFree() if !d.logErr(err) { d.Set("MemFree", memFree) @@ -215,7 +215,7 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { } // EnrichWithSchemaProperties adds schema info to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { +func (d *diagnosticsCollector) EnrichWithSchemaProperties() { var numShards uint64 numFields := 0 numIndexes := 0 diff --git a/server.go b/server.go index 3adf3bb50..38c8d9426 100644 --- a/server.go +++ b/server.go @@ -50,7 +50,7 @@ type Server struct { holder *Holder cluster *cluster translateFile *TranslateFile - diagnostics *DiagnosticsCollector + diagnostics *diagnosticsCollector executor *executor hosts []string clusterDisabled bool From a8c9c30eef16285d925978974c7ae4d2b8d9cc03 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:43:28 -0500 Subject: [PATCH 036/166] Unexport ExpvarStatsClient --- stats.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/stats.go b/stats.go index 23c3e0bc8..130a91a64 100644 --- a/stats.go +++ b/stats.go @@ -82,8 +82,8 @@ func (c *nopStatsClient) SetLogger(logger Logger) func (c *nopStatsClient) Open() {} func (c *nopStatsClient) Close() error { return nil } -// ExpvarStatsClient writes stats out to expvars. -type ExpvarStatsClient struct { +// expvarStatsClient writes stats out to expvars. +type expvarStatsClient struct { mu sync.Mutex m *expvar.Map tags []string @@ -91,41 +91,41 @@ type ExpvarStatsClient struct { // NewExpvarStatsClient returns a new instance of ExpvarStatsClient. // This client points at the root of the expvar index map. -func NewExpvarStatsClient() *ExpvarStatsClient { - return &ExpvarStatsClient{ +func NewExpvarStatsClient() *expvarStatsClient { + return &expvarStatsClient{ m: Expvar, } } // Tags returns a sorted list of tags on the client. -func (c *ExpvarStatsClient) Tags() []string { +func (c *expvarStatsClient) Tags() []string { return nil } // WithTags returns a new client with additional tags appended. -func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient { +func (c *expvarStatsClient) WithTags(tags ...string) StatsClient { m := &expvar.Map{} m.Init() c.m.Set(strings.Join(tags, ","), m) - return &ExpvarStatsClient{ + return &expvarStatsClient{ m: m, tags: unionStringSlice(c.tags, tags), } } // Count tracks the number of times something occurs. -func (c *ExpvarStatsClient) Count(name string, value int64, rate float64) { +func (c *expvarStatsClient) Count(name string, value int64, rate float64) { c.m.Add(name, value) } // CountWithCustomTags Tracks the number of times something occurs per second with custom tags -func (c *ExpvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { +func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { c.m.Add(name, value) } // Gauge sets the value of a metric. -func (c *ExpvarStatsClient) Gauge(name string, value float64, rate float64) { +func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) { var f expvar.Float f.Set(value) c.m.Set(name, &f) @@ -133,19 +133,19 @@ func (c *ExpvarStatsClient) Gauge(name string, value float64, rate float64) { // Histogram tracks statistical distribution of a metric. // This works the same as gauge for this client. -func (c *ExpvarStatsClient) Histogram(name string, value float64, rate float64) { +func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) { c.Gauge(name, value, rate) } // Set tracks number of unique elements. -func (c *ExpvarStatsClient) Set(name string, value string, rate float64) { +func (c *expvarStatsClient) Set(name string, value string, rate float64) { var s expvar.String s.Set(value) c.m.Set(name, &s) } // Timing tracks timing information for a metric. -func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float64) { +func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) { c.mu.Lock() d, _ := c.m.Get(name).(time.Duration) c.m.Set(name, d+value) @@ -153,14 +153,14 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 } // SetLogger has no logger. -func (c *ExpvarStatsClient) SetLogger(logger Logger) { +func (c *expvarStatsClient) SetLogger(logger Logger) { } // Open no-op. -func (c *ExpvarStatsClient) Open() {} +func (c *expvarStatsClient) Open() {} // Close no-op. -func (c *ExpvarStatsClient) Close() error { return nil } +func (c *expvarStatsClient) Close() error { return nil } // MultiStatsClient joins multiple stats clients together. type MultiStatsClient []StatsClient From ba9c5193b559f607ebd2d18b581ad939eff68fce Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:43:34 -0500 Subject: [PATCH 037/166] Unexport Field.ImportValue --- api.go | 2 +- field.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 0b70397e7..5230c8677 100644 --- a/api.go +++ b/api.go @@ -643,7 +643,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error return errors.Wrap(err, "getting field") } // Import into fragment. - err = field.ImportValue(req.ColumnIDs, req.Values) + err = field.importValue(req.ColumnIDs, req.Values) if err != nil { api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } diff --git a/field.go b/field.go index bcdb800f8..b9d84cc83 100644 --- a/field.go +++ b/field.go @@ -1035,8 +1035,8 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return nil } -// ImportValue bulk imports range-encoded value data. -func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { +// importValue bulk imports range-encoded value data. +func (f *Field) importValue(columnIDs []uint64, values []int64) error { viewName := viewBSIGroupPrefix + f.name // Get the bsiGroup so we know bitDepth. bsig := f.bsiGroup(f.name) From 16461345f63b009a6fd008d048189ce2b270ca29 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:43:41 -0500 Subject: [PATCH 038/166] Unexport Field.Keys --- executor.go | 4 ++-- field.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index b39e26f58..9b7f5d4bb 100644 --- a/executor.go +++ b/executor.go @@ -1577,7 +1577,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { if field == nil { return ErrFieldNotFound } - if field.Keys() { + if field.keys() { if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { return errors.New("row value must be a string when field 'keys' option enabled") } @@ -1628,7 +1628,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field == nil { return nil, ErrFieldNotFound } - if field.Keys() { + if field.keys() { other := make([]Pair, len(result)) for i := range result { key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID) diff --git a/field.go b/field.go index b9d84cc83..69d08d504 100644 --- a/field.go +++ b/field.go @@ -428,8 +428,8 @@ func (f *Field) Close() error { return nil } -// Keys returns true if the field uses string keys. -func (f *Field) Keys() bool { +// keys returns true if the field uses string keys. +func (f *Field) keys() bool { f.mu.RLock() defer f.mu.RUnlock() return f.options.Keys From cbf821123ecf243d7c4abdf8694b80eccdd3c184 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:43:48 -0500 Subject: [PATCH 039/166] Unexport Field.MaxShard --- field.go | 4 ++-- index.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/field.go b/field.go index 69d08d504..02d7d9ce5 100644 --- a/field.go +++ b/field.go @@ -183,8 +183,8 @@ func (f *Field) Path() string { return f.path } // RowAttrStore returns the attribute storage. func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } -// MaxShard returns the max shard in the field. -func (f *Field) MaxShard() uint64 { +// maxShard returns the max shard in the field. +func (f *Field) maxShard() uint64 { f.mu.RLock() defer f.mu.RUnlock() diff --git a/index.go b/index.go index 7217bb659..4b8b84ca8 100644 --- a/index.go +++ b/index.go @@ -220,7 +220,7 @@ func (i *Index) maxShard() uint64 { max := i.remoteMaxShard for _, f := range i.fields { - if shard := f.MaxShard(); shard > max { + if shard := f.maxShard(); shard > max { max = shard } } From 55c0dcee23e3011d68216558d3407d3c072320bd Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:43:54 -0500 Subject: [PATCH 040/166] Unexport Field.RangeBetween --- field.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/field.go b/field.go index 02d7d9ce5..69c820b43 100644 --- a/field.go +++ b/field.go @@ -955,7 +955,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) return view.rangeOp(op, bsig.BitDepth(), baseValue) } -func (f *Field) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { +func (f *Field) rangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) if bsig == nil { From fc49d67c0bb4cc00fa43e1a7cc005425a0ccdc4a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:44:01 -0500 Subject: [PATCH 041/166] Unexport Field.RecalculateCaches --- field.go | 4 ++-- index.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/field.go b/field.go index 69c820b43..e57ca0f79 100644 --- a/field.go +++ b/field.go @@ -606,8 +606,8 @@ func (f *Field) viewNames() []string { return other } -// RecalculateCaches recalculates caches on every view in the field. -func (f *Field) RecalculateCaches() { +// recalculateCaches recalculates caches on every view in the field. +func (f *Field) recalculateCaches() { for _, view := range f.views() { view.recalculateCaches() } diff --git a/index.go b/index.go index 4b8b84ca8..fb97ded2f 100644 --- a/index.go +++ b/index.go @@ -265,7 +265,7 @@ func (i *Index) Fields() []*Field { // RecalculateCaches recalculates caches on every field in the index. func (i *Index) RecalculateCaches() { for _, field := range i.Fields() { - field.RecalculateCaches() + field.recalculateCaches() } } From 3faa51544afdcfcf9438cc744644ab8d34e6e50d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:44:07 -0500 Subject: [PATCH 042/166] Unexport Field.SetTimeQuantum --- field.go | 6 +++--- field_internal_test.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/field.go b/field.go index e57ca0f79..7906750b5 100644 --- a/field.go +++ b/field.go @@ -396,7 +396,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Max = 0 f.options.Keys = opt.Keys // Set the time quantum. - if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { + if err := f.setTimeQuantum(opt.TimeQuantum); err != nil { f.Close() return errors.Wrap(err, "setting time quantum") } @@ -533,8 +533,8 @@ func (f *Field) TimeQuantum() TimeQuantum { return f.options.TimeQuantum } -// SetTimeQuantum sets the time quantum for the field. -func (f *Field) SetTimeQuantum(q TimeQuantum) error { +// setTimeQuantum sets the time quantum for the field. +func (f *Field) setTimeQuantum(q TimeQuantum) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/field_internal_test.go b/field_internal_test.go index 4ded2bebe..2d37ea25c 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -282,7 +282,7 @@ func TestField_SetTimeQuantum(t *testing.T) { defer f.Close() // Set & retrieve time quantum. - if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil { + if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") { t.Fatalf("unexpected quantum: %s", q) @@ -300,7 +300,7 @@ func TestField_RowTime(t *testing.T) { f := MustOpenField(OptFieldTypeTime(TimeQuantum(""))) defer f.Close() - if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil { + if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { t.Fatal(err) } From 5d7cb333226a89736f52b658eef64aa6165f2ba8 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:44:13 -0500 Subject: [PATCH 043/166] Unexport Field.Logger --- field.go | 6 +++--- index.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/field.go b/field.go index 7906750b5..54e82091d 100644 --- a/field.go +++ b/field.go @@ -72,7 +72,7 @@ type Field struct { bsiGroups []*bsiGroup - Logger Logger + logger Logger } // FieldOption is a functional option type for pilosa.fieldOptions. @@ -166,7 +166,7 @@ func NewField(path, index, name string, opts FieldOption) (*Field, error) { options: applyDefaultOptions(fo), - Logger: NopLogger, + logger: NopLogger, } return f, nil } @@ -660,7 +660,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) { func (f *Field) newView(path, name string) *view { view := newView(path, f.index, f.name, name, f.options.CacheSize) view.cacheType = f.options.CacheType - view.logger = f.Logger + view.logger = f.logger view.rowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster diff --git a/index.go b/index.go index fb97ded2f..c0490601b 100644 --- a/index.go +++ b/index.go @@ -363,7 +363,7 @@ func (i *Index) newField(path, name string) (*Field, error) { if err != nil { return nil, err } - f.Logger = i.logger + f.logger = i.logger f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) From 3ff6851881ddcb26a30e78e449afd5ad00921eaf Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:44:33 -0500 Subject: [PATCH 044/166] Unexport FieldOptions.Encode --- field.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/field.go b/field.go index 54e82091d..a435148d3 100644 --- a/field.go +++ b/field.go @@ -337,7 +337,7 @@ func (f *Field) loadMeta() error { func (f *Field) saveMeta() error { // Marshal metadata. fo := f.options - buf, err := proto.Marshal(fo.Encode()) + buf, err := proto.Marshal(fo.encode()) if err != nil { return errors.Wrap(err, "marshaling") } @@ -1135,8 +1135,8 @@ func applyDefaultOptions(o FieldOptions) FieldOptions { return o } -// Encode converts o into its internal representation. -func (o *FieldOptions) Encode() *internal.FieldOptions { +// encode converts o into its internal representation. +func (o *FieldOptions) encode() *internal.FieldOptions { return encodeFieldOptions(o) } From 0648d0fc7546c50cd8acaae797d0764566ab783a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:45:26 -0500 Subject: [PATCH 045/166] Unexport Holder.RecalculateCaches --- api.go | 2 +- holder.go | 4 ++-- server.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api.go b/api.go index 5230c8677..6ea2c79b5 100644 --- a/api.go +++ b/api.go @@ -446,7 +446,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { if err != nil { return errors.Wrap(err, "broacasting message") } - api.holder.RecalculateCaches() + api.holder.recalculateCaches() return nil } diff --git a/holder.go b/holder.go index 192c83a46..323f985b2 100644 --- a/holder.go +++ b/holder.go @@ -456,11 +456,11 @@ func (h *Holder) flushCaches() { } } -// RecalculateCaches recalculates caches on every index in the holder. This is +// recalculateCaches recalculates caches on every index in the holder. This is // probably not practical to call in real-world workloads, but makes writing // integration tests much eaiser, since one doesn't have to wait 10 seconds // after setting bits to get expected response. -func (h *Holder) RecalculateCaches() { +func (h *Holder) recalculateCaches() { for _, index := range h.Indexes() { index.RecalculateCaches() } diff --git a/server.go b/server.go index 38c8d9426..3cf40ee19 100644 --- a/server.go +++ b/server.go @@ -513,7 +513,7 @@ func (s *Server) receiveMessage(m Message) error { return err } case *RecalculateCaches: - s.holder.RecalculateCaches() + s.holder.recalculateCaches() case *NodeEvent: s.cluster.ReceiveEvent(obj) case *NodeStatus: From 1b6846d6e6d196b6afde12445acddf265315b27d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:47:39 -0500 Subject: [PATCH 046/166] Unexport Index.FieldPath --- index.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/index.go b/index.go index c0490601b..67b0274a4 100644 --- a/index.go +++ b/index.go @@ -139,7 +139,7 @@ func (i *Index) openFields() error { continue } - fld, err := i.newField(i.FieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { return ErrName } @@ -236,8 +236,8 @@ func (i *Index) setRemoteMaxShard(newmax uint64) { i.remoteMaxShard = newmax } -// FieldPath returns the path to a field in the index. -func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) } +// fieldPath returns the path to a field in the index. +func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) } // Field returns a field in the index by name. func (i *Index) Field(name string) *Field { @@ -331,7 +331,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { } // Initialize field. - f, err := i.newField(i.FieldPath(name), name) + f, err := i.newField(i.fieldPath(name), name) if err != nil { return nil, errors.Wrap(err, "initializing") } @@ -387,7 +387,7 @@ func (i *Index) DeleteField(name string) error { } // Delete field directory. - if err := os.RemoveAll(i.FieldPath(name)); err != nil { + if err := os.RemoveAll(i.fieldPath(name)); err != nil { return errors.Wrap(err, "removing directory") } From 13a6542a15f5a82dba724afe2737cabab00402e8 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:47:45 -0500 Subject: [PATCH 047/166] Unexport Index.RecalculateCaches --- holder.go | 2 +- index.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/holder.go b/holder.go index 323f985b2..b2ce1e67b 100644 --- a/holder.go +++ b/holder.go @@ -462,7 +462,7 @@ func (h *Holder) flushCaches() { // after setting bits to get expected response. func (h *Holder) recalculateCaches() { for _, index := range h.Indexes() { - index.RecalculateCaches() + index.recalculateCaches() } } diff --git a/index.go b/index.go index 67b0274a4..d83b20111 100644 --- a/index.go +++ b/index.go @@ -262,8 +262,8 @@ func (i *Index) Fields() []*Field { return a } -// RecalculateCaches recalculates caches on every field in the index. -func (i *Index) RecalculateCaches() { +// recalculateCaches recalculates caches on every field in the index. +func (i *Index) recalculateCaches() { for _, field := range i.Fields() { field.recalculateCaches() } From 51619e81a69d3b00fc2270076b3b1c6e5ef9cc61 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:47:51 -0500 Subject: [PATCH 048/166] Unexport IndexInfo.Options --- index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.go b/index.go index d83b20111..d8a1bb1bd 100644 --- a/index.go +++ b/index.go @@ -406,7 +406,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` - Options IndexOptions `json:"options"` + options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` } From b4b0fd2e64a5cb0a5b55201e722907d7a05c0903 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:47:58 -0500 Subject: [PATCH 049/166] Unexport LogEntry.HeaderSize --- translate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.go b/translate.go index d58586fc6..a4a4bfcdd 100644 --- a/translate.go +++ b/translate.go @@ -193,7 +193,7 @@ func (s *TranslateFile) appendEntry(entry *LogEntry) error { func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { // Move offset to the start of the id/key pairs. - offset += entry.HeaderSize() + offset += entry.headerSize() var idx *index switch entry.Type { @@ -558,8 +558,8 @@ type LogEntry struct { Length uint64 } -// HeaderSize returns the number of bytes required for size, type, index, frame, & pair count. -func (e *LogEntry) HeaderSize() int64 { +// headerSize returns the number of bytes required for size, type, index, frame, & pair count. +func (e *LogEntry) headerSize() int64 { sz := uVarintSize(e.Length) + // total entry length 1 + // type uVarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data From a1fc587577823dc8c1dfd6d34b01febbe9992b3b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:11 -0500 Subject: [PATCH 050/166] Unexport NewApiMethodNotAllowedError --- api.go | 2 +- pilosa.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 6ea2c79b5..86a739d5f 100644 --- a/api.go +++ b/api.go @@ -90,7 +90,7 @@ func (api *API) validate(f apiMethod) error { if _, ok := validAPIMethods[state][f]; ok { return nil } - return NewApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) + return newApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) } // Query parses a PQL query out of the request and executes it. diff --git a/pilosa.go b/pilosa.go index a63e64f76..7c313d29b 100644 --- a/pilosa.go +++ b/pilosa.go @@ -69,8 +69,8 @@ type apiMethodNotAllowedError struct { error } -// NewApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. -func NewApiMethodNotAllowedError(err error) apiMethodNotAllowedError { +// newApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. +func newApiMethodNotAllowedError(err error) apiMethodNotAllowedError { return apiMethodNotAllowedError{err} } From 59bbbfc1fbd2408346815943069519c98437eb62 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:19 -0500 Subject: [PATCH 051/166] Unexport NewConflictError --- holder.go | 2 +- index.go | 2 +- pilosa.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/holder.go b/holder.go index b2ce1e67b..430253ba9 100644 --- a/holder.go +++ b/holder.go @@ -304,7 +304,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) { // Ensure index doesn't already exist. if h.indexes[name] != nil { - return nil, NewConflictError(ErrIndexExists) + return nil, newConflictError(ErrIndexExists) } return h.createIndex(name, opt) } diff --git a/index.go b/index.go index d8a1bb1bd..2d66a42c1 100644 --- a/index.go +++ b/index.go @@ -276,7 +276,7 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { // Ensure field doesn't already exist. if i.fields[name] != nil { - return nil, NewConflictError(ErrFieldExists) + return nil, newConflictError(ErrFieldExists) } // Apply functional options. diff --git a/pilosa.go b/pilosa.go index 7c313d29b..8474fae89 100644 --- a/pilosa.go +++ b/pilosa.go @@ -93,8 +93,8 @@ type ConflictError struct { error } -// NewConflictError returns err wrapped in a ConflictError. -func NewConflictError(err error) ConflictError { +// newConflictError returns err wrapped in a ConflictError. +func newConflictError(err error) ConflictError { return ConflictError{err} } From 5f74927d03ab573301929de95725566ead295a81 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:25 -0500 Subject: [PATCH 052/166] Unexport NewDiagnosticsCollector --- diagnostics.go | 4 ++-- diagnostics_internal_test.go | 8 ++++---- server.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 8bdcdef3c..16ef71ad8 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -56,8 +56,8 @@ type diagnosticsCollector struct { server *Server } -// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". -func NewDiagnosticsCollector(host string) *diagnosticsCollector { +// newDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". +func newDiagnosticsCollector(host string) *diagnosticsCollector { return &diagnosticsCollector{ host: host, VersionURL: defaultVersionCheckURL, diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index 517dbed3d..f1536e1d1 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -29,7 +29,7 @@ func TestDiagnosticsClient(t *testing.T) { server := httptest.NewServer(nil) // Create a new client. - d := NewDiagnosticsCollector(server.URL) + d := newDiagnosticsCollector(server.URL) d.Set("gg", 10) d.Set("ss", "ss") @@ -76,7 +76,7 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { } func TestDiagnosticsVersion_Compare(t *testing.T) { - d := NewDiagnosticsCollector("localhost:10101") + d := newDiagnosticsCollector("localhost:10101") version := "v0.1.1" d.SetVersion(version) @@ -118,7 +118,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) { })) // Create a new client. - d := NewDiagnosticsCollector("localhost:10101") + d := newDiagnosticsCollector("localhost:10101") version := "0.1.1" d.SetVersion(version) @@ -143,7 +143,7 @@ func BenchmarkDiagnostics(b *testing.B) { server := httptest.NewServer(nil) // Create a new client. - d := NewDiagnosticsCollector(server.URL) + d := newDiagnosticsCollector(server.URL) prev := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(prev) diff --git a/server.go b/server.go index 3cf40ee19..2a7d7d783 100644 --- a/server.go +++ b/server.go @@ -235,7 +235,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { closing: make(chan struct{}), cluster: newCluster(), holder: NewHolder(), - diagnostics: NewDiagnosticsCollector(defaultDiagnosticServer), + diagnostics: newDiagnosticsCollector(defaultDiagnosticServer), systemInfo: NewNopSystemInfo(), gcNotifier: NopGCNotifier, From 0907ee585426a3da133f10fed2c3d79cda5929ea Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:32 -0500 Subject: [PATCH 053/166] Unexport NewNopInternalClient --- client.go | 4 ++-- cluster.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 5c51ae63f..671bd2a76 100644 --- a/client.go +++ b/client.go @@ -72,11 +72,11 @@ var _ InternalQueryClient = NewNopInternalQueryClient() type NopInternalClient struct{} -func NewNopInternalClient() NopInternalClient { +func newNopInternalClient() NopInternalClient { return NopInternalClient{} } -var _ InternalClient = NewNopInternalClient() +var _ InternalClient = newNopInternalClient() func (n NopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { return nil, nil diff --git a/cluster.go b/cluster.go index 45bbb835d..226221b99 100644 --- a/cluster.go +++ b/cluster.go @@ -227,7 +227,7 @@ func newCluster() *cluster { closing: make(chan struct{}), joining: make(chan struct{}), - InternalClient: NewNopInternalClient(), + InternalClient: newNopInternalClient(), logger: NopLogger, } From 631ee915df0d3991b91835db6c888e660330382e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:38 -0500 Subject: [PATCH 054/166] Unexport NewNopInternalQueryClient --- client.go | 4 ++-- executor.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 671bd2a76..5a1ed97c1 100644 --- a/client.go +++ b/client.go @@ -62,11 +62,11 @@ func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index return nil, nil } -func NewNopInternalQueryClient() *NopInternalQueryClient { +func newNopInternalQueryClient() *NopInternalQueryClient { return &NopInternalQueryClient{} } -var _ InternalQueryClient = NewNopInternalQueryClient() +var _ InternalQueryClient = newNopInternalQueryClient() //=============== diff --git a/executor.go b/executor.go index 9b7f5d4bb..579a9aa9d 100644 --- a/executor.go +++ b/executor.go @@ -67,7 +67,7 @@ func optExecutorInternalQueryClient(c InternalQueryClient) executorOption { // newExecutor returns a new instance of Executor. func newExecutor(opts ...executorOption) *executor { e := &executor{ - client: NewNopInternalQueryClient(), + client: newNopInternalQueryClient(), } for _, opt := range opts { err := opt(e) From 92d521912e01d0eee2791873ff80931d312873e9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:44 -0500 Subject: [PATCH 055/166] Unexport NewNopSystemInfo --- diagnostics.go | 4 ++-- server.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 16ef71ad8..11d3f5b9c 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -267,8 +267,8 @@ type SystemInfo interface { MemUsed() (uint64, error) } -// NewNopSystemInfo creates a no-op implementation of SystemInfo. -func NewNopSystemInfo() *NopSystemInfo { +// newNopSystemInfo creates a no-op implementation of SystemInfo. +func newNopSystemInfo() *NopSystemInfo { return &NopSystemInfo{} } diff --git a/server.go b/server.go index 2a7d7d783..6ee4c6c60 100644 --- a/server.go +++ b/server.go @@ -236,7 +236,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { cluster: newCluster(), holder: NewHolder(), diagnostics: newDiagnosticsCollector(defaultDiagnosticServer), - systemInfo: NewNopSystemInfo(), + systemInfo: newNopSystemInfo(), gcNotifier: NopGCNotifier, From 4aa8a41fa097e433ef048a3bb183296ea149d8d2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:52 -0500 Subject: [PATCH 056/166] Unexport NewNotFoundError --- api.go | 12 ++++++------ holder.go | 2 +- index.go | 2 +- pilosa.go | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api.go b/api.go index 86a739d5f..f646112b9 100644 --- a/api.go +++ b/api.go @@ -205,7 +205,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { index := api.holder.Index(indexName) if index == nil { - return nil, NewNotFoundError(ErrIndexNotFound) + return nil, newNotFoundError(ErrIndexNotFound) } return index, nil } @@ -255,7 +255,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str // Find index. index := api.holder.Index(indexName) if index == nil { - return nil, NewNotFoundError(ErrIndexNotFound) + return nil, newNotFoundError(ErrIndexNotFound) } // Create field. @@ -287,7 +287,7 @@ func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field, field := api.holder.Field(indexName, fieldName) if field == nil { - return nil, NewNotFoundError(ErrFieldNotFound) + return nil, newNotFoundError(ErrFieldNotFound) } return field, nil } @@ -303,7 +303,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str // Find index. index := api.holder.Index(indexName) if index == nil { - return NewNotFoundError(ErrIndexNotFound) + return newNotFoundError(ErrIndexNotFound) } // Delete field from the index. @@ -543,7 +543,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At // Retrieve index from holder. index := api.holder.Index(indexName) if index == nil { - return nil, NewNotFoundError(ErrIndexNotFound) + return nil, newNotFoundError(ErrIndexNotFound) } // Retrieve local blocks. @@ -685,7 +685,7 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I index := api.holder.Index(indexName) if index == nil { api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error()) - return nil, nil, NewNotFoundError(ErrIndexNotFound) + return nil, nil, newNotFoundError(ErrIndexNotFound) } // Retrieve field. diff --git a/holder.go b/holder.go index 430253ba9..a8caf8664 100644 --- a/holder.go +++ b/holder.go @@ -374,7 +374,7 @@ func (h *Holder) DeleteIndex(name string) error { // Confirm index exists. index := h.index(name) if index == nil { - return NewNotFoundError(ErrIndexNotFound) + return newNotFoundError(ErrIndexNotFound) } // Close index. diff --git a/index.go b/index.go index 2d66a42c1..1cda5e7c9 100644 --- a/index.go +++ b/index.go @@ -378,7 +378,7 @@ func (i *Index) DeleteField(name string) error { // Confirm field exists. f := i.field(name) if f == nil { - return NewNotFoundError(ErrFieldNotFound) + return newNotFoundError(ErrFieldNotFound) } // Close field. diff --git a/pilosa.go b/pilosa.go index 8474fae89..45d77a9f3 100644 --- a/pilosa.go +++ b/pilosa.go @@ -104,8 +104,8 @@ type NotFoundError struct { error } -// NewNotFoundError returns err wrapped in a NotFoundError. -func NewNotFoundError(err error) NotFoundError { +// newNotFoundError returns err wrapped in a NotFoundError. +func newNotFoundError(err error) NotFoundError { return NotFoundError{err} } From 2031345c86cf1457650e0842e77a5401215f7484 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:48:59 -0500 Subject: [PATCH 057/166] Unexport NewTopology --- cluster.go | 6 +++--- utils_internal_test.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index 226221b99..647db4f24 100644 --- a/cluster.go +++ b/cluster.go @@ -1403,7 +1403,7 @@ type Topology struct { nodeStates map[string]string } -func NewTopology() *Topology { +func newTopology() *Topology { return &Topology{ nodeStates: make(map[string]string), } @@ -1472,7 +1472,7 @@ func (t *Topology) Encode() *internal.Topology { func (c *cluster) loadTopology() error { buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) if os.IsNotExist(err) { - c.Topology = NewTopology() + c.Topology = newTopology() return nil } else if err != nil { return errors.Wrap(err, "reading file") @@ -1784,7 +1784,7 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { return nil, nil } - t := NewTopology() + t := newTopology() t.ClusterID = topology.ClusterID t.NodeIDs = topology.NodeIDs sort.Slice(t.NodeIDs, diff --git a/utils_internal_test.go b/utils_internal_test.go index 7adb110e8..baaac04e3 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -37,7 +37,7 @@ func NewTestCluster(n int) *cluster { c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = NewTopology() + c.Topology = newTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &Node{ @@ -225,7 +225,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = NewTopology() + c.Topology = newTopology() c.holder = h c.Node = node c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator From 9d882469da9399e128d1768eb86107e02852c225 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:49:05 -0500 Subject: [PATCH 058/166] Unexport NewTranslateFileReader --- translate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.go b/translate.go index a4a4bfcdd..04922b7a6 100644 --- a/translate.go +++ b/translate.go @@ -538,7 +538,7 @@ func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (st // Reader returns a reader that streams the underlying data file. func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { - rc := NewTranslateFileReader(ctx, s, offset) + rc := newTranslateFileReader(ctx, s, offset) if err := rc.Open(); err != nil { return nil, err } @@ -910,8 +910,8 @@ type TranslateFileReader struct { closing chan struct{} } -// NewTranslateFileReader returns a new instance of TranslateFileReader. -func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { +// newTranslateFileReader returns a new instance of TranslateFileReader. +func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { return &TranslateFileReader{ ctx: ctx, store: store, From 9222ef0df78d79cbe76ccb227a9add07e5fb9628 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:49:33 -0500 Subject: [PATCH 059/166] Unexport NodeIDs --- cluster.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster.go b/cluster.go index 647db4f24..aa48ba57c 100644 --- a/cluster.go +++ b/cluster.go @@ -1375,14 +1375,14 @@ func (j *resizeJob) distributeResizeInstructions() error { return nil } -type NodeIDs []string +type nodeIDs []string -func (n NodeIDs) Len() int { return len(n) } -func (n NodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] } -func (n NodeIDs) Less(i, j int) bool { return n[i] < n[j] } +func (n nodeIDs) Len() int { return len(n) } +func (n nodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] } +func (n nodeIDs) Less(i, j int) bool { return n[i] < n[j] } // ContainsID returns true if idi matches one of the nodesets's IDs. -func (n NodeIDs) ContainsID(id string) bool { +func (n nodeIDs) ContainsID(id string) bool { for _, nid := range n { if nid == id { return true @@ -1417,7 +1417,7 @@ func (t *Topology) ContainsID(id string) bool { } func (t *Topology) containsID(id string) bool { - return NodeIDs(t.NodeIDs).ContainsID(id) + return nodeIDs(t.NodeIDs).ContainsID(id) } func (t *Topology) positionByID(nodeID string) int { From 3e91a0d32cdfdda1fbce1521141b00a9715c5fdf Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:49:56 -0500 Subject: [PATCH 060/166] Unexport NodeStateReady --- cluster.go | 4 ++-- server.go | 2 +- utils_internal_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index aa48ba57c..6f0a88904 100644 --- a/cluster.go +++ b/cluster.go @@ -45,7 +45,7 @@ const ( ClusterStateResizing = "RESIZING" // NodeState represents the state of a node during startup. - NodeStateReady = "READY" + nodeStateReady = "READY" // resizeJob states. resizeJobStateRunning = "RUNNING" @@ -903,7 +903,7 @@ func (c *cluster) allNodesReady() bool { return true } for _, uri := range c.Topology.NodeIDs { - if c.Topology.nodeStates[uri] != NodeStateReady { + if c.Topology.nodeStates[uri] != nodeStateReady { return false } } diff --git a/server.go b/server.go index 6ee4c6c60..91c2d813a 100644 --- a/server.go +++ b/server.go @@ -337,7 +337,7 @@ func (s *Server) Open() error { if err := s.holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } - if err := s.cluster.setNodeState(NodeStateReady); err != nil { + if err := s.cluster.setNodeState(nodeStateReady); err != nil { return fmt.Errorf("setting nodeState: %v", err) } diff --git a/utils_internal_test.go b/utils_internal_test.go index baaac04e3..816c477ce 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -277,7 +277,7 @@ func (t *ClusterCluster) Open() error { if err := c.holder.Open(); err != nil { return err } - if err := c.setNodeState(NodeStateReady); err != nil { + if err := c.setNodeState(nodeStateReady); err != nil { return err } } From e50ec6dc9d673ad5bdc911d3fc2e9ca06e726f3a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:50:27 -0500 Subject: [PATCH 061/166] Unexport NopInternalClient --- client.go | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/client.go b/client.go index 5a1ed97c1..271c8d170 100644 --- a/client.go +++ b/client.go @@ -70,64 +70,64 @@ var _ InternalQueryClient = newNopInternalQueryClient() //=============== -type NopInternalClient struct{} +type nopInternalClient struct{} -func newNopInternalClient() NopInternalClient { - return NopInternalClient{} +func newNopInternalClient() nopInternalClient { + return nopInternalClient{} } var _ InternalClient = newNopInternalClient() -func (n NopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { +func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { return nil, nil } -func (n NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } -func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { +func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } +func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { return nil } -func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { +func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { return nil, nil } -func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { +func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { +func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { +func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { return nil } -func (n NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { +func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n NopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { +func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { return nil } -func (n NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { +func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } -func (n NopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { +func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } -func (n NopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { +func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { return nil } -func (n NopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } -func (n NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) { +func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } +func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) { return nil, nil } -func (n NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { +func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { return nil, nil, nil } -func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { +func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { return nil } -func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { +func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { return nil, nil } From ae5f9210d5826837cf9344f97d6646a61106d3a0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:50:35 -0500 Subject: [PATCH 062/166] Unexport NopInternalQueryClient --- client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client.go b/client.go index 271c8d170..6c912bd99 100644 --- a/client.go +++ b/client.go @@ -56,14 +56,14 @@ type InternalQueryClient interface { QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) } -type NopInternalQueryClient struct{} +type nopInternalQueryClient struct{} -func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { +func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func newNopInternalQueryClient() *NopInternalQueryClient { - return &NopInternalQueryClient{} +func newNopInternalQueryClient() *nopInternalQueryClient { + return &nopInternalQueryClient{} } var _ InternalQueryClient = newNopInternalQueryClient() From 6b5595d48c2a2b4bc913a8b21a5089210a8c0947 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:50:41 -0500 Subject: [PATCH 063/166] Unexport NopSystemInfo --- diagnostics.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 11d3f5b9c..9649a6505 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -268,50 +268,50 @@ type SystemInfo interface { } // newNopSystemInfo creates a no-op implementation of SystemInfo. -func newNopSystemInfo() *NopSystemInfo { - return &NopSystemInfo{} +func newNopSystemInfo() *nopSystemInfo { + return &nopSystemInfo{} } -// NopSystemInfo is a no-op implementation of SystemInfo. -type NopSystemInfo struct { +// nopSystemInfo is a no-op implementation of SystemInfo. +type nopSystemInfo struct { } // Uptime is a no-op implementation of SystemInfo.Uptime. -func (n *NopSystemInfo) Uptime() (uint64, error) { +func (n *nopSystemInfo) Uptime() (uint64, error) { return 0, nil } // Platform is a no-op implementation of SystemInfo.Platform. -func (n *NopSystemInfo) Platform() (string, error) { +func (n *nopSystemInfo) Platform() (string, error) { return "", nil } // Family is a no-op implementation of SystemInfo.Family. -func (n *NopSystemInfo) Family() (string, error) { +func (n *nopSystemInfo) Family() (string, error) { return "", nil } // OSVersion is a no-op implementation of SystemInfo.OSVersion. -func (n *NopSystemInfo) OSVersion() (string, error) { +func (n *nopSystemInfo) OSVersion() (string, error) { return "", nil } // KernelVersion is a no-op implementation of SystemInfo.KernelVersion. -func (n *NopSystemInfo) KernelVersion() (string, error) { +func (n *nopSystemInfo) KernelVersion() (string, error) { return "", nil } // MemFree is a no-op implementation of SystemInfo.MemFree. -func (n *NopSystemInfo) MemFree() (uint64, error) { +func (n *nopSystemInfo) MemFree() (uint64, error) { return 0, nil } // MemTotal is a no-op implementation of SystemInfo.MemTotal. -func (n *NopSystemInfo) MemTotal() (uint64, error) { +func (n *nopSystemInfo) MemTotal() (uint64, error) { return 0, nil } // MemUsed is a no-op implementation of SystemInfo.MemUsed. -func (n *NopSystemInfo) MemUsed() (uint64, error) { +func (n *nopSystemInfo) MemUsed() (uint64, error) { return 0, nil } From edf87473ee40262d7f28f4d30c7cf34849f3baac Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:53:07 -0500 Subject: [PATCH 064/166] Unexport Row.ClearBit --- fragment.go | 2 +- row.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fragment.go b/fragment.go index 1ae3c5bb3..99b107a91 100644 --- a/fragment.go +++ b/fragment.go @@ -446,7 +446,7 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er // Get the row from cache or fragment.storage. row := f.unprotectedRow(rowID, true, true) - row.ClearBit(columnID) + row.clearBit(columnID) // Update the cache. f.cache.Add(rowID, row.Count()) diff --git a/row.go b/row.go index d6a0037cc..06b116fb0 100644 --- a/row.go +++ b/row.go @@ -159,8 +159,8 @@ func (r *Row) SetBit(i uint64) (changed bool) { return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) } -// ClearBit clears the i-th column of the row. -func (r *Row) ClearBit(i uint64) (changed bool) { +// clearBit clears the i-th column of the row. +func (r *Row) clearBit(i uint64) (changed bool) { s := r.segment(i / ShardWidth) if s == nil { return false From 3025586378881f651187ed83a77ab789c0703d67 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:53:15 -0500 Subject: [PATCH 065/166] Unexport Row.Intersect --- executor.go | 2 +- fragment.go | 12 ++++++------ row.go | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/executor.go b/executor.go index 579a9aa9d..c0513b6ae 100644 --- a/executor.go +++ b/executor.go @@ -712,7 +712,7 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p if i == 0 { other = row } else { - other = other.Intersect(row) + other = other.intersect(row) } } other.InvalidateCount() diff --git a/fragment.go b/fragment.go index 99b107a91..ac1c14061 100644 --- a/fragment.go +++ b/fragment.go @@ -598,7 +598,7 @@ func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error consider := f.row(uint64(bitDepth)) if filter != nil { - consider = consider.Intersect(filter) + consider = consider.intersect(filter) } // If there are no columns to consider, return early. @@ -631,7 +631,7 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error consider := f.row(uint64(bitDepth)) if filter != nil { - consider = consider.Intersect(filter) + consider = consider.intersect(filter) } // If there are no columns to consider, return early. @@ -643,7 +643,7 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error ii := i - 1 // allow for uint range: (bitDepth-1) to 0 row := f.row(uint64(ii)) - x := row.Intersect(consider) + x := row.intersect(consider) count = x.Count() if count > 0 { max += (1 << ii) @@ -682,7 +682,7 @@ func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { bit := (predicate >> uint(i)) & 1 if bit == 1 { - b = b.Intersect(row) + b = b.intersect(row) } else { b = b.Difference(row) } @@ -783,7 +783,7 @@ func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep = keep.Union(b.Intersect(row)) + keep = keep.Union(b.intersect(row)) } } @@ -815,7 +815,7 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64 // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep1 = keep1.Union(b.Intersect(row)) + keep1 = keep1.Union(b.intersect(row)) } } diff --git a/row.go b/row.go index 06b116fb0..a78002034 100644 --- a/row.go +++ b/row.go @@ -82,8 +82,8 @@ func (r *Row) IntersectionCount(other *Row) uint64 { return n } -// Intersect returns the itersection of r and other. -func (r *Row) Intersect(other *Row) *Row { +// intersect returns the itersection of r and other. +func (r *Row) intersect(other *Row) *Row { var segments []RowSegment itr := newMergeSegmentIterator(r.segments, other.segments) From 018905fcd9755fa3ba32b6df8d65a14cb399c731 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:53:23 -0500 Subject: [PATCH 066/166] Unexport Row.IntersectionCount --- fragment.go | 8 ++++---- fragment_internal_test.go | 2 +- row.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fragment.go b/fragment.go index ac1c14061..1614c2163 100644 --- a/fragment.go +++ b/fragment.go @@ -566,7 +566,7 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // Compute count based on the existence row. row := f.row(uint64(bitDepth)) if filter != nil { - count = row.IntersectionCount(filter) + count = row.intersectionCount(filter) } else { count = row.Count() } @@ -582,7 +582,7 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error row := f.row(uint64(i)) cnt := uint64(0) if filter != nil { - cnt = row.IntersectionCount(filter) + cnt = row.intersectionCount(filter) } else { cnt = row.Count() } @@ -938,7 +938,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.row(rowID)) + count = opt.Src.intersectionCount(f.row(rowID)) } if count == 0 { continue @@ -982,7 +982,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - count := opt.Src.IntersectionCount(f.row(rowID)) + count := opt.Src.intersectionCount(f.row(rowID)) if count < threshold { continue } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 4ceb33819..e665a10ba 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1063,7 +1063,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.row(1).IntersectionCount(f.row(2)); n == 0 { + if n := f.row(1).intersectionCount(f.row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } diff --git a/row.go b/row.go index a78002034..125fec08c 100644 --- a/row.go +++ b/row.go @@ -66,8 +66,8 @@ func (r *Row) Merge(other *Row) { r.InvalidateCount() } -// IntersectionCount returns the number of intersections between r and other. -func (r *Row) IntersectionCount(other *Row) uint64 { +// intersectionCount returns the number of intersections between r and other. +func (r *Row) intersectionCount(other *Row) uint64 { var n uint64 itr := newMergeSegmentIterator(r.segments, other.segments) From 259f5ac3b9c96fbf77c5fbb2b6d96db9986546ee Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:53:29 -0500 Subject: [PATCH 067/166] Unexport Row.InvalidateCount --- executor.go | 8 ++++---- fragment.go | 2 +- row.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/executor.go b/executor.go index c0513b6ae..34eca4026 100644 --- a/executor.go +++ b/executor.go @@ -661,7 +661,7 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * other = other.Difference(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } @@ -715,7 +715,7 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p other = other.intersect(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } @@ -937,7 +937,7 @@ func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.C other = other.Union(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } @@ -956,7 +956,7 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal other = other.Xor(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } diff --git a/fragment.go b/fragment.go index 1614c2163..dc7ded5b8 100644 --- a/fragment.go +++ b/fragment.go @@ -349,7 +349,7 @@ func (f *fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac writable: false, }}, } - row.InvalidateCount() + row.invalidateCount() if updateRowCache { f.rowCache.Add(rowID, row) diff --git a/row.go b/row.go index 125fec08c..109dbc296 100644 --- a/row.go +++ b/row.go @@ -63,7 +63,7 @@ func (r *Row) Merge(other *Row) { } r.segments = segments - r.InvalidateCount() + r.invalidateCount() } // intersectionCount returns the number of intersections between r and other. @@ -208,8 +208,8 @@ func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment { return &r.segments[i] } -// InvalidateCount updates the cached count in the row. -func (r *Row) InvalidateCount() { +// invalidateCount updates the cached count in the row. +func (r *Row) invalidateCount() { for i := range r.segments { r.segments[i].InvalidateCount() } From 45f3439a7f317e2ff6240c214de7fa37a0d8a938 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:53:41 -0500 Subject: [PATCH 068/166] Unexport RowSegment --- executor.go | 2 +- fragment.go | 2 +- row.go | 64 ++++++++++++++++++++++++++--------------------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/executor.go b/executor.go index 34eca4026..03d18f936 100644 --- a/executor.go +++ b/executor.go @@ -375,7 +375,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } if opt.ExcludeColumns { - row.segments = []RowSegment{} + row.segments = []rowSegment{} } return row, nil diff --git a/fragment.go b/fragment.go index dc7ded5b8..8c5fbf787 100644 --- a/fragment.go +++ b/fragment.go @@ -343,7 +343,7 @@ func (f *fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac // We Clone() data because otherwise row will contains pointers to containers in storage. // This causes unexpected results when we cache the row and try to use it later. row := &Row{ - segments: []RowSegment{{ + segments: []rowSegment{{ data: *data.Clone(), shard: f.shard, writable: false, diff --git a/row.go b/row.go index 109dbc296..38ba24f9b 100644 --- a/row.go +++ b/row.go @@ -24,7 +24,7 @@ import ( // Row is a set of integers (the associated columns), and attributes which are // arbitrary key/value pairs storing metadata about what the row represents. type Row struct { - segments []RowSegment + segments []rowSegment // String keys translated to/from segment columns. Keys []string @@ -44,7 +44,7 @@ func NewRow(columns ...uint64) *Row { // Merge merges data from other into r. func (r *Row) Merge(other *Row) { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -84,7 +84,7 @@ func (r *Row) intersectionCount(other *Row) uint64 { // intersect returns the itersection of r and other. func (r *Row) intersect(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -100,7 +100,7 @@ func (r *Row) intersect(other *Row) *Row { // Xor returns the xor of r and other. func (r *Row) Xor(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -120,7 +120,7 @@ func (r *Row) Xor(other *Row) *Row { // Union returns the bitwise union of r and other. func (r *Row) Union(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { if s1 == nil { @@ -138,7 +138,7 @@ func (r *Row) Union(other *Row) *Row { // Difference returns the diff of r and other. func (r *Row) Difference(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -169,13 +169,13 @@ func (r *Row) clearBit(i uint64) (changed bool) { } // Segments returns a list of all segments in the row. -func (r *Row) Segments() []RowSegment { +func (r *Row) Segments() []rowSegment { return r.segments } // segment returns a segment for a given shard. // Returns nil if segment does not exist. -func (r *Row) segment(shard uint64) *RowSegment { +func (r *Row) segment(shard uint64) *rowSegment { if i := sort.Search(len(r.segments), func(i int) bool { return r.segments[i].shard >= shard }); i < len(r.segments) && r.segments[i].shard == shard { @@ -184,7 +184,7 @@ func (r *Row) segment(shard uint64) *RowSegment { return nil } -func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment { +func (r *Row) createSegmentIfNotExists(shard uint64) *rowSegment { i := sort.Search(len(r.segments), func(i int) bool { return r.segments[i].shard >= shard }) @@ -195,11 +195,11 @@ func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment { } // Insert new segment. - r.segments = append(r.segments, RowSegment{data: *roaring.NewBitmap()}) + r.segments = append(r.segments, rowSegment{data: *roaring.NewBitmap()}) if i < len(r.segments) { copy(r.segments[i+1:], r.segments[i:]) } - r.segments[i] = RowSegment{ + r.segments[i] = rowSegment{ data: *roaring.NewBitmap(), shard: shard, writable: true, @@ -251,10 +251,10 @@ func (r *Row) Columns() []uint64 { return a } -// RowSegment holds a subset of a row. +// rowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The // width of the segment will always match the shard width. -type RowSegment struct { +type rowSegment struct { // Shard this segment belongs to shard uint64 @@ -270,7 +270,7 @@ type RowSegment struct { // Merge adds chunks from other to s. // Chunks in s are overwritten if they exist in other. -func (s *RowSegment) Merge(other *RowSegment) { +func (s *rowSegment) Merge(other *rowSegment) { s.ensureWritable() itr := other.data.Iterator() @@ -280,15 +280,15 @@ func (s *RowSegment) Merge(other *RowSegment) { } // IntersectionCount returns the number of intersections between s and other. -func (s *RowSegment) IntersectionCount(other *RowSegment) uint64 { +func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { return s.data.IntersectionCount(&other.data) } // Intersect returns the itersection of s and other. -func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { +func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { data := s.data.Intersect(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, shard: s.shard, n: data.Count(), @@ -296,10 +296,10 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { } // Union returns the bitwise union of s and other. -func (s *RowSegment) Union(other *RowSegment) *RowSegment { +func (s *rowSegment) Union(other *rowSegment) *rowSegment { data := s.data.Union(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, shard: s.shard, n: data.Count(), @@ -307,10 +307,10 @@ func (s *RowSegment) Union(other *RowSegment) *RowSegment { } // Difference returns the diff of s and other. -func (s *RowSegment) Difference(other *RowSegment) *RowSegment { +func (s *rowSegment) Difference(other *rowSegment) *rowSegment { data := s.data.Difference(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, shard: s.shard, n: data.Count(), @@ -318,10 +318,10 @@ func (s *RowSegment) Difference(other *RowSegment) *RowSegment { } // Xor returns the xor of s and other. -func (s *RowSegment) Xor(other *RowSegment) *RowSegment { +func (s *rowSegment) Xor(other *rowSegment) *rowSegment { data := s.data.Xor(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, shard: s.shard, n: data.Count(), @@ -329,7 +329,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment { } // SetBit sets the i-th column of the row. -func (s *RowSegment) SetBit(i uint64) (changed bool) { +func (s *rowSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() changed, _ = s.data.Add(i) if changed { @@ -339,7 +339,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) { } // ClearBit clears the i-th column of the row. -func (s *RowSegment) ClearBit(i uint64) (changed bool) { +func (s *rowSegment) ClearBit(i uint64) (changed bool) { s.ensureWritable() changed, _ = s.data.Remove(i) @@ -350,12 +350,12 @@ func (s *RowSegment) ClearBit(i uint64) (changed bool) { } // InvalidateCount updates the cached count in the row. -func (s *RowSegment) InvalidateCount() { +func (s *rowSegment) InvalidateCount() { s.n = s.data.Count() } // Columns returns a list of all columns set in the segment. -func (s *RowSegment) Columns() []uint64 { +func (s *rowSegment) Columns() []uint64 { a := make([]uint64, 0, s.Count()) itr := s.data.Iterator() for v, eof := itr.Next(); !eof; v, eof = itr.Next() { @@ -365,10 +365,10 @@ func (s *RowSegment) Columns() []uint64 { } // Count returns the number of set columns in the row. -func (s *RowSegment) Count() uint64 { return s.n } +func (s *rowSegment) Count() uint64 { return s.n } // ensureWritable clones the segment if it is pointing to non-writable data. -func (s *RowSegment) ensureWritable() { +func (s *rowSegment) ensureWritable() { if s.writable { return } @@ -379,16 +379,16 @@ func (s *RowSegment) ensureWritable() { // mergeSegmentIterator produces an iterator that loops through two sets of segments. type mergeSegmentIterator struct { - a0, a1 []RowSegment + a0, a1 []rowSegment } // newMergeSegmentIterator returns a new instance of mergeSegmentIterator. -func newMergeSegmentIterator(a0, a1 []RowSegment) mergeSegmentIterator { +func newMergeSegmentIterator(a0, a1 []rowSegment) mergeSegmentIterator { return mergeSegmentIterator{a0: a0, a1: a1} } // next returns the next set of segments. -func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) { +func (itr *mergeSegmentIterator) next() (s0, s1 *rowSegment) { // Find current segments. if len(itr.a0) > 0 { s0 = &itr.a0[0] From 5709d329d5373363c8e48e12a109e32d8b8be474 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:54:16 -0500 Subject: [PATCH 069/166] Unexport StandardLogger --- logger.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/logger.go b/logger.go index 93b12c3f8..6afa24c96 100644 --- a/logger.go +++ b/logger.go @@ -43,24 +43,24 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {} // Debugf is a no-op implementation of the Logger Debugf method. func (n *nopLogger) Debugf(format string, v ...interface{}) {} -// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. -type StandardLogger struct { +// standardLogger is a basic implementation of pilosa.Logger based on log.Logger. +type standardLogger struct { logger *log.Logger } -func NewStandardLogger(w io.Writer) *StandardLogger { - return &StandardLogger{ +func NewStandardLogger(w io.Writer) *standardLogger { + return &standardLogger{ logger: log.New(w, "", log.LstdFlags), } } -func (s *StandardLogger) Printf(format string, v ...interface{}) { +func (s *standardLogger) Printf(format string, v ...interface{}) { s.logger.Printf(format, v...) } -func (s *StandardLogger) Debugf(format string, v ...interface{}) {} +func (s *standardLogger) Debugf(format string, v ...interface{}) {} -func (s *StandardLogger) Logger() *log.Logger { +func (s *standardLogger) Logger() *log.Logger { return s.logger } From 3f7f82bf058a2fbbe91cac2d9839423831032d96 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:54:23 -0500 Subject: [PATCH 070/166] Unexport Topology.AddID --- cluster.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster.go b/cluster.go index 6f0a88904..e49e07111 100644 --- a/cluster.go +++ b/cluster.go @@ -323,7 +323,7 @@ func (c *cluster) addNode(node *Node) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.AddID(node.ID) { + if !c.Topology.addID(node.ID) { return nil } @@ -1429,8 +1429,8 @@ func (t *Topology) positionByID(nodeID string) int { return -1 } -// AddID adds the node ID to the topology and returns true if added. -func (t *Topology) AddID(nodeID string) bool { +// addID adds the node ID to the topology and returns true if added. +func (t *Topology) addID(nodeID string) bool { t.mu.Lock() defer t.mu.Unlock() if t.containsID(nodeID) { From 65472609a5e0c13bbdbf1c7ffd94d72376641d89 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:54:30 -0500 Subject: [PATCH 071/166] Unexport Topology.Encode --- cluster.go | 4 ++-- utils_internal_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster.go b/cluster.go index e49e07111..4919e4e10 100644 --- a/cluster.go +++ b/cluster.go @@ -1463,8 +1463,8 @@ func (t *Topology) RemoveID(nodeID string) bool { return true } -// Encode converts t into its internal representation. -func (t *Topology) Encode() *internal.Topology { +// encode converts t into its internal representation. +func (t *Topology) encode() *internal.Topology { return encodeTopology(t) } diff --git a/utils_internal_test.go b/utils_internal_test.go index 816c477ce..1f58c464c 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -185,7 +185,7 @@ func (t *ClusterCluster) addNode() error { // WriteTopology writes the given topology to disk. func (t *ClusterCluster) WriteTopology(path string, top *Topology) error { - if buf, err := proto.Marshal(top.Encode()); err != nil { + if buf, err := proto.Marshal(top.encode()); err != nil { return err } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { return err From e52f71340680a8747c91b355594c22c2006cad37 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:54:37 -0500 Subject: [PATCH 072/166] Unexport Topology.RemoveID --- cluster.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster.go b/cluster.go index 4919e4e10..de707674b 100644 --- a/cluster.go +++ b/cluster.go @@ -343,7 +343,7 @@ func (c *cluster) removeNode(node *Node) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.RemoveID(node.ID) { + if !c.Topology.removeID(node.ID) { return nil } @@ -1446,8 +1446,8 @@ func (t *Topology) addID(nodeID string) bool { return true } -// RemoveID removes the node ID from the topology and returns true if removed. -func (t *Topology) RemoveID(nodeID string) bool { +// removeID removes the node ID from the topology and returns true if removed. +func (t *Topology) removeID(nodeID string) bool { t.mu.Lock() defer t.mu.Unlock() From 9c160ee65e66892ce83919a862d3d850c65d8e51 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:57:00 -0500 Subject: [PATCH 073/166] Unexport Topology.NodeIDs --- cluster.go | 36 ++++++++++++++++++------------------ cluster_internal_test.go | 32 ++++++++++++++++---------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/cluster.go b/cluster.go index de707674b..dd61d0a9f 100644 --- a/cluster.go +++ b/cluster.go @@ -888,21 +888,21 @@ func (c *cluster) markAsJoined() { } func (c *cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) + return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } func (c *cluster) haveTopologyAgreement() bool { if c.Static { return true } - return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) + return stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } func (c *cluster) allNodesReady() bool { if c.Static { return true } - for _, uri := range c.Topology.NodeIDs { + for _, uri := range c.Topology.nodeIDs { if c.Topology.nodeStates[uri] != nodeStateReady { return false } @@ -1394,7 +1394,7 @@ func (n nodeIDs) ContainsID(id string) bool { // Topology represents the list of hosts in the cluster. type Topology struct { mu sync.RWMutex - NodeIDs []string + nodeIDs []string ClusterID string @@ -1417,11 +1417,11 @@ func (t *Topology) ContainsID(id string) bool { } func (t *Topology) containsID(id string) bool { - return nodeIDs(t.NodeIDs).ContainsID(id) + return nodeIDs(t.nodeIDs).ContainsID(id) } func (t *Topology) positionByID(nodeID string) int { - for i, tid := range t.NodeIDs { + for i, tid := range t.nodeIDs { if tid == nodeID { return i } @@ -1436,11 +1436,11 @@ func (t *Topology) addID(nodeID string) bool { if t.containsID(nodeID) { return false } - t.NodeIDs = append(t.NodeIDs, nodeID) + t.nodeIDs = append(t.nodeIDs, nodeID) - sort.Slice(t.NodeIDs, + sort.Slice(t.nodeIDs, func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] + return t.nodeIDs[i] < t.nodeIDs[j] }) return true @@ -1456,9 +1456,9 @@ func (t *Topology) removeID(nodeID string) bool { return false } - copy(t.NodeIDs[i:], t.NodeIDs[i+1:]) - t.NodeIDs[len(t.NodeIDs)-1] = "" - t.NodeIDs = t.NodeIDs[:len(t.NodeIDs)-1] + copy(t.nodeIDs[i:], t.nodeIDs[i+1:]) + t.nodeIDs[len(t.nodeIDs)-1] = "" + t.nodeIDs = t.nodeIDs[:len(t.nodeIDs)-1] return true } @@ -1519,13 +1519,13 @@ func (c *cluster) considerTopology() error { } // If there is no .topology file, it's safe to proceed. - if len(c.Topology.NodeIDs) == 0 { + if len(c.Topology.nodeIDs) == 0 { return nil } // The local node (coordinator) must be in the .topology. if !c.Topology.ContainsID(c.Node.ID) { - return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.NodeIDs) + return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.nodeIDs) } // If local node is the only thing in .topology, continue. @@ -1775,7 +1775,7 @@ func encodeTopology(topology *Topology) *internal.Topology { } return &internal.Topology{ ClusterID: topology.ClusterID, - NodeIDs: topology.NodeIDs, + NodeIDs: topology.nodeIDs, } } @@ -1786,10 +1786,10 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { t := newTopology() t.ClusterID = topology.ClusterID - t.NodeIDs = topology.NodeIDs - sort.Slice(t.NodeIDs, + t.nodeIDs = topology.NodeIDs + sort.Slice(t.nodeIDs, func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] + return t.nodeIDs[i] < t.nodeIDs[j] }) return t, nil diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 06e7dd3d7..5b514e517 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -536,12 +536,12 @@ func TestCluster_ResizeStates(t *testing.T) { } expectedTop := &Topology{ - NodeIDs: []string{node.Node.ID}, + nodeIDs: []string{node.Node.ID}, } // Verify topology file. - if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) + if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs) } // Close TestCluster. @@ -558,7 +558,7 @@ func TestCluster_ResizeStates(t *testing.T) { // write topology to data file top := &Topology{ - NodeIDs: []string{node.Node.ID}, + nodeIDs: []string{node.Node.ID}, } tc.WriteTopology(node.Path, top) @@ -586,7 +586,7 @@ func TestCluster_ResizeStates(t *testing.T) { // write topology to data file top := &Topology{ - NodeIDs: []string{"some-other-host"}, + nodeIDs: []string{"some-other-host"}, } tc.WriteTopology(node.Path, top) @@ -625,14 +625,14 @@ func TestCluster_ResizeStates(t *testing.T) { } expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + nodeIDs: []string{node0.Node.ID, node1.Node.ID}, } // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs) + } else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs) } // Close TestCluster. @@ -648,7 +648,7 @@ func TestCluster_ResizeStates(t *testing.T) { // write topology to data file top := &Topology{ - NodeIDs: []string{"node0", "node2"}, + nodeIDs: []string{"node0", "node2"}, } tc.WriteTopology(node0.Path, top) @@ -721,14 +721,14 @@ func TestCluster_ResizeStates(t *testing.T) { } expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + nodeIDs: []string{node0.Node.ID, node1.Node.ID}, } // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs) + } else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs) } // Bits From 1af66417e14ef3b8a1946edbbe06a01d55eeb8f8 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:54:49 -0500 Subject: [PATCH 074/166] Unexport Topology.ClusterID --- cluster.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster.go b/cluster.go index dd61d0a9f..a36b2c92e 100644 --- a/cluster.go +++ b/cluster.go @@ -364,7 +364,7 @@ func (c *cluster) setID(id string) { c.id = id // Make sure the Topology is updated. - c.Topology.ClusterID = c.id + c.Topology.clusterID = c.id } func (c *cluster) State() string { @@ -818,7 +818,7 @@ func (c *cluster) setup() error { return errors.Wrap(err, "loading topology") } - c.id = c.Topology.ClusterID + c.id = c.Topology.clusterID // Only the coordinator needs to consider the .topology file. if c.isCoordinator() { @@ -1396,7 +1396,7 @@ type Topology struct { mu sync.RWMutex nodeIDs []string - ClusterID string + clusterID string // nodeStates holds the state of each node according to // the coordinator. Used during startup and data load. @@ -1511,7 +1511,7 @@ func (c *cluster) considerTopology() error { if c.id == "" { u := uuid.NewV4() c.id = u.String() - c.Topology.ClusterID = c.id + c.Topology.clusterID = c.id } if c.Static { @@ -1774,7 +1774,7 @@ func encodeTopology(topology *Topology) *internal.Topology { return nil } return &internal.Topology{ - ClusterID: topology.ClusterID, + ClusterID: topology.clusterID, NodeIDs: topology.nodeIDs, } } @@ -1785,7 +1785,7 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { } t := newTopology() - t.ClusterID = topology.ClusterID + t.clusterID = topology.ClusterID t.nodeIDs = topology.NodeIDs sort.Slice(t.nodeIDs, func(i, j int) bool { From ed0372b1d002555c6afbcc9704e1cc23f35d53d4 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:54:55 -0500 Subject: [PATCH 075/166] Unexport TranslateFile.IsReadOnly --- translate.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/translate.go b/translate.go index 04922b7a6..4b56f83c3 100644 --- a/translate.go +++ b/translate.go @@ -150,8 +150,8 @@ func (s *TranslateFile) Size() int64 { return n } -// IsReadOnly returns true if this store is being replicated from a primary store. -func (s *TranslateFile) IsReadOnly() bool { +// isReadOnly returns true if this store is being replicated from a primary store. +func (s *TranslateFile) isReadOnly() bool { return s.PrimaryTranslateStore != nil } @@ -351,7 +351,7 @@ func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string) s.mu.RUnlock() // Return error if not all values could be translated and this store is read-only. - if s.IsReadOnly() { + if s.isReadOnly() { return ret, ErrTranslateStoreReadOnly } @@ -457,7 +457,7 @@ func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []stri s.mu.RUnlock() // Return error if not all values could be translated and this store is read-only. - if s.IsReadOnly() { + if s.isReadOnly() { return ret, ErrTranslateStoreReadOnly } From 4c2ba7b7d3723f8cee318e50bfca96ccc5606f4b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:03 -0500 Subject: [PATCH 076/166] Unexport TranslateFile.Size --- translate.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/translate.go b/translate.go index 4b56f83c3..559e90bcb 100644 --- a/translate.go +++ b/translate.go @@ -142,8 +142,8 @@ func (s *TranslateFile) Closing() <-chan struct{} { return s.closing } -// Size returns the number of bytes in use in the data file. -func (s *TranslateFile) Size() int64 { +// size returns the number of bytes in use in the data file. +func (s *TranslateFile) size() int64 { s.mu.RLock() n := s.n s.mu.RUnlock() @@ -277,7 +277,7 @@ func (s *TranslateFile) monitorReplication() { } func (s *TranslateFile) replicate(ctx context.Context) error { - off := s.Size() + off := s.size() // Connect to remote primary. log.Printf("pilosa: replicating from offset %d", off) @@ -967,7 +967,7 @@ func (r *TranslateFileReader) Read(p []byte) (n int, err error) { // read writes the bytes for zero or more valid entries to p. func (r *TranslateFileReader) read(p []byte) (n int, err error) { - sz := r.store.Size() + sz := r.store.size() // Exit if there is no new data. if sz < r.offset { From 7a1d2c69804b46621cd1da62c485419f180df28e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:09 -0500 Subject: [PATCH 077/166] Unexport TranslateFile.MapSize --- translate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.go b/translate.go index 559e90bcb..20c886cdf 100644 --- a/translate.go +++ b/translate.go @@ -67,7 +67,7 @@ type TranslateFile struct { rows map[frameKey]*index Path string - MapSize int + mapSize int // If non-nil, data is streamed from a primary and this is a read-only store. PrimaryTranslateStore TranslateStore @@ -84,7 +84,7 @@ func NewTranslateFile() *TranslateFile { cols: make(map[string]*index), rows: make(map[frameKey]*index), - MapSize: defaultMapSize, + mapSize: defaultMapSize, ReplicationRetryInterval: defaultReplicationRetryInterval, } @@ -100,7 +100,7 @@ func (s *TranslateFile) Open() (err error) { s.w = bufio.NewWriter(s.file) // Memory map data file. - if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.MapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { return err } From e4847c498a136d8df62913944eca5bc013c4341a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:17 -0500 Subject: [PATCH 078/166] Unexport TranslateFile.ReplicationRetryInterval --- translate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translate.go b/translate.go index 20c886cdf..b75b9767a 100644 --- a/translate.go +++ b/translate.go @@ -73,7 +73,7 @@ type TranslateFile struct { PrimaryTranslateStore TranslateStore // Delay after attempting to connect to a primary that the store will retry. - ReplicationRetryInterval time.Duration + replicationRetryInterval time.Duration } // NewTranslateFile returns a new instance of TranslateFile. @@ -86,7 +86,7 @@ func NewTranslateFile() *TranslateFile { mapSize: defaultMapSize, - ReplicationRetryInterval: defaultReplicationRetryInterval, + replicationRetryInterval: defaultReplicationRetryInterval, } } @@ -270,7 +270,7 @@ func (s *TranslateFile) monitorReplication() { select { case <-s.closing: return - case <-time.After(s.ReplicationRetryInterval): + case <-time.After(s.replicationRetryInterval): log.Printf("pilosa: reconnecting to primary replica") } } From a7c1795cf7b9f8eb697c4ef9d0833abcdbdfdbae Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:25 -0500 Subject: [PATCH 079/166] Unexport TranslateFileReader --- translate.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/translate.go b/translate.go index b75b9767a..2341efe51 100644 --- a/translate.go +++ b/translate.go @@ -898,8 +898,8 @@ func pow2(v uint64) uint64 { panic("unreachable") } -// TranslateFileReader implements a reader that continuously streams data from a store. -type TranslateFileReader struct { +// translateFileReader implements a reader that continuously streams data from a store. +type translateFileReader struct { ctx context.Context store *TranslateFile file *os.File @@ -911,8 +911,8 @@ type TranslateFileReader struct { } // newTranslateFileReader returns a new instance of TranslateFileReader. -func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { - return &TranslateFileReader{ +func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *translateFileReader { + return &translateFileReader{ ctx: ctx, store: store, offset: offset, @@ -922,7 +922,7 @@ func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset in } // Open initializes the reader. -func (r *TranslateFileReader) Open() (err error) { +func (r *translateFileReader) Open() (err error) { if r.file, err = os.Open(r.store.Path); err != nil { return err } @@ -930,7 +930,7 @@ func (r *TranslateFileReader) Open() (err error) { } // Close closes the underlying file reader. -func (r *TranslateFileReader) Close() error { +func (r *translateFileReader) Close() error { r.once.Do(func() { close(r.closing) }) if r.file != nil { @@ -941,7 +941,7 @@ func (r *TranslateFileReader) Close() error { // Read reads the next section of the available data to p. This should always // read from the start of an entry and read n bytes to the end of another entry. -func (r *TranslateFileReader) Read(p []byte) (n int, err error) { +func (r *translateFileReader) Read(p []byte) (n int, err error) { for { // Obtain notification channel before we check for new data. notify := r.store.WriteNotify() @@ -966,7 +966,7 @@ func (r *TranslateFileReader) Read(p []byte) (n int, err error) { } // read writes the bytes for zero or more valid entries to p. -func (r *TranslateFileReader) read(p []byte) (n int, err error) { +func (r *translateFileReader) read(p []byte) (n int, err error) { sz := r.store.size() // Exit if there is no new data. From 6f256e0edbcf4613190dbad66eae1db7591fb78d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:31 -0500 Subject: [PATCH 080/166] Unexport URI.Normalize --- uri.go | 6 +++--- uri_internal_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uri.go b/uri.go index 332166b4e..605e5cb82 100644 --- a/uri.go +++ b/uri.go @@ -117,8 +117,8 @@ func (u *URI) HostPort() string { return s } -// Normalize returns the address in a form usable by a HTTP client. -func (u *URI) Normalize() string { +// normalize returns the address in a form usable by a HTTP client. +func (u *URI) normalize() string { scheme := u.Scheme index := strings.Index(scheme, "+") if index >= 0 { @@ -134,7 +134,7 @@ func (u URI) String() string { // Path returns URI with path func (u *URI) Path(path string) string { - return fmt.Sprintf("%s%s", u.Normalize(), path) + return fmt.Sprintf("%s%s", u.normalize(), path) } // The following methods are required to implement pflag Value interface. diff --git a/uri_internal_test.go b/uri_internal_test.go index 64db3fd8e..4d2500010 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -60,7 +60,7 @@ func TestNormalizedAddress(t *testing.T) { if err != nil { t.Fatalf("Can't parse address") } - if uri.Normalize() != "http://big-data.pilosa.com:6888" { + if uri.normalize() != "http://big-data.pilosa.com:6888" { t.Fatalf("Normalized address is not normal") } } From ac83bf44225e649fd89b0795b06908e5bee55fbe Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:38 -0500 Subject: [PATCH 081/166] Unexport URI.SetHost --- uri.go | 6 +++--- uri_internal_test.go | 4 ++-- utils_internal_test.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/uri.go b/uri.go index 605e5cb82..401ec1ad7 100644 --- a/uri.go +++ b/uri.go @@ -69,7 +69,7 @@ func (u URIs) HostPortStrings() []string { // NewURIFromHostPort returns a URI with specified host and port. func NewURIFromHostPort(host string, port uint16) (*URI, error) { uri := defaultURI() - err := uri.SetHost(host) + err := uri.setHost(host) if err != nil { return nil, errors.Wrap(err, "setting uri host") } @@ -92,8 +92,8 @@ func (u *URI) SetScheme(scheme string) error { return nil } -// SetHost sets the host of this URI. -func (u *URI) SetHost(host string) error { +// setHost sets the host of this URI. +func (u *URI) setHost(host string) error { m := hostRegexp.FindStringSubmatch(host) if m == nil { return errors.New("invalid host") diff --git a/uri_internal_test.go b/uri_internal_test.go index 4d2500010..07da55226 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -91,7 +91,7 @@ func TestSetScheme(t *testing.T) { func TestSetHost(t *testing.T) { uri := defaultURI() target := "10.20.30.40" - err := uri.SetHost(target) + err := uri.setHost(target) if err != nil { t.Fatal(err) } @@ -119,7 +119,7 @@ func TestSetInvalidScheme(t *testing.T) { func TestSetInvalidHost(t *testing.T) { uri := defaultURI() - err := uri.SetHost("index?.pilosa.com") + err := uri.setHost("index?.pilosa.com") if err == nil { t.Fatalf("Should have failed") } diff --git a/utils_internal_test.go b/utils_internal_test.go index 1f58c464c..88b1473c0 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -57,14 +57,14 @@ func NewTestCluster(n int) *cluster { func NewTestURI(scheme, host string, port uint16) URI { uri := defaultURI() uri.SetScheme(scheme) - uri.SetHost(host) + uri.setHost(host) uri.SetPort(port) return *uri } func NewTestURIFromHostPort(host string, port uint16) URI { uri := defaultURI() - uri.SetHost(host) + uri.setHost(host) uri.SetPort(port) return *uri } From 7807b92b133d61830563972f3bba4eabba9df8c5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:55:45 -0500 Subject: [PATCH 082/166] Unexport URI.SetScheme --- uri.go | 4 ++-- uri_internal_test.go | 4 ++-- utils_internal_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/uri.go b/uri.go index 401ec1ad7..8f9df2b72 100644 --- a/uri.go +++ b/uri.go @@ -82,8 +82,8 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// SetScheme sets the scheme of this URI. -func (u *URI) SetScheme(scheme string) error { +// setScheme sets the scheme of this URI. +func (u *URI) setScheme(scheme string) error { m := schemeRegexp.FindStringSubmatch(scheme) if m == nil { return errors.New("invalid scheme") diff --git a/uri_internal_test.go b/uri_internal_test.go index 07da55226..9bc6403d7 100644 --- a/uri_internal_test.go +++ b/uri_internal_test.go @@ -79,7 +79,7 @@ func TestURIPath(t *testing.T) { func TestSetScheme(t *testing.T) { uri := defaultURI() target := "fun" - err := uri.SetScheme(target) + err := uri.setScheme(target) if err != nil { t.Fatal(err) } @@ -111,7 +111,7 @@ func TestSetPort(t *testing.T) { func TestSetInvalidScheme(t *testing.T) { uri := defaultURI() - err := uri.SetScheme("?invalid") + err := uri.setScheme("?invalid") if err == nil { t.Fatalf("Should have failed") } diff --git a/utils_internal_test.go b/utils_internal_test.go index 88b1473c0..bf2c4f01f 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -56,7 +56,7 @@ func NewTestCluster(n int) *cluster { // NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { uri := defaultURI() - uri.SetScheme(scheme) + uri.setScheme(scheme) uri.setHost(host) uri.SetPort(port) return *uri From cd3eac00d2c85dcdc130e77c16623fcd74072bc5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:04 -0500 Subject: [PATCH 083/166] Unexport ValCount.Add --- executor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 03d18f936..4ca0c7e36 100644 --- a/executor.go +++ b/executor.go @@ -234,7 +234,7 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(ValCount) - return other.Add(v.(ValCount)) + return other.add(v.(ValCount)) } result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -1713,7 +1713,7 @@ type ValCount struct { Count int64 `json:"count"` } -func (vc *ValCount) Add(other ValCount) ValCount { +func (vc *ValCount) add(other ValCount) ValCount { return ValCount{ Val: vc.Val + other.Val, Count: vc.Count + other.Count, From d7cebfa7c4a0c9914c84e9bf3ba3aceed8cee73a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:10 -0500 Subject: [PATCH 084/166] Unexport ValCount.Larger --- executor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 4ca0c7e36..5d8386e4c 100644 --- a/executor.go +++ b/executor.go @@ -300,7 +300,7 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(ValCount) - return other.Larger(v.(ValCount)) + return other.larger(v.(ValCount)) } result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -1731,8 +1731,8 @@ func (vc *ValCount) Smaller(other ValCount) ValCount { } } -// Larger returns the larger of the two ValCounts. -func (vc *ValCount) Larger(other ValCount) ValCount { +// larger returns the larger of the two ValCounts. +func (vc *ValCount) larger(other ValCount) ValCount { if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { return other } From ffde862679f7f07181a47aecfe2bb1e787a50861 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:16 -0500 Subject: [PATCH 085/166] Unexport ValCount.Smaller --- executor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 5d8386e4c..afea60112 100644 --- a/executor.go +++ b/executor.go @@ -267,7 +267,7 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(ValCount) - return other.Smaller(v.(ValCount)) + return other.smaller(v.(ValCount)) } result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -1720,8 +1720,8 @@ func (vc *ValCount) add(other ValCount) ValCount { } } -// Smaller returns the smaller of the two ValCounts. -func (vc *ValCount) Smaller(other ValCount) ValCount { +// smaller returns the smaller of the two ValCounts. +func (vc *ValCount) smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { return other } From d88f27552363ab9fb1a35346f5b0d165aedc1089 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:22 -0500 Subject: [PATCH 086/166] Unexport VerboseLogger --- logger.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/logger.go b/logger.go index 6afa24c96..28b35b999 100644 --- a/logger.go +++ b/logger.go @@ -64,25 +64,25 @@ func (s *standardLogger) Logger() *log.Logger { return s.logger } -// VerboseLogger is an implementation of pilosa.Logger which includes debug messages. -type VerboseLogger struct { +// verboseLogger is an implementation of pilosa.Logger which includes debug messages. +type verboseLogger struct { logger *log.Logger } -func NewVerboseLogger(w io.Writer) *VerboseLogger { - return &VerboseLogger{ +func NewVerboseLogger(w io.Writer) *verboseLogger { + return &verboseLogger{ logger: log.New(w, "", log.LstdFlags), } } -func (vb *VerboseLogger) Printf(format string, v ...interface{}) { +func (vb *verboseLogger) Printf(format string, v ...interface{}) { vb.logger.Printf(format, v...) } -func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { +func (vb *verboseLogger) Debugf(format string, v ...interface{}) { vb.logger.Printf(format, v...) } -func (vb *VerboseLogger) Logger() *log.Logger { +func (vb *verboseLogger) Logger() *log.Logger { return vb.logger } From 817ede49e6421c92f3499f86690d04bfa3f0bec6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:41 -0500 Subject: [PATCH 087/166] Unexport boltdb.AttrBlockSize --- boltdb/attrstore.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 3604de5b6..347fc74e4 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -30,8 +30,8 @@ import ( "github.com/pkg/errors" ) -// AttrBlockSize is the size of attribute blocks for anti-entropy. -const AttrBlockSize = 100 +// attrBlockSize is the size of attribute blocks for anti-entropy. +const attrBlockSize = 100 // AttrCache represents a cache for attributes. type AttrCache struct { @@ -228,7 +228,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { defer tx.Rollback() // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize) + cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) // Iterate over each block. var blocks []pilosa.AttrBlock @@ -262,8 +262,8 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro defer tx.Rollback() // Move to the start of the block. - min := u64tob(uint64(i) * AttrBlockSize) - max := u64tob(uint64(i+1) * AttrBlockSize) + min := u64tob(uint64(i) * attrBlockSize) + max := u64tob(uint64(i+1) * attrBlockSize) cur := tx.Bucket([]byte("attrs")).Cursor() for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { // Exit if we're past the end of the block. From 43cac45d406410c01f9fd8409b6dd7eeccb19af0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:46 -0500 Subject: [PATCH 088/166] Unexport boltdb.AttrCache --- boltdb/attrstore.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 347fc74e4..40d478f3a 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -33,14 +33,14 @@ import ( // attrBlockSize is the size of attribute blocks for anti-entropy. const attrBlockSize = 100 -// AttrCache represents a cache for attributes. -type AttrCache struct { +// attrCache represents a cache for attributes. +type attrCache struct { mu sync.RWMutex attrs map[uint64]map[string]interface{} } // Get returns the cached attributes for a given id. -func (c *AttrCache) Get(id uint64) map[string]interface{} { +func (c *attrCache) Get(id uint64) map[string]interface{} { c.mu.RLock() defer c.mu.RUnlock() attrs := c.attrs[id] @@ -57,7 +57,7 @@ func (c *AttrCache) Get(id uint64) map[string]interface{} { } // Set updates the cached attributes for a given id. -func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { +func (c *attrCache) Set(id uint64, attrs map[string]interface{}) { c.mu.Lock() defer c.mu.Unlock() c.attrs[id] = attrs @@ -68,12 +68,12 @@ type AttrStore struct { mu sync.RWMutex path string db *bolt.DB - attrCache *AttrCache + attrCache *attrCache } // NewAttrCache returns a new instance of AttrCache. -func NewAttrCache() *AttrCache { - return &AttrCache{ +func NewAttrCache() *attrCache { + return &attrCache{ attrs: make(map[uint64]map[string]interface{}), } } From fcf6517c7e15319a2ad0a398246e88cda5e69f16 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:51 -0500 Subject: [PATCH 089/166] Unexport boltdb.AttrStore --- boltdb/attrstore.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 40d478f3a..3849e0964 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -63,8 +63,8 @@ func (c *attrCache) Set(id uint64, attrs map[string]interface{}) { c.attrs[id] = attrs } -// AttrStore represents a storage layer for attributes. -type AttrStore struct { +// attrStore represents a storage layer for attributes. +type attrStore struct { mu sync.RWMutex path string db *bolt.DB @@ -80,17 +80,17 @@ func NewAttrCache() *attrCache { // NewAttrStore returns a new instance of AttrStore. func NewAttrStore(path string) pilosa.AttrStore { - return &AttrStore{ + return &attrStore{ path: path, attrCache: NewAttrCache(), } } // Path returns path to the store's data file. -func (s *AttrStore) Path() string { return s.path } +func (s *attrStore) Path() string { return s.path } // Open opens and initializes the store. -func (s *AttrStore) Open() error { +func (s *attrStore) Open() error { // Open storage. db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { @@ -112,7 +112,7 @@ func (s *AttrStore) Open() error { } // Close closes the store. -func (s *AttrStore) Close() error { +func (s *attrStore) Close() error { if s.db != nil { s.db.Close() } @@ -120,7 +120,7 @@ func (s *AttrStore) Close() error { } // Attrs returns a set of attributes by ID. -func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { +func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) { s.mu.RLock() defer s.mu.RUnlock() @@ -147,7 +147,7 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { } // SetAttrs sets attribute values for a given ID. -func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { +func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error { // Ignore empty maps. if len(m) == 0 { return nil @@ -184,7 +184,7 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { } // SetBulkAttrs sets attribute values for a set of ids. -func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { +func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { s.mu.Lock() defer s.mu.Unlock() @@ -220,7 +220,7 @@ func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } // Blocks returns a list of all blocks in the store. -func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { +func (s *attrStore) Blocks() ([]pilosa.AttrBlock, error) { tx, err := s.db.Begin(false) if err != nil { return nil, errors.Wrap(err, "starting transaction") @@ -251,7 +251,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { } // BlockData returns all data for a single block. -func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { +func (s *attrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { m := make(map[uint64]map[string]interface{}) // Start read-only transaction. From 9db210927838c29993b722a08989231ffb7eab9f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:56:57 -0500 Subject: [PATCH 090/166] Unexport boltdb.NewAttrCache --- boltdb/attrstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 3849e0964..1a9ddcf6e 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -71,8 +71,8 @@ type attrStore struct { attrCache *attrCache } -// NewAttrCache returns a new instance of AttrCache. -func NewAttrCache() *attrCache { +// newAttrCache returns a new instance of AttrCache. +func newAttrCache() *attrCache { return &attrCache{ attrs: make(map[uint64]map[string]interface{}), } @@ -82,7 +82,7 @@ func NewAttrCache() *attrCache { func NewAttrStore(path string) pilosa.AttrStore { return &attrStore{ path: path, - attrCache: NewAttrCache(), + attrCache: newAttrCache(), } } From 4c3600f1c4e1737d788b2a3f1f7fbe05b3512d68 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:03 -0500 Subject: [PATCH 091/166] Unexport cmd.Checker --- cmd/check.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/check.go b/cmd/check.go index 56ae13265..f4b53231a 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -25,10 +25,10 @@ import ( "github.com/pilosa/pilosa/ctl" ) -var Checker *ctl.CheckCommand +var checker *ctl.CheckCommand func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) + checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) checkCmd := &cobra.Command{ Use: "check [path2]...", Short: "Do a consistency check on a pilosa data file.", @@ -39,8 +39,8 @@ Performs a consistency check on data files. if len(args) == 0 { return fmt.Errorf("path required") } - Checker.Paths = args - if err := Checker.Run(context.Background()); err != nil { + checker.Paths = args + if err := checker.Run(context.Background()); err != nil { return err } return nil From 618f6c8af45ec081f5ed5037749d299042ca1d11 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:10 -0500 Subject: [PATCH 092/166] Unexport cmd.Conf --- cmd/config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 9452cbc3c..953074f9c 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -25,10 +25,10 @@ import ( "github.com/pilosa/pilosa/server" ) -var Conf *ctl.ConfigCommand +var conf *ctl.ConfigCommand func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) + conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) Server := server.NewCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "config", @@ -36,8 +36,8 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command Long: `config prints the current configuration to stdout`, RunE: func(cmd *cobra.Command, args []string) error { - Conf.Config = Server.Config - if err := Conf.Run(context.Background()); err != nil { + conf.Config = Server.Config + if err := conf.Run(context.Background()); err != nil { return err } return nil From 35ae352599b65be45181b9901fe8e8d46dadd857 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:16 -0500 Subject: [PATCH 093/166] Unexport cmd.GenerateConf --- cmd/generate_config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/generate_config.go b/cmd/generate_config.go index b0622b64d..346d38b56 100644 --- a/cmd/generate_config.go +++ b/cmd/generate_config.go @@ -24,17 +24,17 @@ import ( "github.com/pilosa/pilosa/ctl" ) -var GenerateConf *ctl.GenerateConfigCommand +var generateConf *ctl.GenerateConfigCommand func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - GenerateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) + generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) confCmd := &cobra.Command{ Use: "generate-config", Short: "Print the default configuration.", Long: `generate-config prints the default configuration to stdout `, RunE: func(cmd *cobra.Command, args []string) error { - if err := GenerateConf.Run(context.Background()); err != nil { + if err := generateConf.Run(context.Background()); err != nil { return err } return nil From 1d941efbb22dfc38b18bc55b14984f3f43e462f1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:22 -0500 Subject: [PATCH 094/166] Unexport cmd.Inspector --- cmd/inspect.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index e8526a414..08eacb441 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -25,10 +25,10 @@ import ( "github.com/pilosa/pilosa/ctl" ) -var Inspector *ctl.InspectCommand +var inspector *ctl.InspectCommand func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) + inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) inspectCmd := &cobra.Command{ Use: "inspect", @@ -42,8 +42,8 @@ Inspects a data file and provides stats. } else if len(args) > 1 { return fmt.Errorf("only one path allowed") } - Inspector.Path = args[0] - if err := Inspector.Run(context.Background()); err != nil { + inspector.Path = args[0] + if err := inspector.Run(context.Background()); err != nil { return err } return nil From 004f1c7df105e737c79969772323bbff9ca2ce20 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:29 -0500 Subject: [PATCH 095/166] Unexport cmd.NewCheckCommand --- cmd/check.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/check.go b/cmd/check.go index f4b53231a..8785a78e1 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -27,7 +27,7 @@ import ( var checker *ctl.CheckCommand -func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +func newCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) checkCmd := &cobra.Command{ Use: "check [path2]...", @@ -50,5 +50,5 @@ Performs a consistency check on data files. } func init() { - subcommandFns["check"] = NewCheckCommand + subcommandFns["check"] = newCheckCommand } From b42c24eaceb3cba6f80f4146af0ee33d79716e8f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:35 -0500 Subject: [PATCH 096/166] Unexport cmd.NewConfigCommand --- cmd/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 953074f9c..3d65fa131 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -27,7 +27,7 @@ import ( var conf *ctl.ConfigCommand -func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +func newConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) Server := server.NewCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ @@ -51,5 +51,5 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command } func init() { - subcommandFns["config"] = NewConfigCommand + subcommandFns["config"] = newConfigCommand } From da4e3b0e140528b224efa3f3ff87f32ca1d03ceb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:41 -0500 Subject: [PATCH 097/166] Unexport cmd.NewExportCommand --- cmd/export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index 9613d9544..d0f63edbf 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -26,7 +26,7 @@ import ( var Exporter *ctl.ExportCommand -func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) exportCmd := &cobra.Command{ Use: "export", @@ -60,5 +60,5 @@ The file does not contain any headers. } func init() { - subcommandFns["export"] = NewExportCommand + subcommandFns["export"] = newExportCommand } From 4ec7a05524fc6cfabec824d69fcff200a06e36b1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:48 -0500 Subject: [PATCH 098/166] Unexport cmd.NewGenerateConfigCommand --- cmd/generate_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/generate_config.go b/cmd/generate_config.go index 346d38b56..0b5b81462 100644 --- a/cmd/generate_config.go +++ b/cmd/generate_config.go @@ -26,7 +26,7 @@ import ( var generateConf *ctl.GenerateConfigCommand -func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +func newGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) confCmd := &cobra.Command{ Use: "generate-config", @@ -45,5 +45,5 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra. } func init() { - subcommandFns["generate-config"] = NewGenerateConfigCommand + subcommandFns["generate-config"] = newGenerateConfigCommand } From c86758e998b5661925797463a5d367329bc79c57 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:57:54 -0500 Subject: [PATCH 099/166] Unexport cmd.NewImportCommand --- cmd/import.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 4d565adb7..7b4d00efd 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -25,8 +25,8 @@ import ( var Importer *ctl.ImportCommand -// NewImportCommand runs the Pilosa import subcommand for ingesting bulk data. -func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +// newImportCommand runs the Pilosa import subcommand for ingesting bulk data. +func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Importer = ctl.NewImportCommand(stdin, stdout, stderr) importCmd := &cobra.Command{ Use: "import", @@ -67,5 +67,5 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. } func init() { - subcommandFns["import"] = NewImportCommand + subcommandFns["import"] = newImportCommand } From 18fd2e14c54da495a37b0c37e7ae702ae37fbe66 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:00 -0500 Subject: [PATCH 100/166] Unexport cmd.NewInspectCommand --- cmd/inspect.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index 08eacb441..096787337 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -27,7 +27,7 @@ import ( var inspector *ctl.InspectCommand -func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) inspectCmd := &cobra.Command{ @@ -53,5 +53,5 @@ Inspects a data file and provides stats. } func init() { - subcommandFns["inspect"] = NewInspectCommand + subcommandFns["inspect"] = newInspectCommand } From 73a7588e1b4bbf8b19d7d638683ee1e5df7b64f5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:06 -0500 Subject: [PATCH 101/166] Unexport cmd.NewServeCmd --- cmd/server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index d906cf189..83d18504c 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -27,8 +27,8 @@ import ( // Server is global so that tests can control and verify it. var Server *server.Command -// NewServeCmd creates a pilosa server and runs it with command line flags. -func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +// newServeCmd creates a pilosa server and runs it with command line flags. +func newServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Server = server.NewCommand(stdin, stdout, stderr) serveCmd := &cobra.Command{ Use: "server", @@ -52,5 +52,5 @@ on the configured port.`, } func init() { - subcommandFns["server"] = NewServeCmd + subcommandFns["server"] = newServeCmd } From 7185c0f79139e2a43d74b5e2af1ec94b7d1f71c4 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:12 -0500 Subject: [PATCH 102/166] Unexport ctl.CommandClient --- ctl/common.go | 4 ++-- ctl/export.go | 2 +- ctl/import.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ctl/common.go b/ctl/common.go index 44e01d406..f7f429c58 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -36,8 +36,8 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)") } -// CommandClient returns a pilosa.InternalHTTPClient for the command -func CommandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) { +// commandClient returns a pilosa.InternalHTTPClient for the command +func commandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) { tlsConfig := cmd.TLSConfiguration() var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { diff --git a/ctl/export.go b/ctl/export.go index 5bfbd529f..4a20b0cc2 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -75,7 +75,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Create a client to the server. - client, err := CommandClient(cmd) + client, err := commandClient(cmd) if err != nil { return errors.Wrap(err, "creating client") } diff --git a/ctl/import.go b/ctl/import.go index 370425115..83763f3ab 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -89,7 +89,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { return errors.New("path required") } // Create a client to the server. - client, err := CommandClient(cmd) + client, err := commandClient(cmd) if err != nil { return errors.Wrap(err, "creating client") } From f9a792ea49de1001f8d639efbe787d8dfca111d9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:17 -0500 Subject: [PATCH 103/166] Unexport ctl.ImportCommand.IndexOptions --- ctl/import.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctl/import.go b/ctl/import.go index 83763f3ab..52d0c0c68 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -41,7 +41,7 @@ type ImportCommand struct { Field string `json:"field"` // Options for index & field to be created if they don't exist - IndexOptions pilosa.IndexOptions + indexOptions pilosa.IndexOptions // CreateSchema ensures the schema exists before import CreateSchema bool @@ -130,7 +130,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { - err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions) + err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions) if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } From eea29f664fc9ce9d7814c25d6079985460d64471 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:24 -0500 Subject: [PATCH 104/166] Unexport ctl.ImportCommand.Client --- ctl/import.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ctl/import.go b/ctl/import.go index 52d0c0c68..76b794a93 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -59,7 +59,7 @@ type ImportCommand struct { Sort bool `json:"sort"` // Reusable client. - Client pilosa.InternalClient `json:"-"` + client pilosa.InternalClient `json:"-"` // Standard input/output *pilosa.CmdIO @@ -93,7 +93,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { if err != nil { return errors.Wrap(err, "creating client") } - cmd.Client = client + cmd.client = client if cmd.CreateSchema { err := cmd.ensureSchema(ctx) @@ -104,7 +104,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Determine the field type in order to correctly handle the input data. fieldType := pilosa.DefaultFieldType - schema, err := cmd.Client.Schema(ctx) + schema, err := cmd.client.Schema(ctx) if err != nil { return errors.Wrap(err, "getting schema") } @@ -130,11 +130,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { - err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions) + err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions) if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Field) + err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field) if err != nil { return fmt.Errorf("Error Creating Field: %s", err) } @@ -254,7 +254,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err } logger.Printf("importing shard: %d, n=%d", shard, len(chunk)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil { + if err := cmd.client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil { return errors.Wrap(err, "importing") } } @@ -351,7 +351,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er // TODO: does it help to sort the rowKeys? logger.Printf("importing keys: n=%d", len(bits)) - if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil { + if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil { return errors.Wrap(err, "importing keys") } @@ -448,7 +448,7 @@ func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldV } logger.Printf("importing shard: %d, n=%d", shard, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil { + if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil { return errors.Wrap(err, "importing values") } } From dc97e34fb77da0d775b40b9d78f86b108c109959 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:30 -0500 Subject: [PATCH 105/166] Unexport proto.EncodeColumnAttrSet --- encoding/proto/proto.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 0020260ab..e8ccabe5e 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -953,12 +953,12 @@ func decodeValCount(pb *internal.ValCount) pilosa.ValCount { func EncodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { - other[i] = EncodeColumnAttrSet(a[i]) + other[i] = encodeColumnAttrSet(a[i]) } return other } -func EncodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { +func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { return &internal.ColumnAttrSet{ ID: set.ID, Attrs: encodeAttrs(set.Attrs), From 17a0bb62b79162b0e14ff81e98fda414b41d4664 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:37 -0500 Subject: [PATCH 106/166] Unexport proto.EncodeColumnAttrSets --- encoding/proto/proto.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index e8ccabe5e..cfa16e98d 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -337,7 +337,7 @@ func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { pb := &internal.QueryResponse{ Results: make([]*internal.QueryResult, len(m.Results)), - ColumnAttrSets: EncodeColumnAttrSets(m.ColumnAttrSets), + ColumnAttrSets: encodeColumnAttrSets(m.ColumnAttrSets), } for i := range m.Results { @@ -950,7 +950,7 @@ func decodeValCount(pb *internal.ValCount) pilosa.ValCount { } } -func EncodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { +func encodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { other[i] = encodeColumnAttrSet(a[i]) From 052355014d3a4d2b63a46d9662e1d9d057065bb9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:43 -0500 Subject: [PATCH 107/166] Unexport proto.EncodeNodes --- encoding/proto/proto.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index cfa16e98d..b0fe9d813 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -457,8 +457,8 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { } } -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*pilosa.Node) []*internal.Node { +// encodeNodes converts a slice of Nodes into its internal representation. +func encodeNodes(a []*pilosa.Node) []*internal.Node { other := make([]*internal.Node, len(a)) for i := range a { other[i] = encodeNode(a[i]) @@ -487,7 +487,7 @@ func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { return &internal.ClusterStatus{ State: m.State, ClusterID: m.ClusterID, - Nodes: EncodeNodes(m.Nodes), + Nodes: encodeNodes(m.Nodes), } } From bafc170420ccc0c59f52d69b48c1ff719b430844 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:48 -0500 Subject: [PATCH 108/166] Unexport proto.EncodePairs --- encoding/proto/proto.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index b0fe9d813..f6c1fd2d9 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -349,7 +349,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { pb.Results[i].Row = EncodeRow(result) case []pilosa.Pair: pb.Results[i].Type = queryResultTypePairs - pb.Results[i].Pairs = EncodePairs(result) + pb.Results[i].Pairs = encodePairs(result) case pilosa.ValCount: pb.Results[i].Type = queryResultTypeValCount pb.Results[i].ValCount = EncodeValCount(result) @@ -976,7 +976,7 @@ func EncodeRow(r *pilosa.Row) *internal.Row { } } -func EncodePairs(a pilosa.Pairs) []*internal.Pair { +func encodePairs(a pilosa.Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { other[i] = encodePair(a[i]) From b1d968f95e9748b585de720cdd74307f7c8d3312 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:54 -0500 Subject: [PATCH 109/166] Unexport proto.EncodeRow --- encoding/proto/proto.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index f6c1fd2d9..55983a702 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -346,7 +346,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { switch result := m.Results[i].(type) { case *pilosa.Row: pb.Results[i].Type = queryResultTypeRow - pb.Results[i].Row = EncodeRow(result) + pb.Results[i].Row = encodeRow(result) case []pilosa.Pair: pb.Results[i].Type = queryResultTypePairs pb.Results[i].Pairs = encodePairs(result) @@ -965,7 +965,7 @@ func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { } } -func EncodeRow(r *pilosa.Row) *internal.Row { +func encodeRow(r *pilosa.Row) *internal.Row { if r == nil { return nil } From c9497ec612aa5ea2a3465eac045acf4aa820299a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:58:59 -0500 Subject: [PATCH 110/166] Unexport proto.EncodeValCount --- encoding/proto/proto.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 55983a702..d8367c468 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -352,7 +352,7 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { pb.Results[i].Pairs = encodePairs(result) case pilosa.ValCount: pb.Results[i].Type = queryResultTypeValCount - pb.Results[i].ValCount = EncodeValCount(result) + pb.Results[i].ValCount = encodeValCount(result) case uint64: pb.Results[i].Type = queryResultTypeUint64 pb.Results[i].N = result @@ -992,7 +992,7 @@ func encodePair(p pilosa.Pair) *internal.Pair { } } -func EncodeValCount(vc pilosa.ValCount) *internal.ValCount { +func encodeValCount(vc pilosa.ValCount) *internal.ValCount { return &internal.ValCount{ Val: vc.Val, Count: vc.Count, From 446bfff91a5d74119fcc95d01accf05a136cf874 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:18 -0500 Subject: [PATCH 111/166] Unexport b.BTreeContainers --- enterprise/b/containers_btree.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 71727700f..443cb71d9 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -27,15 +27,15 @@ func cmp(a, b uint64) int { return int(a - b) } -type BTreeContainers struct { +type bTreeContainers struct { tree *Tree lastKey uint64 lastContainer *roaring.Container } -func NewBTreeContainers() *BTreeContainers { - return &BTreeContainers{ +func NewBTreeContainers() *bTreeContainers { + return &bTreeContainers{ tree: TreeNew(cmp), } } @@ -48,7 +48,7 @@ func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { return b } -func (btc *BTreeContainers) Get(key uint64) *roaring.Container { +func (btc *bTreeContainers) Get(key uint64) *roaring.Container { // Check the last* cache for same container. if key == btc.lastKey && btc.lastContainer != nil { return btc.lastContainer @@ -64,7 +64,7 @@ func (btc *BTreeContainers) Get(key uint64) *roaring.Container { return c } -func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) { +func (btc *bTreeContainers) Put(key uint64, c *roaring.Container) { // If a mapped container is added to the tree, reset the // lastContainer cache so that the cache is not pointing // at a read-only mmap. @@ -93,16 +93,16 @@ type updater struct { mapped bool } -func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { a := updater{key, containerType, n, mapped} btc.tree.Put(key, a.update) } -func (btc *BTreeContainers) Remove(key uint64) { +func (btc *bTreeContainers) Remove(key uint64) { btc.tree.Delete(key) } -func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { +func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container { // Check the last* cache for same container. if key == btc.lastKey && btc.lastContainer != nil { return btc.lastContainer @@ -121,7 +121,7 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { return btc.lastContainer } -func (btc *BTreeContainers) Count() (n uint64) { +func (btc *bTreeContainers) Count() (n uint64) { e, _ := btc.tree.Seek(0) _, c, err := e.Next() for err != io.EOF { @@ -131,7 +131,7 @@ func (btc *BTreeContainers) Count() (n uint64) { return } -func (btc *BTreeContainers) Clone() roaring.Containers { +func (btc *bTreeContainers) Clone() roaring.Containers { nbtc := NewBTreeContainers() itr, err := btc.tree.SeekFirst() @@ -148,7 +148,7 @@ func (btc *BTreeContainers) Clone() roaring.Containers { return nbtc } -func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) { +func (btc *bTreeContainers) Last() (key uint64, c *roaring.Container) { if btc.tree.Len() == 0 { return 0, nil } @@ -156,17 +156,17 @@ func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) { return k, v } -func (btc *BTreeContainers) Size() int { +func (btc *bTreeContainers) Size() int { return btc.tree.Len() } -func (btc *BTreeContainers) Reset() { +func (btc *bTreeContainers) Reset() { btc.tree = TreeNew(cmp) btc.lastKey = 0 btc.lastContainer = nil } -func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { +func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { e, ok := btc.tree.Seek(key) if ok { found = true From eac1bec54b356e8600e31f4d045b909c511584b7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:20 -0500 Subject: [PATCH 112/166] Unexport b.Enumerator --- enterprise/b/btree.go | 32 ++++++++++++++++---------------- enterprise/b/containers_btree.go | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go index 3d4c09888..8d853083c 100644 --- a/enterprise/b/btree.go +++ b/enterprise/b/btree.go @@ -56,7 +56,7 @@ func init() { var ( btDPool = sync.Pool{New: func() interface{} { return &d{} }} - btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}} + btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}} btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} btXPool = sync.Pool{New: func() interface{} { return &x{} }} ) @@ -71,8 +71,8 @@ func (p *btTpool) get(cmp Cmp) *Tree { type btEpool struct{ sync.Pool } -func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *Enumerator { - x := p.Get().(*Enumerator) +func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *enumerator { + x := p.Get().(*enumerator) x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver return x } @@ -98,15 +98,15 @@ type ( v *roaring.Container } - // Enumerator captures the state of enumerating a tree. It is returned + // enumerator captures the state of enumerating a tree. It is returned // from the Seek* methods. The enumerator is aware of any mutations // made to the tree in the process of enumerating it and automatically // resumes the enumeration at the proper key, if possible. // - // However, once an Enumerator returns io.EOF to signal "no more + // However, once an enumerator returns io.EOF to signal "no more // items", it does no more attempt to "resync" on tree mutation(s). In - // other words, io.EOF from an Enumerator is "sticky" (idempotent). - Enumerator struct { + // other words, io.EOF from an enumerator is "sticky" (idempotent). + enumerator struct { err error hit bool i int @@ -140,7 +140,7 @@ type ( var ( // R/O zero values zd d zde de - ze Enumerator + ze enumerator zk uint64 zt Tree zx x @@ -528,7 +528,7 @@ func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { // Seek returns an Enumerator positioned on an item such that k >= item's key. // ok reports if k == item.key The Enumerator's position is possibly after the // last item in the tree. -func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) { +func (t *Tree) Seek(k uint64) (e *enumerator, ok bool) { q := t.r if q == nil { e = btEPool.get(nil, false, 0, k, nil, t, t.ver) @@ -558,7 +558,7 @@ func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) { // SeekFirst returns an enumerator positioned on the first KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekFirst() (e *Enumerator, err error) { +func (t *Tree) SeekFirst() (e *enumerator, err error) { q := t.first if q == nil { return nil, io.EOF @@ -569,7 +569,7 @@ func (t *Tree) SeekFirst() (e *Enumerator, err error) { // SeekLast returns an enumerator positioned on the last KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekLast() (e *Enumerator, err error) { +func (t *Tree) SeekLast() (e *enumerator, err error) { q := t.last if q == nil { return nil, io.EOF @@ -850,7 +850,7 @@ func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { // Close recycles e to a pool for possible later reuse. No references to e // should exist or such references must not be used afterwards. -func (e *Enumerator) Close() { +func (e *enumerator) Close() { *e = ze btEPool.Put(e) } @@ -858,7 +858,7 @@ func (e *Enumerator) Close() { // Next returns the currently enumerated item, if it exists and moves to the // next item in the key collation order. If there is no item to return, err == // io.EOF is returned. -func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) { +func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) { if err = e.err; err != nil { return } @@ -886,7 +886,7 @@ func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) { return } -func (e *Enumerator) next() error { +func (e *enumerator) next() error { if e.q == nil { e.err = io.EOF return io.EOF @@ -906,7 +906,7 @@ func (e *Enumerator) next() error { // Prev returns the currently enumerated item, if it exists and moves to the // previous item in the key collation order. If there is no item to return, err // == io.EOF is returned. -func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) { +func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) { if err = e.err; err != nil { return } @@ -941,7 +941,7 @@ func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) { return } -func (e *Enumerator) prev() error { +func (e *enumerator) prev() error { if e.q == nil { e.err = io.EOF return io.EOF diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 443cb71d9..4c8527c8b 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -178,7 +178,7 @@ func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterato } type btcIterator struct { - e *Enumerator + e *enumerator key uint64 val *roaring.Container } From fede4ac9f0af1a1c060b9847ac3fecc7cf46062a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:22 -0500 Subject: [PATCH 113/166] Unexport b.NewBTreeContainers --- enterprise/b/containers_btree.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 4c8527c8b..a61c862e6 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -34,7 +34,7 @@ type bTreeContainers struct { lastContainer *roaring.Container } -func NewBTreeContainers() *bTreeContainers { +func newBTreeContainers() *bTreeContainers { return &bTreeContainers{ tree: TreeNew(cmp), } @@ -42,7 +42,7 @@ func NewBTreeContainers() *bTreeContainers { func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { b := &roaring.Bitmap{ - Containers: NewBTreeContainers(), + Containers: newBTreeContainers(), } b.Add(a...) return b @@ -132,7 +132,7 @@ func (btc *bTreeContainers) Count() (n uint64) { } func (btc *bTreeContainers) Clone() roaring.Containers { - nbtc := NewBTreeContainers() + nbtc := newBTreeContainers() itr, err := btc.tree.SeekFirst() if err == io.EOF { From 9a1f348580a6c7ce57b993c82a0b777c8d847f2e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:24 -0500 Subject: [PATCH 114/166] Unexport b.Tree --- enterprise/b/btree.go | 62 ++++++++++++++++---------------- enterprise/b/containers_btree.go | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go index 8d853083c..5ae5a7eb3 100644 --- a/enterprise/b/btree.go +++ b/enterprise/b/btree.go @@ -57,21 +57,21 @@ func init() { var ( btDPool = sync.Pool{New: func() interface{} { return &d{} }} btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}} - btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} + btTPool = btTpool{sync.Pool{New: func() interface{} { return &tree{} }}} btXPool = sync.Pool{New: func() interface{} { return &x{} }} ) type btTpool struct{ sync.Pool } -func (p *btTpool) get(cmp Cmp) *Tree { - x := p.Get().(*Tree) +func (p *btTpool) get(cmp Cmp) *tree { + x := p.Get().(*tree) x.cmp = cmp return x } type btEpool struct{ sync.Pool } -func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *enumerator { +func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *tree, ver int64) *enumerator { x := p.Get().(*enumerator) x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver return x @@ -112,12 +112,12 @@ type ( i int k uint64 q *d - t *Tree + t *tree ver int64 } - // Tree is a B+tree. - Tree struct { + // tree is a B+tree. + tree struct { c int cmp Cmp first *d @@ -142,7 +142,7 @@ var ( // R/O zero values zde de ze enumerator zk uint64 - zt Tree + zt tree zx x zxe xe ) @@ -235,12 +235,12 @@ func (l *d) mvR(r *d, c int) { // TreeNew returns a newly created, empty Tree. The compare function is used // for key collation. -func TreeNew(cmp Cmp) *Tree { +func TreeNew(cmp Cmp) *tree { return btTPool.get(cmp) } // Clear removes all K/V pairs from the tree. -func (t *Tree) Clear() { +func (t *tree) Clear() { if t.r == nil { return } @@ -252,13 +252,13 @@ func (t *Tree) Clear() { // Close performs Clear and recycles t to a pool for possible later reuse. No // references to t should exist or such references must not be used afterwards. -func (t *Tree) Close() { +func (t *tree) Close() { t.Clear() *t = zt btTPool.Put(t) } -func (t *Tree) cat(p *x, q, r *d, pi int) { +func (t *tree) cat(p *x, q, r *d, pi int) { t.ver++ q.mvL(r, r.c) if r.n != nil { @@ -286,7 +286,7 @@ func (t *Tree) cat(p *x, q, r *d, pi int) { t.r = q } -func (t *Tree) catX(p, q, r *x, pi int) { +func (t *tree) catX(p, q, r *x, pi int) { t.ver++ q.x[q.c].k = p.x[pi].k copy(q.x[q.c+1:], r.x[:r.c]) @@ -320,7 +320,7 @@ func (t *Tree) catX(p, q, r *x, pi int) { // Delete removes the k's KV pair, if it exists, in which case Delete returns // true. -func (t *Tree) Delete(k uint64) (ok bool) { +func (t *tree) Delete(k uint64) (ok bool) { pi := -1 var p *x q := t.r @@ -370,7 +370,7 @@ func (t *Tree) Delete(k uint64) (ok bool) { } } -func (t *Tree) extract(q *d, i int) { // (r *container) { +func (t *tree) extract(q *d, i int) { // (r *container) { t.ver++ //r = q.d[i].v // prepared for Extract q.c-- @@ -381,7 +381,7 @@ func (t *Tree) extract(q *d, i int) { // (r *container) { t.c-- } -func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { +func (t *tree) find(q interface{}, k uint64) (i int, ok bool) { var mk uint64 l := 0 switch x := q.(type) { @@ -419,7 +419,7 @@ func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { // First returns the first item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) First() (k uint64, v *roaring.Container) { +func (t *tree) First() (k uint64, v *roaring.Container) { if q := t.first; q != nil { q := &q.d[0] k, v = q.k, q.v @@ -429,7 +429,7 @@ func (t *Tree) First() (k uint64, v *roaring.Container) { // Get returns the value associated with k and true if it exists. Otherwise Get // returns (zero-value, false). -func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) { +func (t *tree) Get(k uint64) (v *roaring.Container, ok bool) { q := t.r if q == nil { return @@ -455,7 +455,7 @@ func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) { } } -func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { +func (t *tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { t.ver++ c := q.c if i < c { @@ -470,7 +470,7 @@ func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { // Last returns the last item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) Last() (k uint64, v *roaring.Container) { +func (t *tree) Last() (k uint64, v *roaring.Container) { if q := t.last; q != nil { q := &q.d[q.c-1] k, v = q.k, q.v @@ -479,11 +479,11 @@ func (t *Tree) Last() (k uint64, v *roaring.Container) { } // Len returns the number of items in the tree. -func (t *Tree) Len() int { +func (t *tree) Len() int { return t.c } -func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { +func (t *tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.ver++ l, r := p.siblings(pi) @@ -528,7 +528,7 @@ func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { // Seek returns an Enumerator positioned on an item such that k >= item's key. // ok reports if k == item.key The Enumerator's position is possibly after the // last item in the tree. -func (t *Tree) Seek(k uint64) (e *enumerator, ok bool) { +func (t *tree) Seek(k uint64) (e *enumerator, ok bool) { q := t.r if q == nil { e = btEPool.get(nil, false, 0, k, nil, t, t.ver) @@ -558,7 +558,7 @@ func (t *Tree) Seek(k uint64) (e *enumerator, ok bool) { // SeekFirst returns an enumerator positioned on the first KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekFirst() (e *enumerator, err error) { +func (t *tree) SeekFirst() (e *enumerator, err error) { q := t.first if q == nil { return nil, io.EOF @@ -569,7 +569,7 @@ func (t *Tree) SeekFirst() (e *enumerator, err error) { // SeekLast returns an enumerator positioned on the last KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekLast() (e *enumerator, err error) { +func (t *tree) SeekLast() (e *enumerator, err error) { q := t.last if q == nil { return nil, io.EOF @@ -579,7 +579,7 @@ func (t *Tree) SeekLast() (e *enumerator, err error) { } // Set sets the value associated with k. -func (t *Tree) Set(k uint64, v *roaring.Container) { +func (t *tree) Set(k uint64, v *roaring.Container) { //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) @@ -645,7 +645,7 @@ func (t *Tree) Set(k uint64, v *roaring.Container) { // tree.Put(k, func(uint64, bool){ return v, true }) // // modulo the differing return values. -func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { +func (t *tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { pi := -1 var p *x q := t.r @@ -712,7 +712,7 @@ func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (new } } -func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { +func (t *tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.ver++ r := btDPool.Get().(*d) if q.n != nil { @@ -747,7 +747,7 @@ func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.insert(q, i, k, v) } -func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { +func (t *tree) splitX(p *x, q *x, pi int, i int) (*x, int) { t.ver++ r := btXPool.Get().(*x) copy(r.x[:], q.x[kx+1:]) @@ -771,7 +771,7 @@ func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { return q, i } -func (t *Tree) underflow(p *x, q *d, pi int) { +func (t *tree) underflow(p *x, q *d, pi int) { t.ver++ l, r := p.siblings(pi) @@ -796,7 +796,7 @@ func (t *Tree) underflow(p *x, q *d, pi int) { t.cat(p, q, r, pi) } -func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { +func (t *tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { t.ver++ var l, r *x diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index a61c862e6..02564dcc8 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -28,7 +28,7 @@ func cmp(a, b uint64) int { } type bTreeContainers struct { - tree *Tree + tree *tree lastKey uint64 lastContainer *roaring.Container From 6fbd373256a70c60c0c29d174be935d7f4e7b206 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:25 -0500 Subject: [PATCH 115/166] Unexport b.TreeNew --- enterprise/b/btree.go | 4 ++-- enterprise/b/containers_btree.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go index 5ae5a7eb3..044411bd7 100644 --- a/enterprise/b/btree.go +++ b/enterprise/b/btree.go @@ -233,9 +233,9 @@ func (l *d) mvR(r *d, c int) { // ----------------------------------------------------------------------- Tree -// TreeNew returns a newly created, empty Tree. The compare function is used +// treeNew returns a newly created, empty Tree. The compare function is used // for key collation. -func TreeNew(cmp Cmp) *tree { +func treeNew(cmp Cmp) *tree { return btTPool.get(cmp) } diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 02564dcc8..95fcb09b3 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -36,7 +36,7 @@ type bTreeContainers struct { func newBTreeContainers() *bTreeContainers { return &bTreeContainers{ - tree: TreeNew(cmp), + tree: treeNew(cmp), } } @@ -161,7 +161,7 @@ func (btc *bTreeContainers) Size() int { } func (btc *bTreeContainers) Reset() { - btc.tree = TreeNew(cmp) + btc.tree = treeNew(cmp) btc.lastKey = 0 btc.lastContainer = nil } From 7db07ea72e889c6558bc866d5bba4368415f155f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:30 -0500 Subject: [PATCH 116/166] Unexport gcnotify.ActiveGCNotifier --- gcnotify/gcnotify.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gcnotify/gcnotify.go b/gcnotify/gcnotify.go index 76953a378..54313403d 100644 --- a/gcnotify/gcnotify.go +++ b/gcnotify/gcnotify.go @@ -20,25 +20,25 @@ import ( ) // Ensure ActiveGCNotifier implements interface. -var _ pilosa.GCNotifier = &ActiveGCNotifier{} +var _ pilosa.GCNotifier = &activeGCNotifier{} -type ActiveGCNotifier struct { +type activeGCNotifier struct { gcn *gcnotifier.GCNotifier } // NewActiveGCNotifier creates an active GCNotifier. -func NewActiveGCNotifier() *ActiveGCNotifier { - return &ActiveGCNotifier{ +func NewActiveGCNotifier() *activeGCNotifier { + return &activeGCNotifier{ gcn: gcnotifier.New(), } } // Close implements the GCNotifier interface. -func (n *ActiveGCNotifier) Close() { +func (n *activeGCNotifier) Close() { n.gcn.Close() } // AfterGC implements the GCNotifier interface. -func (n *ActiveGCNotifier) AfterGC() <-chan struct{} { +func (n *activeGCNotifier) AfterGC() <-chan struct{} { return n.gcn.AfterGC() } From 6c53ecc333d69f327d2700d2915843999e811786 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:36 -0500 Subject: [PATCH 117/166] Unexport gopsutil.SystemInfo --- gopsutil/systeminfo.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index 3310aeae1..8cdd77778 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -22,15 +22,15 @@ import ( var _ pilosa.SystemInfo = NewSystemInfo() -// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS. -type SystemInfo struct { +// systemInfo is an implementation of pilosa.systemInfo that uses gopsutil to collect information about the host OS. +type systemInfo struct { platform string family string osVersion string } // Uptime returns the system uptime in seconds. -func (s *SystemInfo) Uptime() (uptime uint64, err error) { +func (s *systemInfo) Uptime() (uptime uint64, err error) { hostInfo, err := host.Info() if err != nil { return 0, err @@ -39,7 +39,7 @@ func (s *SystemInfo) Uptime() (uptime uint64, err error) { } // collectPlatformInfo fetches and caches system platform information. -func (s *SystemInfo) collectPlatformInfo() error { +func (s *systemInfo) collectPlatformInfo() error { var err error if s.platform == "" { s.platform, s.family, s.osVersion, err = host.PlatformInformation() @@ -51,7 +51,7 @@ func (s *SystemInfo) collectPlatformInfo() error { } // Platform returns the system platform. -func (s *SystemInfo) Platform() (string, error) { +func (s *systemInfo) Platform() (string, error) { err := s.collectPlatformInfo() if err != nil { return "", err @@ -60,7 +60,7 @@ func (s *SystemInfo) Platform() (string, error) { } // Family returns the system family. -func (s *SystemInfo) Family() (string, error) { +func (s *systemInfo) Family() (string, error) { err := s.collectPlatformInfo() if err != nil { return "", err @@ -69,7 +69,7 @@ func (s *SystemInfo) Family() (string, error) { } // OSVersion returns the OS Version. -func (s *SystemInfo) OSVersion() (string, error) { +func (s *systemInfo) OSVersion() (string, error) { err := s.collectPlatformInfo() if err != nil { return "", err @@ -78,7 +78,7 @@ func (s *SystemInfo) OSVersion() (string, error) { } // MemFree returns the amount of free memory in bytes. -func (s *SystemInfo) MemFree() (uint64, error) { +func (s *systemInfo) MemFree() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { return 0, err @@ -87,7 +87,7 @@ func (s *SystemInfo) MemFree() (uint64, error) { } // MemTotal returns the amount of total memory in bytes. -func (s *SystemInfo) MemTotal() (uint64, error) { +func (s *systemInfo) MemTotal() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { return 0, err @@ -96,7 +96,7 @@ func (s *SystemInfo) MemTotal() (uint64, error) { } // MemUsed returns the amount of used memory in bytes. -func (s *SystemInfo) MemUsed() (uint64, error) { +func (s *systemInfo) MemUsed() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { return 0, err @@ -105,11 +105,11 @@ func (s *SystemInfo) MemUsed() (uint64, error) { } // KernelVersion returns the kernel version as a string. -func (s *SystemInfo) KernelVersion() (string, error) { +func (s *systemInfo) KernelVersion() (string, error) { return host.KernelVersion() } // NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. -func NewSystemInfo() *SystemInfo { - return &SystemInfo{} +func NewSystemInfo() *systemInfo { + return &systemInfo{} } From 6bc7470b0cba6c8cfb738468e7151e0d1540bb56 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:41 -0500 Subject: [PATCH 118/166] Unexport gossip.GossipMemberSet --- gossip/gossip.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 849b33d01..289eb505d 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -33,10 +33,10 @@ import ( ) // Ensure GossipMemberSet implements interfaces. -var _ memberlist.Delegate = &GossipMemberSet{} +var _ memberlist.Delegate = &gossipMemberSet{} -// GossipMemberSet represents a gossip implementation of MemberSet using memberlist. -type GossipMemberSet struct { +// gossipMemberSet represents a gossip implementation of MemberSet using memberlist. +type gossipMemberSet struct { mu sync.RWMutex memberlist *memberlist.Memberlist @@ -54,7 +54,7 @@ type GossipMemberSet struct { } // Open implements the MemberSet interface to start network activity. -func (g *GossipMemberSet) Open() (err error) { +func (g *gossipMemberSet) Open() (err error) { g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() @@ -94,7 +94,7 @@ func (g *GossipMemberSet) Open() (err error) { } // joinWithRetry wraps the standard memberlist Join function in a retry. -func (g *GossipMemberSet) joinWithRetry(hosts []string) error { +func (g *gossipMemberSet) joinWithRetry(hosts []string) error { err := retry(60, 2*time.Second, func() error { _, err := g.memberlist.Join(hosts) return err @@ -126,11 +126,11 @@ type gossipConfig struct { } // GossipMemberSetOption describes a functional option for GossipMemberSet. -type GossipMemberSetOption func(*GossipMemberSet) error +type GossipMemberSetOption func(*gossipMemberSet) error // WithTransport is a functional option for providing a transport to NewGossipMemberSet. func WithTransport(transport *Transport) GossipMemberSetOption { - return func(g *GossipMemberSet) error { + return func(g *gossipMemberSet) error { g.transport = transport return nil } @@ -138,16 +138,16 @@ func WithTransport(transport *Transport) GossipMemberSetOption { // WithLogger is a functional option for providing a logger to NewGossipMemberSet. func WithLogger(logger *log.Logger) GossipMemberSetOption { - return func(g *GossipMemberSet) error { + return func(g *gossipMemberSet) error { g.logger = logger return nil } } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { +func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*gossipMemberSet, error) { host := api.Node().URI.Host - g := &GossipMemberSet{ + g := &gossipMemberSet{ papi: api, Logger: pilosa.NopLogger, } @@ -219,7 +219,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO } // NodeMeta implementation of the memberlist.Delegate interface. -func (g *GossipMemberSet) NodeMeta(limit int) []byte { +func (g *gossipMemberSet) NodeMeta(limit int) []byte { buf, err := g.papi.Serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) @@ -230,7 +230,7 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte { // NotifyMsg implementation of the memberlist.Delegate interface // called when a user-data message is received. -func (g *GossipMemberSet) NotifyMsg(b []byte) { +func (g *gossipMemberSet) NotifyMsg(b []byte) { err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b)) if err != nil { g.Logger.Printf("cluster message error: %s", err) @@ -239,13 +239,13 @@ func (g *GossipMemberSet) NotifyMsg(b []byte) { // GetBroadcasts implementation of the memberlist.Delegate interface // called when user data messages can be broadcast. -func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { +func (g *gossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. -func (g *GossipMemberSet) LocalState(join bool) []byte { +func (g *gossipMemberSet) LocalState(join bool) []byte { m := &pilosa.NodeStatus{ Node: g.papi.Node(), MaxShards: g.papi.MaxShards(context.Background()), @@ -263,7 +263,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte { // MergeRemoteState implementation of the memberlist.Delegate interface // receive and process the remote side's LocalState. -func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { +func (g *gossipMemberSet) MergeRemoteState(buf []byte, join bool) { err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)) if err != nil { g.Logger.Printf("merge state error: %s", err) From d512b187b13262a1287cdf96160f4815206d431d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:46 -0500 Subject: [PATCH 119/166] Unexport gossip.GossipMemberSetOption --- gossip/gossip.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 289eb505d..0148ccd40 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -125,11 +125,11 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } -// GossipMemberSetOption describes a functional option for GossipMemberSet. -type GossipMemberSetOption func(*gossipMemberSet) error +// gossipMemberSetOption describes a functional option for GossipMemberSet. +type gossipMemberSetOption func(*gossipMemberSet) error // WithTransport is a functional option for providing a transport to NewGossipMemberSet. -func WithTransport(transport *Transport) GossipMemberSetOption { +func WithTransport(transport *Transport) gossipMemberSetOption { return func(g *gossipMemberSet) error { g.transport = transport return nil @@ -137,7 +137,7 @@ func WithTransport(transport *Transport) GossipMemberSetOption { } // WithLogger is a functional option for providing a logger to NewGossipMemberSet. -func WithLogger(logger *log.Logger) GossipMemberSetOption { +func WithLogger(logger *log.Logger) gossipMemberSetOption { return func(g *gossipMemberSet) error { g.logger = logger return nil @@ -145,7 +145,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*gossipMemberSet, error) { +func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetOption) (*gossipMemberSet, error) { host := api.Node().URI.Host g := &gossipMemberSet{ papi: api, From d95aeae1758afcce8e01d404029eb211de838738 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:51 -0500 Subject: [PATCH 120/166] Unexport gossip.Transport.Net --- gossip/gossip.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 0148ccd40..7650cff86 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -176,7 +176,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetO g.transport = transport } - port := g.transport.Net.GetAutoBindPort() + port := g.transport.net.GetAutoBindPort() var gossipKey []byte var err error @@ -189,7 +189,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetO // memberlist config conf := memberlist.DefaultWANConfig() - conf.Transport = g.transport.Net + conf.Transport = g.transport.net conf.Name = api.Node().ID conf.BindAddr = api.Node().URI.Host conf.BindPort = port @@ -343,7 +343,7 @@ func (g *gossipEventReceiver) listen() { // Transport is a gossip transport for binding to a port. type Transport struct { //memberlist.Transport - Net *memberlist.NetTransport + net *memberlist.NetTransport URI *pilosa.URI } @@ -370,7 +370,7 @@ func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) } return &Transport{ - Net: net, + net: net, URI: uri, }, nil } From 47c402c123e47d5bdd76ed3ccf16205f8c9a9b3c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 21:59:57 -0500 Subject: [PATCH 121/166] Unexport http.Error.Code --- http/error.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/error.go b/http/error.go index 90fac3206..1a3f07c04 100644 --- a/http/error.go +++ b/http/error.go @@ -17,7 +17,7 @@ package http // Error defines a standard application error. type Error struct { // Machine-readable error code. - Code string `json:"code,omitempty"` + code string `json:"code,omitempty"` // Human-readable message. Message string `json:"message"` From 0a2b482945aac4ec2a6c0112953c8e38a016b25a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:02 -0500 Subject: [PATCH 122/166] Unexport http.Handler.Logger --- http/handler.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/http/handler.go b/http/handler.go index 0693a4d17..a827df410 100644 --- a/http/handler.go +++ b/http/handler.go @@ -44,7 +44,7 @@ import ( type Handler struct { Handler http.Handler - Logger pilosa.Logger + logger pilosa.Logger // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec @@ -97,7 +97,7 @@ func OptHandlerAPI(api *pilosa.API) HandlerOption { func OptHandlerLogger(logger pilosa.Logger) HandlerOption { return func(h *Handler) error { - h.Logger = logger + h.logger = logger return nil } } @@ -112,7 +112,7 @@ func OptHandlerListener(ln net.Listener) HandlerOption { // NewHandler returns a new instance of Handler with a default logger. func NewHandler(opts ...HandlerOption) (*Handler, error) { handler := &Handler{ - Logger: pilosa.NopLogger, + logger: pilosa.NopLogger, } handler.Handler = NewRouter(handler) handler.populateValidators() @@ -140,7 +140,7 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) { func (h *Handler) Serve() error { err := h.server.Serve(h.ln) if err != nil && err.Error() != "http: Server closed" { - h.Logger.Printf("HTTP handler terminated with error: %s\n", err) + h.logger.Printf("HTTP handler terminated with error: %s\n", err) return errors.Wrap(err, "serve http") } return nil @@ -240,7 +240,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) stack := debug.Stack() msg := "PANIC: %s\n%s" - h.Logger.Printf(msg, err, stack) + h.logger.Printf(msg, err, stack) fmt.Fprintf(w, msg, err, stack) } }() @@ -254,7 +254,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { longQueryTime := h.api.LongQueryTime() if longQueryTime > 0 && dif > longQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + h.logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) statsTags = append(statsTags, "slow_query") } @@ -357,7 +357,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { schema := h.api.Schema(r.Context()) if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { - h.Logger.Printf("write schema response error: %s", err) + h.logger.Printf("write schema response error: %s", err) } } @@ -373,7 +373,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { LocalID: h.api.Node().ID, } if err := json.NewEncoder(w).Encode(status); err != nil { - h.Logger.Printf("write status response error: %s", err) + h.logger.Printf("write status response error: %s", err) } } @@ -384,7 +384,7 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { } info := h.api.Info() if err := json.NewEncoder(w).Encode(info); err != nil { - h.Logger.Printf("write info response error: %s", err) + h.logger.Printf("write info response error: %s", err) } } @@ -436,7 +436,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Write response back to client. if err := h.writeQueryResponse(w, r, &resp); err != nil { - h.Logger.Printf("write query response error: %s", err) + h.logger.Printf("write query response error: %s", err) } } @@ -449,7 +449,7 @@ func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getShardsMaxResponse{ Standard: h.api.MaxShards(r.Context()), }); err != nil { - h.Logger.Printf("write shards-max response error: %s", err) + h.logger.Printf("write shards-max response error: %s", err) } } @@ -472,7 +472,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { for _, idx := range h.api.Schema(r.Context()) { if idx.Name == indexName { if err := json.NewEncoder(w).Encode(idx); err != nil { - h.Logger.Printf("write response error: %s", err) + h.logger.Printf("write response error: %s", err) } return } @@ -618,7 +618,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.Logger.Printf("response encoding error: %s", err) + h.logger.Printf("response encoding error: %s", err) } } @@ -795,7 +795,7 @@ func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.Logger.Printf("response encoding error: %s", err) + h.logger.Printf("response encoding error: %s", err) } } @@ -1026,7 +1026,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.Logger.Printf("json write error: %s", err) + h.logger.Printf("json write error: %s", err) } } @@ -1078,7 +1078,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ Blocks: blocks, }); err != nil { - h.Logger.Printf("block response encoding error: %s", err) + h.logger.Printf("block response encoding error: %s", err) } } @@ -1098,7 +1098,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { Version: h.api.Version(), }) if err != nil { - h.Logger.Printf("write version response error: %s", err) + h.logger.Printf("write version response error: %s", err) } } @@ -1166,7 +1166,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r Old: oldNode, New: newNode, }); err != nil { - h.Logger.Printf("response encoding error: %s", err) + h.logger.Printf("response encoding error: %s", err) } } @@ -1207,7 +1207,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht if err := json.NewEncoder(w).Encode(removeNodeResponse{ Remove: removeNode, }); err != nil { - h.Logger.Printf("response encoding error: %s", err) + h.logger.Printf("response encoding error: %s", err) } } @@ -1243,7 +1243,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ Info: msg, }); err != nil { - h.Logger.Printf("response encoding error: %s", err) + h.logger.Printf("response encoding error: %s", err) } } @@ -1279,7 +1279,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) + h.logger.Printf("response encoding error: %s", err) } } From 46e40f71e4a4c8bc52988d06682e93197257ba9a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:08 -0500 Subject: [PATCH 123/166] Unexport http.Handler.AllowedOrigins --- http/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index a827df410..041743fcf 100644 --- a/http/handler.go +++ b/http/handler.go @@ -51,7 +51,7 @@ type Handler struct { api *pilosa.API - AllowedOrigins []string + allowedOrigins []string ln net.Listener From a3bd753cfad4661c3f59d91086ac993718f2f3a7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:14 -0500 Subject: [PATCH 124/166] Unexport http.HandlerOption --- http/handler.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/http/handler.go b/http/handler.go index 041743fcf..cd8ebddba 100644 --- a/http/handler.go +++ b/http/handler.go @@ -75,10 +75,10 @@ type errorResponse struct { Error string `json:"error"` } -// HandlerOption is a functional option type for pilosa.Handler -type HandlerOption func(s *Handler) error +// handlerOption is a functional option type for pilosa.Handler +type handlerOption func(s *Handler) error -func OptHandlerAllowedOrigins(origins []string) HandlerOption { +func OptHandlerAllowedOrigins(origins []string) handlerOption { return func(h *Handler) error { h.Handler = handlers.CORS( handlers.AllowedOrigins(origins), @@ -88,21 +88,21 @@ func OptHandlerAllowedOrigins(origins []string) HandlerOption { } } -func OptHandlerAPI(api *pilosa.API) HandlerOption { +func OptHandlerAPI(api *pilosa.API) handlerOption { return func(h *Handler) error { h.api = api return nil } } -func OptHandlerLogger(logger pilosa.Logger) HandlerOption { +func OptHandlerLogger(logger pilosa.Logger) handlerOption { return func(h *Handler) error { h.logger = logger return nil } } -func OptHandlerListener(ln net.Listener) HandlerOption { +func OptHandlerListener(ln net.Listener) handlerOption { return func(h *Handler) error { h.ln = ln return nil @@ -110,7 +110,7 @@ func OptHandlerListener(ln net.Listener) HandlerOption { } // NewHandler returns a new instance of Handler with a default logger. -func NewHandler(opts ...HandlerOption) (*Handler, error) { +func NewHandler(opts ...handlerOption) (*Handler, error) { handler := &Handler{ logger: pilosa.NopLogger, } From e460a58bfa640878e89b07fe20d1347c4c315c43 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:20 -0500 Subject: [PATCH 125/166] Unexport http.InternalClient.HTTPClient --- http/client.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/http/client.go b/http/client.go index 7453c87fd..bf5c9f5dc 100644 --- a/http/client.go +++ b/http/client.go @@ -38,7 +38,7 @@ type InternalClient struct { serializer pilosa.Serializer // The client to use for HTTP communication. - HTTPClient *http.Client + httpClient *http.Client } // NewInternalClient returns a new instance of InternalClient to connect to host. @@ -60,7 +60,7 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) return &InternalClient{ defaultURI: defaultURI, serializer: proto.Serializer{}, - HTTPClient: remoteClient, + httpClient: remoteClient, } } @@ -84,7 +84,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64 req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -115,7 +115,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -152,7 +152,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return errors.Wrap(err, "executing request") } @@ -191,7 +191,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -238,7 +238,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -390,7 +390,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return errors.Wrap(err, "executing request") } @@ -512,7 +512,7 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return errors.Wrap(err, "executing request") } @@ -555,7 +555,7 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -599,7 +599,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return errors.Wrap(err, "executing request") } @@ -645,7 +645,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -693,7 +693,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, req.Header.Set("Accept", "application/protobuf") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, nil, errors.Wrap(err, "executing request") } @@ -741,7 +741,7 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -785,7 +785,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "executing request") } @@ -820,7 +820,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [ req.Header.Set("Accept", "application/json") // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + resp, err := c.httpClient.Do(req.WithContext(ctx)) if err != nil { return fmt.Errorf("executing http request: %v", err) } From ab26a1be4b8d0085be38b4387a3761e5328b27d2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:25 -0500 Subject: [PATCH 126/166] Unexport http.NewRouter --- http/handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http/handler.go b/http/handler.go index cd8ebddba..6df406e14 100644 --- a/http/handler.go +++ b/http/handler.go @@ -114,7 +114,7 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { handler := &Handler{ logger: pilosa.NopLogger, } - handler.Handler = NewRouter(handler) + handler.Handler = newRouter(handler) handler.populateValidators() for _, opt := range opts { @@ -183,8 +183,8 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { }) } -// NewRouter creates a new mux http router. -func NewRouter(handler *Handler) *mux.Router { +// newRouter creates a new mux http router. +func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() router.HandleFunc("/", handler.handleHome).Methods("GET") router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") From 88b45edd7c73d7dfaff401d88ee5cb7eaca5f389 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:31 -0500 Subject: [PATCH 127/166] Unexport http.QueryResultTypeBool --- http/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index 6df406e14..a51a1e0d0 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1109,7 +1109,7 @@ const ( QueryResultTypePairs QueryResultTypeValCount QueryResultTypeUint64 - QueryResultTypeBool + queryResultTypeBool ) // parseUint64Slice returns a slice of uint64s from a comma-delimited string. From 9a708234c868a6c2db0953c18354a95efe47e255 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:36 -0500 Subject: [PATCH 128/166] Unexport http.QueryResultTypeNil --- http/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index a51a1e0d0..312483d72 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1104,7 +1104,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { // QueryResult types. const ( - QueryResultTypeNil uint32 = iota + queryResultTypeNil uint32 = iota QueryResultTypeRow QueryResultTypePairs QueryResultTypeValCount From 04417d870d356b375b5c0ed1e962bd59c9c6c1e6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:42 -0500 Subject: [PATCH 129/166] Unexport http.QueryResultTypeValCount --- http/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index 312483d72..4946f69e0 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1107,7 +1107,7 @@ const ( queryResultTypeNil uint32 = iota QueryResultTypeRow QueryResultTypePairs - QueryResultTypeValCount + queryResultTypeValCount QueryResultTypeUint64 queryResultTypeBool ) From 55d13d869fe5301ad3adc1a7feb90060813df5d2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:48 -0500 Subject: [PATCH 130/166] Unexport http.TranslateStore --- http/translator.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/http/translator.go b/http/translator.go index 08235bf47..d094a8c28 100644 --- a/http/translator.go +++ b/http/translator.go @@ -14,41 +14,41 @@ import ( ) // Ensure implementation implements inteface. -var _ pilosa.TranslateStore = (*TranslateStore)(nil) +var _ pilosa.TranslateStore = (*translateStore)(nil) -// TranslateStore represents an implementation of TranslateStore that +// translateStore represents an implementation of translateStore that // communicates over HTTP. This is used with the TranslateHandler. -type TranslateStore struct { +type translateStore struct { URL string } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore(rawurl string) *TranslateStore { - return &TranslateStore{URL: rawurl} +func NewTranslateStore(rawurl string) *translateStore { + return &translateStore{URL: rawurl} } // TranslateColumnsToUint64 is not currently implemented. -func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { +func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { return nil, pilosa.ErrNotImplemented } // TranslateColumnToString is not currently implemented. -func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { +func (s *translateStore) TranslateColumnToString(index string, values uint64) (string, error) { return "", pilosa.ErrNotImplemented } // TranslateRowsToUint64 is not currently implemented. -func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { +func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { return nil, pilosa.ErrNotImplemented } // TranslateRowToString is not currently implemented. -func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { +func (s *translateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { return "", pilosa.ErrNotImplemented } // Reader returns a reader that can stream data from a remote store. -func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { +func (s *translateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { // Generate remote URL. u, err := url.Parse(s.URL) if err != nil { From 536f6cff8a1ffbaf4b43da9cad6d181a91f5b326 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:52 -0500 Subject: [PATCH 131/166] Unexport inmem.TranslateStore --- inmem/translator.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/inmem/translator.go b/inmem/translator.go index b620b2d03..4e1e96b55 100644 --- a/inmem/translator.go +++ b/inmem/translator.go @@ -9,10 +9,10 @@ import ( ) // Ensure type implements interface. -var _ pilosa.TranslateStore = &TranslateStore{} +var _ pilosa.TranslateStore = &translateStore{} -// TranslateStore is an in-memory storage engine for translating string-to-uint64 values. -type TranslateStore struct { +// translateStore is an in-memory storage engine for translating string-to-uint64 values. +type translateStore struct { mu sync.RWMutex cols map[string]*translateIndex @@ -20,21 +20,21 @@ type TranslateStore struct { } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore() *TranslateStore { - return &TranslateStore{ +func NewTranslateStore() *translateStore { + return &translateStore{ cols: make(map[string]*translateIndex), rows: make(map[frameKey]*translateIndex), } } // Reader returns an error because it is not supported by the inmem store. -func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { +func (s *translateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { return nil, pilosa.ErrReplicationNotSupported } // TranslateColumnsToUint64 converts value to a uint64 id. // If value does not have an associated id then one is created. -func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { +func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { ret := make([]uint64, len(values)) // Read value under read lock. @@ -103,7 +103,7 @@ func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) // TranslateColumnToString converts a uint64 id to its associated string value. // If the id is not associated with a string value then a blank string is returned. -func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) { +func (s *translateStore) TranslateColumnToString(index string, value uint64) (string, error) { s.mu.RLock() if idx := s.cols[index]; idx != nil { if ret, ok := idx.reverse[value]; ok { @@ -115,7 +115,7 @@ func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (st return "", nil } -func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { +func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { key := frameKey{index, frame} ret := make([]uint64, len(values)) @@ -184,7 +184,7 @@ func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []str return ret, nil } -func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { +func (s *translateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { s.mu.RLock() if idx := s.rows[frameKey{index, frame}]; idx != nil { if ret, ok := idx.reverse[value]; ok { From a22d83b880a972d2e636b1f190fb07e053e703cb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:55 -0500 Subject: [PATCH 132/166] Unexport lru.Cache.Clear --- lru/lru.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lru/lru.go b/lru/lru.go index 532cc45e6..31bcd1375 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -120,8 +120,8 @@ func (c *Cache) Len() int { return c.ll.Len() } -// Clear purges all stored items from the cache. -func (c *Cache) Clear() { +// clear purges all stored items from the cache. +func (c *Cache) clear() { if c.OnEvicted != nil { for _, e := range c.cache { kv := e.Value.(*entry) From ffae51549bbf51abc32e1913a49b040b817e618c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:00:59 -0500 Subject: [PATCH 133/166] Unexport lru.Cache.Remove --- lru/lru.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lru/lru.go b/lru/lru.go index 31bcd1375..bd70df780 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -82,8 +82,8 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) { return } -// Remove removes the provided key from the cache. -func (c *Cache) Remove(key Key) { +// remove removes the provided key from the cache. +func (c *Cache) remove(key Key) { if c.cache == nil { return } From 91ce0d6c576d78e0f4a6c3d7d87169d714d252fa Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:01:03 -0500 Subject: [PATCH 134/166] Unexport lru.Cache.RemoveOldest --- lru/lru.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lru/lru.go b/lru/lru.go index bd70df780..9e944703f 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -66,7 +66,7 @@ func (c *Cache) Add(key Key, value interface{}) { ele := c.ll.PushFront(&entry{key, value}) c.cache[key] = ele if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { - c.RemoveOldest() + c.removeOldest() } } @@ -92,8 +92,8 @@ func (c *Cache) remove(key Key) { } } -// RemoveOldest removes the oldest item from the cache. -func (c *Cache) RemoveOldest() { +// removeOldest removes the oldest item from the cache. +func (c *Cache) removeOldest() { if c.cache == nil { return } From d6f0789511f5651ff2d7e83c9447a8f5ed253f19 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:01:07 -0500 Subject: [PATCH 135/166] Unexport lru.Cache.MaxEntries --- lru/lru.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lru/lru.go b/lru/lru.go index 9e944703f..59f896f7e 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -21,9 +21,9 @@ import "container/list" // Cache is an LRU cache. It is not safe for concurrent access. type Cache struct { - // MaxEntries is the maximum number of cache entries before + // maxEntries is the maximum number of cache entries before // an item is evicted. Zero means no limit. - MaxEntries int + maxEntries int // OnEvicted optionally specificies a callback function to be // executed when an entry is purged from the cache. @@ -46,7 +46,7 @@ type entry struct { // that eviction is done by the caller. func New(maxEntries int) *Cache { return &Cache{ - MaxEntries: maxEntries, + maxEntries: maxEntries, ll: list.New(), cache: make(map[interface{}]*list.Element), } @@ -65,7 +65,7 @@ func (c *Cache) Add(key Key, value interface{}) { } ele := c.ll.PushFront(&entry{key, value}) c.cache[key] = ele - if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { + if c.maxEntries != 0 && c.ll.Len() > c.maxEntries { c.removeOldest() } } From 3a4763858014d0f4b2563dff6a0f7cabbfb1099a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:01:39 -0500 Subject: [PATCH 136/166] Unexport pql.Call.Keys --- pql/ast.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 47baa9d6c..f95da9f40 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -312,8 +312,8 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } -// Keys returns a list of argument keys in sorted order. -func (c *Call) Keys() []string { +// keys returns a list of argument keys in sorted order. +func (c *Call) keys() []string { a := make([]string, 0, len(c.Args)) for k := range c.Args { a = append(a, k) @@ -369,7 +369,7 @@ func (c *Call) String() string { } // Write arguments in key order. - for i, key := range c.Keys() { + for i, key := range c.keys() { if i > 0 { buf.WriteString(", ") } From aa7f6fca75d1b6fb8eed0e9daca81d66d5c4e30b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:01:45 -0500 Subject: [PATCH 137/166] Unexport pql.FormatValue --- pql/ast.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index f95da9f40..88095c49b 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -379,7 +379,7 @@ func (c *Call) String() string { case *Condition: fmt.Fprintf(&buf, "%v %s", key, v.String()) default: - fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v)) + fmt.Fprintf(&buf, "%v=%s", key, formatValue(v)) } } @@ -408,7 +408,7 @@ type Condition struct { // String returns the string representation of the condition. func (cond *Condition) String() string { - return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value)) + return fmt.Sprintf("%s %s", cond.Op.String(), formatValue(cond.Value)) } // IntSliceValue reads cond.Value as a slice of uint64. @@ -436,7 +436,7 @@ func (cond *Condition) IntSliceValue() ([]int64, error) { } } -func FormatValue(v interface{}) string { +func formatValue(v interface{}) string { switch v := v.(type) { case string: return fmt.Sprintf("%q", v) From 75ea43c9da999b07171b81162ad80476160ad1ac Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:01:50 -0500 Subject: [PATCH 138/166] Unexport pql.Parser --- pql/parser.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pql/parser.go b/pql/parser.go index 83498f207..3bc91a2ee 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -25,16 +25,16 @@ import ( // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// Parser represents a parser for the PQL language. -type Parser struct { +// parser represents a parser for the PQL language. +type parser struct { r io.Reader //scanner *bufScanner PQL } // NewParser returns a new instance of Parser. -func NewParser(r io.Reader) *Parser { - return &Parser{ +func NewParser(r io.Reader) *parser { + return &parser{ r: r, // scanner: newBufScanner(r), } @@ -46,7 +46,7 @@ func ParseString(s string) (*Query, error) { } // Parse parses the next node in the query. -func (p *Parser) Parse() (*Query, error) { +func (p *parser) Parse() (*Query, error) { buf, err := ioutil.ReadAll(p.r) if err != nil { return nil, errors.Wrap(err, "reading buffer to parse") From fb7de28557ba33ccb247651bbfa5ffe26136f102 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:01:56 -0500 Subject: [PATCH 139/166] Unexport pql.TimeFormat --- pql/ast.go | 2 +- pql/parser.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 88095c49b..88af892ad 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -445,7 +445,7 @@ func formatValue(v interface{}) string { case []uint64: return fmt.Sprintf("%s", joinUint64Slice(v)) case time.Time: - return fmt.Sprintf("\"%s\"", v.Format(TimeFormat)) + return fmt.Sprintf("\"%s\"", v.Format(timeFormat)) case *Condition: return v.String() default: diff --git a/pql/parser.go b/pql/parser.go index 3bc91a2ee..611294971 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -22,8 +22,8 @@ import ( "github.com/pkg/errors" ) -// TimeFormat is the go-style time format used to parse string dates. -const TimeFormat = "2006-01-02T15:04" +// timeFormat is the go-style time format used to parse string dates. +const timeFormat = "2006-01-02T15:04" // parser represents a parser for the PQL language. type parser struct { From c73fc795da203aaec96d99700171e3b9ef5959fe Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:03 -0500 Subject: [PATCH 140/166] Unexport roaring.BitmapInfo --- roaring/roaring.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 07f1600a7..1e0f72a83 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -725,8 +725,8 @@ func (b *Bitmap) Iterator() *Iterator { } // Info returns stats for the bitmap. -func (b *Bitmap) Info() BitmapInfo { - info := BitmapInfo{ +func (b *Bitmap) Info() bitmapInfo { + info := bitmapInfo{ OpN: b.opN, Containers: make([]ContainerInfo, 0, b.Containers.Size()), } @@ -788,8 +788,8 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap { return result } -// BitmapInfo represents a point-in-time snapshot of bitmap stats. -type BitmapInfo struct { +// bitmapInfo represents a point-in-time snapshot of bitmap stats. +type bitmapInfo struct { OpN int Containers []ContainerInfo } From 72fcb37f807c9b7395e60287bb5aa85d7cba9949 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:08 -0500 Subject: [PATCH 141/166] Unexport roaring.Container.Optimize --- roaring/roaring.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 1e0f72a83..a1295fd80 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -500,7 +500,7 @@ func (b *Bitmap) Optimize() { citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - c.Optimize() + c.optimize() } } @@ -1315,9 +1315,9 @@ func (c *Container) countRuns() (r int) { return 0 } -// Optimize converts the container to the type which will take up the least +// optimize converts the container to the type which will take up the least // amount of space. -func (c *Container) Optimize() { +func (c *Container) optimize() { if c.n == 0 { return } @@ -2142,7 +2142,7 @@ func intersectBitmapBitmap(a, b *Container) *Container { output.n += int(popcount(v)) } - output.Optimize() + output.optimize() return output } @@ -2621,7 +2621,7 @@ RUNLOOP: output.n += int(run.last - start + 1) } } - output.Optimize() + output.optimize() return output } From 9098b0d1312c2f46acb42fff0d06055c864cf5c6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:15 -0500 Subject: [PATCH 142/166] Unexport roaring.ContainerArray --- roaring/roaring.go | 46 +++++++++++++------------- roaring/roaring_helpers_test.go | 24 +++++++------- roaring/roaring_internal_test.go | 56 ++++++++++++++++---------------- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index a1295fd80..ed6667c79 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -51,8 +51,8 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - //ContainerArray indicates a container of bit position values - ContainerArray = byte(1) + //containerArray indicates a container of bit position values + containerArray = byte(1) //ContainerBitmap indicates a container of bits packed in a uint64 array block ContainerBitmap = byte(2) @@ -663,7 +663,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount] opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size - case ContainerArray: + case containerArray: c.runs = nil c.bitmap = nil c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] @@ -1021,7 +1021,7 @@ func (iv interval16) runlen() int { // newContainer returns a new instance of container. func NewContainer() *Container { - return &Container{containerType: ContainerArray} + return &Container{containerType: containerArray} } // Mapped returns true if the container is mapped directly to a byte slice @@ -1043,7 +1043,7 @@ func (c *Container) Update(containerType byte, n int, mapped bool) { // isArray returns true if the container is an array container. func (c *Container) isArray() bool { - return c.containerType == ContainerArray + return c.containerType == containerArray } // isBitmap returns true if the container is a bitmap container. @@ -1066,7 +1066,7 @@ func (c *Container) unmap() { } switch c.containerType { - case ContainerArray: + case containerArray: tmp := make([]uint16, len(c.array)) copy(tmp, c.array) c.array = tmp @@ -1327,7 +1327,7 @@ func (c *Container) optimize() { if runs <= RunMaxSize && runs <= c.n/2 { newType = ContainerRun } else if c.n < ArrayMaxSize { - newType = ContainerArray + newType = containerArray } else { newType = ContainerBitmap } @@ -1340,7 +1340,7 @@ func (c *Container) optimize() { c.arrayToRun() } } else if c.isBitmap() { - if newType == ContainerArray { + if newType == containerArray { c.bitmapToArray() } else if newType == ContainerRun { c.bitmapToRun() @@ -1348,7 +1348,7 @@ func (c *Container) optimize() { } else if c.isRun() { if newType == ContainerBitmap { c.runToBitmap() - } else if newType == ContainerArray { + } else if newType == containerArray { c.runToArray() } } @@ -1487,7 +1487,7 @@ func (c *Container) runMax() uint16 { // bitmapToArray converts from bitmap format to array format. func (c *Container) bitmapToArray() { c.array = make([]uint16, 0, c.n) - c.containerType = ContainerArray + c.containerType = containerArray // return early if empty if c.n == 0 { @@ -1634,7 +1634,7 @@ func (c *Container) arrayToRun() { // runToArray converts from RLE format to array format. func (c *Container) runToArray() { - c.containerType = ContainerArray + c.containerType = containerArray c.array = make([]uint16, 0, c.n) // return early if empty @@ -1658,7 +1658,7 @@ func (c *Container) Clone() *Container { other := &Container{n: c.n, containerType: c.containerType} switch c.containerType { - case ContainerArray: + case containerArray: other.array = make([]uint16, len(c.array)) copy(other.array, c.array) case ContainerBitmap: @@ -1977,7 +1977,7 @@ func intersect(a, b *Container) *Container { } func intersectArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1998,7 +1998,7 @@ func intersectArrayArray(a, b *Container) *Container { // container. The return is always an array container (since it's guaranteed to // be low-cardinality) func intersectArrayRun(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.runs[j] @@ -2059,7 +2059,7 @@ func intersectBitmapRun(a, b *Container) *Container { var output *Container if b.n < ArrayMaxSize { // output is array container - output = &Container{containerType: ContainerArray} + output = &Container{containerType: containerArray} for _, iv := range b.runs { for i := iv.start; i <= iv.last; i++ { if a.bitmapContains(i) { @@ -2119,7 +2119,7 @@ func intersectBitmapRun(a, b *Container) *Container { } func intersectArrayBitmap(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2175,7 +2175,7 @@ func union(a, b *Container) *Container { } func unionArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; ; { if i >= na && j >= nb { @@ -2387,7 +2387,7 @@ func (c *Container) equals(c2 *Container) bool { if c.mapped != c2.mapped || c.containerType != c2.containerType || c.n != c2.n { return false } - if c.containerType == ContainerArray { + if c.containerType == containerArray { if len(c.array) != len(c2.array) { return false } @@ -2476,7 +2476,7 @@ func difference(a, b *Container) *Container { // differenceArrayArray computes the difference bween two arrays. func differenceArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na; { va := a.array[i] @@ -2507,7 +2507,7 @@ func differenceArrayRun(a, b *Container) *Container { return a.Clone() } - output := &Container{array: make([]uint16, 0, a.n), containerType: ContainerArray} + output := &Container{array: make([]uint16, 0, a.n), containerType: containerArray} // cardinality upper bound: card(A) i := 0 // array index @@ -2542,7 +2542,7 @@ func differenceArrayRun(a, b *Container) *Container { // keep all array elements after end of runs // It's possible that output was converted from array to bitmap in output.add() // so check container type before proceeding. - if output.containerType == ContainerArray { + if output.containerType == containerArray { output.array = append(output.array, a.array[i:]...) // TODO: consider handling container.n mutations in one place // like we do with container.add(). @@ -2747,7 +2747,7 @@ func differenceRunRun(a, b *Container) *Container { } func differenceArrayBitmap(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2821,7 +2821,7 @@ func xor(a, b *Container) *Container { } func xorArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 24db8e1ca..3ed42d8db 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -236,7 +236,7 @@ func doContainer(containerType byte, data interface{}) *Container { } switch containerType { - case ContainerArray: + case containerArray: c.array = data.([]uint16) case ContainerBitmap: c.bitmap = data.([]uint64) @@ -253,17 +253,17 @@ func setupContainerTests() map[byte]map[string]*Container { cts := make(map[byte]map[string]*Container) // array containers - cts[ContainerArray] = map[string]*Container{ - "empty": doContainer(ContainerArray, arrayEmpty()), - "full": doContainer(ContainerArray, arrayFull()), - "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), - "lastBitSet": doContainer(ContainerArray, arrayLastBitSet()), - "firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()), - "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), - "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), - "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), - "oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()), - "evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()), + cts[containerArray] = map[string]*Container{ + "empty": doContainer(containerArray, arrayEmpty()), + "full": doContainer(containerArray, arrayFull()), + "firstBitSet": doContainer(containerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(containerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(containerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(containerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()), } // bitmap containers diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 6e1f105bf..c478c6209 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -290,7 +290,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { for i, test := range tests { a.array = test.array - a.containerType = ContainerArray + a.containerType = containerArray b.bitmap = test.bitmap b.containerType = ContainerBitmap ret := intersectionCountArrayBitmap(a, b) @@ -349,7 +349,7 @@ func TestRunMax(t *testing.T) { } func TestIntersectionCountArrayRun(t *testing.T) { - a := &Container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} + a := &Container{containerType: containerArray, array: []uint16{1, 5, 10, 11, 12}} b := &Container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} ret := intersectionCountArrayRun(a, b) @@ -458,7 +458,7 @@ func TestIntersectArrayRun(t *testing.T) { } for i, test := range tests { - a.containerType = ContainerArray + a.containerType = containerArray b.containerType = ContainerRun a.array = test.array b.runs = test.runs @@ -660,7 +660,7 @@ func TestUnionMixed(t *testing.T) { // array container a := &Container{} a.array = []uint16{1, 4, 5, 7, 10, 11, 12} - a.containerType = ContainerArray + a.containerType = containerArray a.n = 7 // bitmap container @@ -716,7 +716,7 @@ func TestIntersectMixed(t *testing.T) { a.containerType = ContainerRun b.array = []uint16{1, 4, 5, 7, 10, 11, 12} b.n = 7 - b.containerType = ContainerArray + b.containerType = containerArray res := intersect(a, b) if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) @@ -766,11 +766,11 @@ func TestDifferenceMixed(t *testing.T) { b.array = []uint16{0, 2, 4, 6, 8, 10, 12} b.n = len(b.array) - b.containerType = ContainerArray + b.containerType = containerArray d.array = []uint16{1, 3, 5, 7, 9, 11, 12} d.n = len(d.array) - d.containerType = ContainerArray + d.containerType = containerArray res := difference(a, b) @@ -927,7 +927,7 @@ func TestUnionArrayRun(t *testing.T) { for i, test := range tests { a.array = test.array b.runs = test.runs - a.containerType = ContainerArray + a.containerType = containerArray b.containerType = ContainerRun ret := unionArrayRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { @@ -977,7 +977,7 @@ func TestBitmapSetRange(t *testing.T) { } func TestArrayToBitmap(t *testing.T) { - a := &Container{containerType: ContainerArray} + a := &Container{containerType: containerArray} tests := []struct { array []uint16 exp []uint64 @@ -1171,7 +1171,7 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := &Container{containerType: ContainerArray} + a := &Container{containerType: containerArray} tests := []struct { array []uint16 exp []interval16 @@ -1372,7 +1372,7 @@ func TestBitmapCountRuns(t *testing.T) { } func TestArrayCountRuns(t *testing.T) { - c := &Container{containerType: ContainerArray} + c := &Container{containerType: containerArray} tests := []struct { array []uint16 exp int @@ -1413,7 +1413,7 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := &Container{containerType: ContainerArray} + a := &Container{containerType: containerArray} b := &Container{containerType: ContainerRun} tests := []struct { array []uint16 @@ -1440,7 +1440,7 @@ func TestDifferenceArrayRun(t *testing.T) { func TestDifferenceRunArray(t *testing.T) { a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerArray} + b := &Container{containerType: containerArray} tests := []struct { runs []interval16 array []uint16 @@ -1667,7 +1667,7 @@ func TestDifferenceBitmapRun(t *testing.T) { func TestDifferenceBitmapArray(t *testing.T) { b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - a := &Container{containerType: ContainerArray} + a := &Container{containerType: containerArray} tests := []struct { bitmap []uint64 array []uint16 @@ -1780,7 +1780,7 @@ func TestDifferenceRunRun(t *testing.T) { } func TestWriteReadArray(t *testing.T) { - ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} + ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: containerArray} ba := NewFileBitmap() ba.Containers.Put(0, ca) ba2 := NewFileBitmap() @@ -1877,21 +1877,21 @@ func TestXorArrayRun(t *testing.T) { exp *Container }{ { - a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray}, + a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: containerArray}, b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12}, + exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: containerArray, n: 12}, }, { - a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray}, + a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: containerArray}, b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12}, + exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: containerArray, n: 12}, }, { - a: &Container{array: []uint16{65535}, containerType: ContainerArray}, + a: &Container{array: []uint16{65535}, containerType: containerArray}, b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{65534}, containerType: ContainerArray, n: 1}, + exp: &Container{array: []uint16{65534}, containerType: containerArray, n: 1}, }, { - a: &Container{array: []uint16{65535}, containerType: ContainerArray}, + a: &Container{array: []uint16{65535}, containerType: containerArray}, b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{}, containerType: ContainerArray, n: 0}, + exp: &Container{array: []uint16{}, containerType: containerArray, n: 0}, }, } @@ -2546,7 +2546,7 @@ func TestSearch64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := &Container{containerType: ContainerArray}, &Container{ + a, b := &Container{containerType: containerArray}, &Container{ containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN), } @@ -2594,7 +2594,7 @@ func TestIntersectArrayBitmap(t *testing.T) { for i, test := range tests { a.array = test.array - a.containerType = ContainerArray + a.containerType = containerArray for i, bmval := range test.bitmap { b.bitmap[i] = bmval } @@ -2722,11 +2722,11 @@ func TestContainerCombinations(t *testing.T) { cts := setupContainerTests() - containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun} + containerTypes := []byte{containerArray, ContainerBitmap, ContainerRun} // map used for a more descriptive print cm := map[byte]string{ - ContainerArray: "array", + containerArray: "array", ContainerBitmap: "bitmap", ContainerRun: "run", } @@ -3198,7 +3198,7 @@ func TestContainerCombinations(t *testing.T) { // Convert to all container types and check result. for _, ct := range containerTypes { clone := ret.Clone() - if ct == ContainerArray { + if ct == containerArray { if clone.isBitmap() { clone.bitmapToArray() } else if clone.isRun() { From 692b72be18cb549f0802e883a0701114e8473ad1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:21 -0500 Subject: [PATCH 143/166] Unexport roaring.ContainerBitmap --- roaring/roaring.go | 38 ++++++++++---------- roaring/roaring_helpers_test.go | 24 ++++++------- roaring/roaring_internal_test.go | 62 ++++++++++++++++---------------- 3 files changed, 62 insertions(+), 62 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ed6667c79..3f048a544 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -54,8 +54,8 @@ const ( //containerArray indicates a container of bit position values containerArray = byte(1) - //ContainerBitmap indicates a container of bits packed in a uint64 array block - ContainerBitmap = byte(2) + //containerBitmap indicates a container of bits packed in a uint64 array block + containerBitmap = byte(2) //ContainerRun indicates a container of run encoded bits ContainerRun = byte(3) @@ -668,7 +668,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { c.bitmap = nil c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32) - case ContainerBitmap: + case containerBitmap: c.array = nil c.runs = nil c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] @@ -1048,7 +1048,7 @@ func (c *Container) isArray() bool { // isBitmap returns true if the container is a bitmap container. func (c *Container) isBitmap() bool { - return c.containerType == ContainerBitmap + return c.containerType == containerBitmap } // isRun returns true if the container is a run-length-encoded container. @@ -1070,7 +1070,7 @@ func (c *Container) unmap() { tmp := make([]uint16, len(c.array)) copy(tmp, c.array) c.array = tmp - case ContainerBitmap: + case containerBitmap: tmp := make([]uint64, len(c.bitmap)) copy(tmp, c.bitmap) c.bitmap = tmp @@ -1329,12 +1329,12 @@ func (c *Container) optimize() { } else if c.n < ArrayMaxSize { newType = containerArray } else { - newType = ContainerBitmap + newType = containerBitmap } // Then convert accordingly. if c.isArray() { - if newType == ContainerBitmap { + if newType == containerBitmap { c.arrayToBitmap() } else if newType == ContainerRun { c.arrayToRun() @@ -1346,7 +1346,7 @@ func (c *Container) optimize() { c.bitmapToRun() } } else if c.isRun() { - if newType == ContainerBitmap { + if newType == containerBitmap { c.runToBitmap() } else if newType == containerArray { c.runToArray() @@ -1510,7 +1510,7 @@ func (c *Container) bitmapToArray() { // arrayToBitmap converts from array format to bitmap format. func (c *Container) arrayToBitmap() { c.bitmap = make([]uint64, bitmapN) - c.containerType = ContainerBitmap + c.containerType = containerBitmap // return early if empty if c.n == 0 { @@ -1529,7 +1529,7 @@ func (c *Container) arrayToBitmap() { // runToBitmap converts from RLE format to bitmap format. func (c *Container) runToBitmap() { c.bitmap = make([]uint64, bitmapN) - c.containerType = ContainerBitmap + c.containerType = containerBitmap // return early if empty if c.n == 0 { @@ -1661,7 +1661,7 @@ func (c *Container) Clone() *Container { case containerArray: other.array = make([]uint16, len(c.array)) copy(other.array, c.array) - case ContainerBitmap: + case containerBitmap: other.bitmap = make([]uint64, len(c.bitmap)) copy(other.bitmap, c.bitmap) case ContainerRun: @@ -1816,7 +1816,7 @@ func flipArray(b *Container) *Container { } func flipBitmap(b *Container) *Container { - other := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + other := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} for i, bitmap := range b.bitmap { other.bitmap[i] = ^bitmap @@ -2078,7 +2078,7 @@ func intersectBitmapRun(a, b *Container) *Container { // the bitmap which are between runs. output = &Container{ bitmap: make([]uint64, bitmapN), - containerType: ContainerBitmap, + containerType: containerBitmap, } for j := 0; j < len(b.runs); j++ { vb := b.runs[j] @@ -2134,7 +2134,7 @@ func intersectArrayBitmap(a, b *Container) *Container { } func intersectBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} for i := range a.bitmap { v := a.bitmap[i] & b.bitmap[i] @@ -2396,7 +2396,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.containerType == ContainerBitmap { + } else if c.containerType == containerBitmap { if len(c.bitmap) != len(c2.bitmap) { return false } @@ -2434,7 +2434,7 @@ func unionArrayBitmap(a, b *Container) *Container { func unionBitmapBitmap(a, b *Container) *Container { output := &Container{ bitmap: make([]uint64, bitmapN), - containerType: ContainerBitmap, + containerType: containerBitmap, } for i := 0; i < bitmapN; i++ { @@ -2778,7 +2778,7 @@ func differenceBitmapArray(a, b *Container) *Container { } func differenceBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} for i := range a.bitmap { v := a.bitmap[i] & (^b.bitmap[i]) @@ -2861,7 +2861,7 @@ func xorArrayBitmap(a, b *Container) *Container { // It's possible that output was converted from bitmap to array in output.remove() // so we only do this conversion if output is still a bitmap container. - if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize { + if output.containerType == containerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -2871,7 +2871,7 @@ func xorArrayBitmap(a, b *Container) *Container { func xorBitmapBitmap(a, b *Container) *Container { output := &Container{ bitmap: make([]uint64, bitmapN), - containerType: ContainerBitmap, + containerType: containerBitmap, } for i := 0; i < bitmapN; i++ { v := a.bitmap[i] ^ b.bitmap[i] diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 3ed42d8db..7d0265efa 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -238,7 +238,7 @@ func doContainer(containerType byte, data interface{}) *Container { switch containerType { case containerArray: c.array = data.([]uint16) - case ContainerBitmap: + case containerBitmap: c.bitmap = data.([]uint64) case ContainerRun: c.runs = data.([]interval16) @@ -267,17 +267,17 @@ func setupContainerTests() map[byte]map[string]*Container { } // bitmap containers - cts[ContainerBitmap] = map[string]*Container{ - "empty": doContainer(ContainerBitmap, bitmapEmpty()), - "full": doContainer(ContainerBitmap, bitmapFull()), - "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), - "lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()), - "firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()), - "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), - "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), - "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), - "oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()), - "evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()), + cts[containerBitmap] = map[string]*Container{ + "empty": doContainer(containerBitmap, bitmapEmpty()), + "full": doContainer(containerBitmap, bitmapFull()), + "firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()), } // run containers diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index c478c6209..dc2693b85 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -203,7 +203,7 @@ func TestRunContains(t *testing.T) { } func TestBitmapCountRange(t *testing.T) { - c := Container{containerType: ContainerBitmap} + c := Container{containerType: containerBitmap} tests := []struct { start int end int @@ -229,11 +229,11 @@ func TestBitmapCountRange(t *testing.T) { func TestIntersectionCountArrayBitmap3(t *testing.T) { a, b := &Container{}, &Container{} - a.containerType = ContainerBitmap + a.containerType = containerBitmap a.bitmap = getFullBitmap() a.n = maxContainerVal + 1 - b.containerType = ContainerBitmap + b.containerType = containerBitmap b.bitmap = getFullBitmap() b.n = maxContainerVal + 1 res := intersectBitmapBitmap(a, b) @@ -292,7 +292,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { a.array = test.array a.containerType = containerArray b.bitmap = test.bitmap - b.containerType = ContainerBitmap + b.containerType = containerBitmap ret := intersectionCountArrayBitmap(a, b) if ret != test.exp { t.Fatalf("test #%v intersectCountArrayBitmap fail received: %v exp: %v", i, ret, test.exp) @@ -359,7 +359,7 @@ func TestIntersectionCountArrayRun(t *testing.T) { } func TestIntersectionCountBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} + a := &Container{containerType: containerBitmap, bitmap: []uint64{0x8000000000000000}} b := &Container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}} ret := intersectionCountBitmapRun(a, b) @@ -367,7 +367,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) } - a = &Container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} + a = &Container{containerType: containerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} b = &Container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} ret = intersectionCountBitmapRun(a, b) @@ -581,7 +581,7 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { for i, v := range test.exp { exp[i] = v } - a.containerType = ContainerBitmap + a.containerType = containerBitmap b.containerType = ContainerRun ret := intersectBitmapRun(a, b) if ret.isArray() { @@ -642,7 +642,7 @@ func TestIntersectBitmapRunArray(t *testing.T) { a.bitmap[i] = v } b.runs = test.runs - a.containerType = ContainerBitmap + a.containerType = containerBitmap b.containerType = ContainerRun ret := intersectBitmapRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { @@ -667,7 +667,7 @@ func TestUnionMixed(t *testing.T) { b := &Container{bitmap: make([]uint64, bitmapN)} b.bitmap[0] = uint64(0x3) b.n = 2 - b.containerType = ContainerBitmap + b.containerType = containerBitmap // run container r := &Container{} @@ -732,7 +732,7 @@ func TestIntersectMixed(t *testing.T) { } c.bitmap = []uint64{0x60} c.n = 2 - c.containerType = ContainerBitmap + c.containerType = containerBitmap res = intersect(c, a) if !reflect.DeepEqual(res.array, []uint16{5, 6}) { @@ -790,7 +790,7 @@ func TestDifferenceMixed(t *testing.T) { c.bitmap = []uint64{0x64} c.n = c.countRange(0, 100) - c.containerType = ContainerBitmap + c.containerType = containerBitmap res = difference(c, a) if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) { t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap) @@ -937,7 +937,7 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -1008,7 +1008,7 @@ func TestArrayToBitmap(t *testing.T) { } func TestBitmapToArray(t *testing.T) { - a := &Container{containerType: ContainerBitmap} + a := &Container{containerType: containerBitmap} tests := []struct { bitmap []uint64 exp []uint16 @@ -1093,7 +1093,7 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap} + a := &Container{containerType: containerBitmap} tests := []struct { bitmap []uint64 exp []interval16 @@ -1239,7 +1239,7 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -1283,7 +1283,7 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} b := &Container{containerType: ContainerRun} tests := []struct { bitmap []uint64 @@ -1322,7 +1322,7 @@ func TestUnionBitmapRun(t *testing.T) { } func TestBitmapCountRuns(t *testing.T) { - c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 exp int @@ -1521,7 +1521,7 @@ func MakeLastBitSet() []uint64 { func TestDifferenceRunBitmap(t *testing.T) { a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { runs []interval16 bitmap []uint64 @@ -1583,7 +1583,7 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} b := &Container{containerType: ContainerRun} tests := []struct { bitmap []uint64 @@ -1666,7 +1666,7 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} a := &Container{containerType: containerArray} tests := []struct { bitmap []uint64 @@ -1716,8 +1716,8 @@ func TestDifferenceBitmapArray(t *testing.T) { } func TestDifferenceBitmapBitmap(t *testing.T) { - a := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - b := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + a := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + b := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1800,7 +1800,7 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap} + cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: containerBitmap} for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } @@ -1823,7 +1823,7 @@ func TestWriteReadBitmap(t *testing.T) { func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap} + cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: containerBitmap} for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } @@ -2025,7 +2025,7 @@ func TestXorRunRun(t *testing.T) { } func TestBitmapXorRange(t *testing.T) { - c := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + c := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} tests := []struct { bitmap []uint64 start uint64 @@ -2093,7 +2093,7 @@ func TestBitmapXorRange(t *testing.T) { } func TestXorBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap} + a := &Container{containerType: containerBitmap} b := &Container{containerType: ContainerRun} tests := []struct { bitmap []uint64 @@ -2547,7 +2547,7 @@ func TestSearch64(t *testing.T) { func TestIntersectArrayBitmap(t *testing.T) { a, b := &Container{containerType: containerArray}, &Container{ - containerType: ContainerBitmap, + containerType: containerBitmap, bitmap: make([]uint64, bitmapN), } tests := []struct { @@ -2598,7 +2598,7 @@ func TestIntersectArrayBitmap(t *testing.T) { for i, bmval := range test.bitmap { b.bitmap[i] = bmval } - b.containerType = ContainerBitmap + b.containerType = containerBitmap ret := intersectArrayBitmap(a, b).array if len(ret) == 0 && len(test.exp) == 0 { continue @@ -2722,12 +2722,12 @@ func TestContainerCombinations(t *testing.T) { cts := setupContainerTests() - containerTypes := []byte{containerArray, ContainerBitmap, ContainerRun} + containerTypes := []byte{containerArray, containerBitmap, ContainerRun} // map used for a more descriptive print cm := map[byte]string{ containerArray: "array", - ContainerBitmap: "bitmap", + containerBitmap: "bitmap", ContainerRun: "run", } @@ -3212,7 +3212,7 @@ func TestContainerCombinations(t *testing.T) { if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) } - } else if ct == ContainerBitmap { + } else if ct == containerBitmap { if clone.isArray() { clone.arrayToBitmap() } else if clone.isRun() { From bdd4fb51ef72de5416d58cb3fc1ada1554dac51a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:27 -0500 Subject: [PATCH 144/166] Unexport roaring.ContainerInfo --- roaring/roaring.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 3f048a544..3793af4a0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -728,7 +728,7 @@ func (b *Bitmap) Iterator() *Iterator { func (b *Bitmap) Info() bitmapInfo { info := bitmapInfo{ OpN: b.opN, - Containers: make([]ContainerInfo, 0, b.Containers.Size()), + Containers: make([]containerInfo, 0, b.Containers.Size()), } citer, _ := b.Containers.Iterator(0) @@ -791,7 +791,7 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap { // bitmapInfo represents a point-in-time snapshot of bitmap stats. type bitmapInfo struct { OpN int - Containers []ContainerInfo + Containers []containerInfo } // Iterator represents an iterator over a Bitmap. @@ -1730,8 +1730,8 @@ func (c *Container) size() int { } // info returns the current stats about the container. -func (c *Container) info() ContainerInfo { - info := ContainerInfo{N: c.n} +func (c *Container) info() containerInfo { + info := containerInfo{N: c.n} if c.isArray() { info.Type = "array" @@ -1787,8 +1787,8 @@ func (c *Container) check() error { return a } -// ContainerInfo represents a point-in-time snapshot of container stats. -type ContainerInfo struct { +// containerInfo represents a point-in-time snapshot of container stats. +type containerInfo struct { Key uint64 // container key Type string // container type (array, bitmap, or run) N int // number of bits From 67cfe4dcee6c162fa37f7b2d2d7b4f2970f4603c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:33 -0500 Subject: [PATCH 145/166] Unexport roaring.ContainerRun --- roaring/roaring.go | 40 +++++++------- roaring/roaring_helpers_test.go | 24 ++++----- roaring/roaring_internal_test.go | 90 ++++++++++++++++---------------- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 3793af4a0..ab4c1cabd 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -57,8 +57,8 @@ const ( //containerBitmap indicates a container of bits packed in a uint64 array block containerBitmap = byte(2) - //ContainerRun indicates a container of run encoded bits - ContainerRun = byte(3) + //containerRun indicates a container of run encoded bits + containerRun = byte(3) maxContainerVal = 0xffff ) @@ -657,7 +657,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { citer.Next() _, c := citer.Value() switch c.containerType { - case ContainerRun: + case containerRun: c.array = nil c.bitmap = nil runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) @@ -1053,7 +1053,7 @@ func (c *Container) isBitmap() bool { // isRun returns true if the container is a run-length-encoded container. func (c *Container) isRun() bool { - return c.containerType == ContainerRun + return c.containerType == containerRun } // unmap creates copies of the containers data in the heap. @@ -1074,7 +1074,7 @@ func (c *Container) unmap() { tmp := make([]uint64, len(c.bitmap)) copy(tmp, c.bitmap) c.bitmap = tmp - case ContainerRun: + case containerRun: tmp := make([]interval16, len(c.runs)) copy(tmp, c.runs) c.runs = tmp @@ -1325,7 +1325,7 @@ func (c *Container) optimize() { var newType byte if runs <= RunMaxSize && runs <= c.n/2 { - newType = ContainerRun + newType = containerRun } else if c.n < ArrayMaxSize { newType = containerArray } else { @@ -1336,13 +1336,13 @@ func (c *Container) optimize() { if c.isArray() { if newType == containerBitmap { c.arrayToBitmap() - } else if newType == ContainerRun { + } else if newType == containerRun { c.arrayToRun() } } else if c.isBitmap() { if newType == containerArray { c.bitmapToArray() - } else if newType == ContainerRun { + } else if newType == containerRun { c.bitmapToRun() } } else if c.isRun() { @@ -1551,7 +1551,7 @@ func (c *Container) runToBitmap() { // bitmapToRun converts from bitmap format to RLE format. func (c *Container) bitmapToRun() { - c.containerType = ContainerRun + c.containerType = containerRun // return early if empty if c.n == 0 { c.runs = make([]interval16, 0) @@ -1607,7 +1607,7 @@ func (c *Container) bitmapToRun() { // arrayToRun converts from array format to RLE format. func (c *Container) arrayToRun() { - c.containerType = ContainerRun + c.containerType = containerRun // return early if empty if c.n == 0 { c.runs = make([]interval16, 0) @@ -1664,7 +1664,7 @@ func (c *Container) Clone() *Container { case containerBitmap: other.bitmap = make([]uint64, len(c.bitmap)) copy(other.bitmap, c.bitmap) - case ContainerRun: + case containerRun: other.runs = make([]interval16, len(c.runs)) copy(other.runs, c.runs) } @@ -2017,7 +2017,7 @@ func intersectArrayRun(a, b *Container) *Container { // intersectRunRun computes the intersect of two run containers. func intersectRunRun(a, b *Container) *Container { - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} na, nb := len(a.runs), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.runs[i], b.runs[j] @@ -2211,7 +2211,7 @@ func unionArrayRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { return b.Clone() } - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -2274,7 +2274,7 @@ func unionRunRun(a, b *Container) *Container { na, nb := len(a.runs), len(b.runs) output := &Container{ runs: make([]interval16, 0, na+nb), - containerType: ContainerRun, + containerType: containerRun, } var va, vb interval16 for i, j := 0, 0; i < na || j < nb; { @@ -2405,7 +2405,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.containerType == ContainerRun { + } else if c.containerType == containerRun { if len(c.runs) != len(c2.runs) { return false } @@ -2575,7 +2575,7 @@ func differenceRunArray(a, b *Container) *Container { if a.n == 0 || b.n == 0 { return a.Clone() } - output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun} + output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: containerRun} bidx := 0 vb := b.array[bidx] @@ -2631,7 +2631,7 @@ func differenceRunBitmap(a, b *Container) *Container { if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { return flipBitmap(b) } - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} output.n = a.n if len(a.runs) == 0 { return output @@ -2697,7 +2697,7 @@ func differenceRunRun(a, b *Container) *Container { alen := len(a.runs) blen := len(b.runs) - output := &Container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else + output := &Container{runs: make([]interval16, 0, alen+blen), containerType: containerRun} // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -3079,7 +3079,7 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) { // xorArrayRun computes the exclusive or of an array and a run container. func xorArrayRun(a, b *Container) *Container { - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -3248,7 +3248,7 @@ func xorRunRun(a, b *Container) *Container { if nb == 0 { return a.Clone() } - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} lastI, lastJ := -1, -1 diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 7d0265efa..cd22978fb 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -240,7 +240,7 @@ func doContainer(containerType byte, data interface{}) *Container { c.array = data.([]uint16) case containerBitmap: c.bitmap = data.([]uint64) - case ContainerRun: + case containerRun: c.runs = data.([]interval16) } c.n = c.count() @@ -281,17 +281,17 @@ func setupContainerTests() map[byte]map[string]*Container { } // run containers - cts[ContainerRun] = map[string]*Container{ - "empty": doContainer(ContainerRun, runEmpty()), - "full": doContainer(ContainerRun, runFull()), - "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), - "lastBitSet": doContainer(ContainerRun, runLastBitSet()), - "firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()), - "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), - "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), - "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), - "oddBitsSet": doContainer(ContainerRun, runOddBitsSet()), - "evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()), + cts[containerRun] = map[string]*Container{ + "empty": doContainer(containerRun, runEmpty()), + "full": doContainer(containerRun, runFull()), + "firstBitSet": doContainer(containerRun, runFirstBitSet()), + "lastBitSet": doContainer(containerRun, runLastBitSet()), + "firstBitUnset": doContainer(containerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(containerRun, runLastBitUnset()), + "innerBitsSet": doContainer(containerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(containerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(containerRun, runOddBitsSet()), + "evenBitsSet": doContainer(containerRun, runEvenBitsSet()), } return cts diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index dc2693b85..5836782b8 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -33,7 +33,7 @@ func (c *Container) String() string { } func TestRunAppendInterval(t *testing.T) { - a := Container{containerType: ContainerRun} + a := Container{containerType: containerRun} tests := []struct { base []interval16 app interval16 @@ -82,7 +82,7 @@ func TestInterval16RunLen(t *testing.T) { } func TestContainerRunAdd(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} tests := []struct { op uint16 exp []interval16 @@ -113,7 +113,7 @@ func TestContainerRunAdd(t *testing.T) { } func TestContainerRunAdd2(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} ret := c.add(0) if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs) @@ -128,7 +128,7 @@ func TestContainerRunAdd2(t *testing.T) { } func TestRunCountRange(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} cnt := c.runCountRange(2, 9) if cnt != 0 { t.Fatalf("should get 0 from empty container, but got: %v", cnt) @@ -181,7 +181,7 @@ func TestRunCountRange(t *testing.T) { } func TestRunContains(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} if c.runContains(5) { t.Fatalf("empty run container should not contain 5") } @@ -301,7 +301,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } func TestRunRemove(t *testing.T) { - c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} + c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun} tests := []struct { op uint16 exp []interval16 @@ -335,7 +335,7 @@ func TestRunRemove(t *testing.T) { } func TestRunMax(t *testing.T) { - c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} + c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun} max := c.max() if max != 16 { t.Fatalf("max for %v should be 16", c.runs) @@ -350,7 +350,7 @@ func TestRunMax(t *testing.T) { func TestIntersectionCountArrayRun(t *testing.T) { a := &Container{containerType: containerArray, array: []uint16{1, 5, 10, 11, 12}} - b := &Container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + b := &Container{containerType: containerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} ret := intersectionCountArrayRun(a, b) if ret != 3 { @@ -360,7 +360,7 @@ func TestIntersectionCountArrayRun(t *testing.T) { func TestIntersectionCountBitmapRun(t *testing.T) { a := &Container{containerType: containerBitmap, bitmap: []uint64{0x8000000000000000}} - b := &Container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}} + b := &Container{containerType: containerRun, runs: []interval16{{start: 63, last: 64}}} ret := intersectionCountBitmapRun(a, b) if ret != 1 { @@ -368,7 +368,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) { } a = &Container{containerType: containerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} - b = &Container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + b = &Container{containerType: containerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} ret = intersectionCountBitmapRun(a, b) if ret != 14 { @@ -416,8 +416,8 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { - a.containerType = ContainerRun - b.containerType = ContainerRun + a.containerType = containerRun + b.containerType = containerRun a.runs = test.aruns b.runs = test.bruns ret := intersectionCountRunRun(a, b) @@ -459,7 +459,7 @@ func TestIntersectArrayRun(t *testing.T) { for i, test := range tests { a.containerType = containerArray - b.containerType = ContainerRun + b.containerType = containerRun a.array = test.array b.runs = test.runs ret := intersectArrayRun(a, b) @@ -516,8 +516,8 @@ func TestIntersectRunRun(t *testing.T) { }, } for i, test := range tests { - a.containerType = ContainerRun - b.containerType = ContainerRun + a.containerType = containerRun + b.containerType = containerRun a.runs = test.aruns b.runs = test.bruns ret := intersectRunRun(a, b) @@ -582,7 +582,7 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { exp[i] = v } a.containerType = containerBitmap - b.containerType = ContainerRun + b.containerType = containerRun ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -643,7 +643,7 @@ func TestIntersectBitmapRunArray(t *testing.T) { } b.runs = test.runs a.containerType = containerBitmap - b.containerType = ContainerRun + b.containerType = containerRun ret := intersectBitmapRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -672,7 +672,7 @@ func TestUnionMixed(t *testing.T) { // run container r := &Container{} r.runs = []interval16{{start: 5, last: 10}} - r.containerType = ContainerRun + r.containerType = containerRun r.n = 6 t.Run("various container Unions", func(t *testing.T) { @@ -713,7 +713,7 @@ func TestIntersectMixed(t *testing.T) { a.runs = []interval16{{start: 5, last: 10}} a.n = 6 - a.containerType = ContainerRun + a.containerType = containerRun b.array = []uint16{1, 4, 5, 7, 10, 11, 12} b.n = 7 b.containerType = containerArray @@ -762,7 +762,7 @@ func TestDifferenceMixed(t *testing.T) { a.runs = []interval16{{start: 5, last: 10}} a.n = a.runCountRange(0, 100) - a.containerType = ContainerRun + a.containerType = containerRun b.array = []uint16{0, 2, 4, 6, 8, 10, 12} b.n = len(b.array) @@ -885,8 +885,8 @@ func TestUnionRunRun(t *testing.T) { for i, test := range tests { a.runs = test.aruns b.runs = test.bruns - a.containerType = ContainerRun - b.containerType = ContainerRun + a.containerType = containerRun + b.containerType = containerRun ret := unionRunRun(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -928,7 +928,7 @@ func TestUnionArrayRun(t *testing.T) { a.array = test.array b.runs = test.runs a.containerType = containerArray - b.containerType = ContainerRun + b.containerType = containerRun ret := unionArrayRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -1039,7 +1039,7 @@ func TestBitmapToArray(t *testing.T) { } func TestRunToBitmap(t *testing.T) { - a := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} tests := []struct { runs []interval16 exp []uint64 @@ -1205,7 +1205,7 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} tests := []struct { runs []interval16 exp []uint16 @@ -1284,7 +1284,7 @@ func TestBitmapZeroRange(t *testing.T) { func TestUnionBitmapRun(t *testing.T) { a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} - b := &Container{containerType: ContainerRun} + b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1414,7 +1414,7 @@ func TestArrayCountRuns(t *testing.T) { func TestDifferenceArrayRun(t *testing.T) { a := &Container{containerType: containerArray} - b := &Container{containerType: ContainerRun} + b := &Container{containerType: containerRun} tests := []struct { array []uint16 runs []interval16 @@ -1439,7 +1439,7 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} b := &Container{containerType: containerArray} tests := []struct { runs []interval16 @@ -1520,7 +1520,7 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { runs []interval16 @@ -1584,7 +1584,7 @@ func TestDifferenceRunBitmap(t *testing.T) { func TestDifferenceBitmapRun(t *testing.T) { a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} - b := &Container{containerType: ContainerRun} + b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1746,8 +1746,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) { } func TestDifferenceRunRun(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -1852,7 +1852,7 @@ func TestWriteReadFullBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} + cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: containerRun} br := NewFileBitmap() br.Containers.Put(0, cr) br2 := NewFileBitmap() @@ -1878,19 +1878,19 @@ func TestXorArrayRun(t *testing.T) { }{ { a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, + b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}, exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: containerArray, n: 12}, }, { a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, + b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}, exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: containerArray, n: 12}, }, { a: &Container{array: []uint16{65535}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun}, + b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: containerRun}, exp: &Container{array: []uint16{65534}, containerType: containerArray, n: 1}, }, { a: &Container{array: []uint16{65535}, containerType: containerArray}, - b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun}, + b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: containerRun}, exp: &Container{array: []uint16{}, containerType: containerArray, n: 0}, }, } @@ -1912,8 +1912,8 @@ func TestXorArrayRun(t *testing.T) { //special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerRun} a.runs = []interval16{{start: 4, last: 10}} b.runs = []interval16{{start: 5, last: 10}} ret := xorRunRun(a, b) @@ -1927,8 +1927,8 @@ func TestXorRunRun1(t *testing.T) { } func TestXorRunRun(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -2094,7 +2094,7 @@ func TestBitmapXorRange(t *testing.T) { func TestXorBitmapRun(t *testing.T) { a := &Container{containerType: containerBitmap} - b := &Container{containerType: ContainerRun} + b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -2722,13 +2722,13 @@ func TestContainerCombinations(t *testing.T) { cts := setupContainerTests() - containerTypes := []byte{containerArray, containerBitmap, ContainerRun} + containerTypes := []byte{containerArray, containerBitmap, containerRun} // map used for a more descriptive print cm := map[byte]string{ containerArray: "array", containerBitmap: "bitmap", - ContainerRun: "run", + containerRun: "run", } testOps := []testOp{ @@ -3224,7 +3224,7 @@ func TestContainerCombinations(t *testing.T) { if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) } - } else if ct == ContainerRun { + } else if ct == containerRun { if clone.isArray() { clone.arrayToRun() } else if clone.isBitmap() { From 5ec9d3af222f220c4e85e2b9f8a789339408eab2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:39 -0500 Subject: [PATCH 146/166] Unexport roaring.NewSliceContainers --- roaring/containers.go | 4 ++-- roaring/roaring.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index 19871050b..a993ebfdb 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -21,7 +21,7 @@ type SliceContainers struct { lastContainer *Container } -func NewSliceContainers() *SliceContainers { +func newSliceContainers() *SliceContainers { return &SliceContainers{} } @@ -102,7 +102,7 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *Container { } func (sc *SliceContainers) Clone() Containers { - other := NewSliceContainers() + other := newSliceContainers() other.keys = make([]uint64, len(sc.keys)) other.containers = make([]*Container, len(sc.containers)) copy(other.keys, sc.keys) diff --git a/roaring/roaring.go b/roaring/roaring.go index ab4c1cabd..c15b965e1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -117,7 +117,7 @@ type Bitmap struct { // NewBitmap returns a Bitmap with an initial set of values. func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ - Containers: NewSliceContainers(), + Containers: newSliceContainers(), } b.Add(a...) return b From 89c28043dd078133240d8d500af2258a72866998 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:46 -0500 Subject: [PATCH 147/166] Unexport roaring.RunMaxSize --- roaring/roaring.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index c15b965e1..2544c5007 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -987,8 +987,8 @@ func (itr *Iterator) peek() uint64 { // ArrayMaxSize represents the maximum size of array containers. const ArrayMaxSize = 4096 -// RunMaxSize represents the maximum size of run length encoded containers. -const RunMaxSize = 2048 +// runMaxSize represents the maximum size of run length encoded containers. +const runMaxSize = 2048 // Container represents a Container for uint16 integers. // @@ -1324,7 +1324,7 @@ func (c *Container) optimize() { runs := c.countRuns() var newType byte - if runs <= RunMaxSize && runs <= c.n/2 { + if runs <= runMaxSize && runs <= c.n/2 { newType = containerRun } else if c.n < ArrayMaxSize { newType = containerArray @@ -2047,7 +2047,7 @@ func intersectRunRun(a, b *Container) *Container { } if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2232,7 +2232,7 @@ func unionArrayRun(a, b *Container) *Container { } if output.n < ArrayMaxSize { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2292,7 +2292,7 @@ func unionRunRun(a, b *Container) *Container { j++ } } - if len(output.runs) > RunMaxSize { + if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2676,7 +2676,7 @@ func differenceRunBitmap(a, b *Container) *Container { if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -3135,7 +3135,7 @@ func xorArrayRun(a, b *Container) *Container { } if output.n < ArrayMaxSize { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -3281,7 +3281,7 @@ func xorRunRun(a, b *Container) *Container { if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -3296,7 +3296,7 @@ func xorBitmapRun(a, b *Container) *Container { if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output From c134535229e87bd8d20796d769053fddc97a5450 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:52 -0500 Subject: [PATCH 148/166] Unexport roaring.SliceContainers --- roaring/containers.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index a993ebfdb..8cdddf607 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -14,18 +14,18 @@ package roaring -type SliceContainers struct { +type sliceContainers struct { keys []uint64 containers []*Container lastKey uint64 lastContainer *Container } -func newSliceContainers() *SliceContainers { - return &SliceContainers{} +func newSliceContainers() *sliceContainers { + return &sliceContainers{} } -func (sc *SliceContainers) Get(key uint64) *Container { +func (sc *sliceContainers) Get(key uint64) *Container { i := search64(sc.keys, key) if i < 0 { return nil @@ -33,7 +33,7 @@ func (sc *SliceContainers) Get(key uint64) *Container { return sc.containers[i] } -func (sc *SliceContainers) Put(key uint64, c *Container) { +func (sc *sliceContainers) Put(key uint64, c *Container) { i := search64(sc.keys, key) // If index is negative then there's not an exact match @@ -46,7 +46,7 @@ func (sc *SliceContainers) Put(key uint64, c *Container) { } -func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { i := search64(sc.keys, key) if i < 0 { c := NewContainer() @@ -63,7 +63,7 @@ func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n } -func (sc *SliceContainers) Remove(key uint64) { +func (sc *sliceContainers) Remove(key uint64) { i := search64(sc.keys, key) if i < 0 { return @@ -72,7 +72,7 @@ func (sc *SliceContainers) Remove(key uint64) { sc.containers = append(sc.containers[:i], sc.containers[i+1:]...) } -func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { +func (sc *sliceContainers) insertAt(key uint64, c *Container, i int) { sc.keys = append(sc.keys, 0) copy(sc.keys[i+1:], sc.keys[i:]) sc.keys[i] = key @@ -82,7 +82,7 @@ func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { sc.containers[i] = c } -func (sc *SliceContainers) GetOrCreate(key uint64) *Container { +func (sc *sliceContainers) GetOrCreate(key uint64) *Container { // Check the last* cache for same container. if key == sc.lastKey && sc.lastContainer != nil { return sc.lastContainer @@ -101,7 +101,7 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *Container { return sc.lastContainer } -func (sc *SliceContainers) Clone() Containers { +func (sc *sliceContainers) Clone() Containers { other := newSliceContainers() other.keys = make([]uint64, len(sc.keys)) other.containers = make([]*Container, len(sc.containers)) @@ -112,19 +112,19 @@ func (sc *SliceContainers) Clone() Containers { return other } -func (sc *SliceContainers) Last() (key uint64, c *Container) { +func (sc *sliceContainers) Last() (key uint64, c *Container) { if len(sc.keys) == 0 { return 0, nil } return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1] } -func (sc *SliceContainers) Size() int { +func (sc *sliceContainers) Size() int { return len(sc.keys) } -func (sc *SliceContainers) Count() uint64 { +func (sc *sliceContainers) Count() uint64 { n := uint64(0) for i := range sc.containers { n += uint64(sc.containers[i].n) @@ -132,14 +132,14 @@ func (sc *SliceContainers) Count() uint64 { return n } -func (sc *SliceContainers) Reset() { +func (sc *sliceContainers) Reset() { sc.keys = sc.keys[:0] sc.containers = sc.containers[:0] sc.lastContainer = nil sc.lastKey = 0 } -func (sc *SliceContainers) seek(key uint64) (int, bool) { +func (sc *sliceContainers) seek(key uint64) (int, bool) { i := search64(sc.keys, key) found := true if i < 0 { @@ -149,13 +149,13 @@ func (sc *SliceContainers) seek(key uint64) (int, bool) { return i, found } -func (sc *SliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) { +func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) { i, found := sc.seek(key) return &SliceIterator{e: sc, i: i}, found } type SliceIterator struct { - e *SliceContainers + e *sliceContainers i int key uint64 value *Container From 31cab33fbe1db68e096f655c657f096d7d447172 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:02:57 -0500 Subject: [PATCH 149/166] Unexport roaring.SliceIterator --- roaring/containers.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index 8cdddf607..5275d4f04 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -151,17 +151,17 @@ func (sc *sliceContainers) seek(key uint64) (int, bool) { func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) { i, found := sc.seek(key) - return &SliceIterator{e: sc, i: i}, found + return &sliceIterator{e: sc, i: i}, found } -type SliceIterator struct { +type sliceIterator struct { e *sliceContainers i int key uint64 value *Container } -func (si *SliceIterator) Next() bool { +func (si *sliceIterator) Next() bool { if si.e == nil || si.i > len(si.e.keys)-1 { return false } @@ -172,6 +172,6 @@ func (si *SliceIterator) Next() bool { return true } -func (si *SliceIterator) Value() (uint64, *Container) { +func (si *sliceIterator) Value() (uint64, *Container) { return si.key, si.value } From 6023474ed4a8f18c670b0cfc23762cdf7a271240 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:05 -0500 Subject: [PATCH 150/166] Unexport server.Command.SetupNetworking --- server/server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index 401da09e8..3c91c9fb2 100644 --- a/server/server.go +++ b/server/server.go @@ -124,7 +124,7 @@ func (m *Command) Start() (err error) { } // SetupNetworking - err = m.SetupNetworking() + err = m.setupNetworking() if err != nil { return errors.Wrap(err, "setting up networking") } @@ -299,8 +299,8 @@ func (m *Command) SetupServer() error { } -// SetupNetworking sets up internode communication based on the configuration. -func (m *Command) SetupNetworking() error { +// setupNetworking sets up internode communication based on the configuration. +func (m *Command) setupNetworking() error { if m.Config.Cluster.Disabled { return nil } From 94e633b7eb53ac4363a538a905a0251524cdc77f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:10 -0500 Subject: [PATCH 151/166] Unexport server.DefaultDiagnosticsInterval --- server/default.go | 4 ++-- server/server.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/default.go b/server/default.go index a7131c82a..ce2fe8aaa 100644 --- a/server/default.go +++ b/server/default.go @@ -20,5 +20,5 @@ package server import "time" -// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics. -const DefaultDiagnosticsInterval = time.Duration(0) +// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics. +const defaultDiagnosticsInterval = time.Duration(0) diff --git a/server/server.go b/server/server.go index 3c91c9fb2..76414dec4 100644 --- a/server/server.go +++ b/server/server.go @@ -222,7 +222,7 @@ func (m *Command) SetupServer() error { diagnosticsInterval := time.Duration(0) if m.Config.Metric.Diagnostics { - diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval) + diagnosticsInterval = time.Duration(defaultDiagnosticsInterval) } statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) From 167e41787fcd28bdf15f3a672a25a043fbd76175 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:16 -0500 Subject: [PATCH 152/166] Unexport server.NewStatsClient --- server/server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index 76414dec4..d71139096 100644 --- a/server/server.go +++ b/server/server.go @@ -225,7 +225,7 @@ func (m *Command) SetupServer() error { diagnosticsInterval = time.Duration(defaultDiagnosticsInterval) } - statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) if err != nil { return errors.Wrap(err, "new stats client") } @@ -351,8 +351,8 @@ func (m *Command) Close() error { return nil } -// NewStatsClient creates a stats client from the config -func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { +// newStatsClient creates a stats client from the config +func newStatsClient(name string, host string) (pilosa.StatsClient, error) { switch name { case "expvar": return pilosa.NewExpvarStatsClient(), nil From 989cc55ece4eaf3d10b2f36c7bb042c5693b2865 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:22 -0500 Subject: [PATCH 153/166] Unexport statsd.BufferLen --- statsd/statsd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/statsd/statsd.go b/statsd/statsd.go index 81cfbb2b4..0bb369fac 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -29,8 +29,8 @@ const ( // Prefix is appended to each metric event name Prefix = "pilosa." - // BufferLen Stats lient buffer size. - BufferLen = 1024 + // bufferLen Stats lient buffer size. + bufferLen = 1024 ) // Ensure client implements interface. @@ -45,7 +45,7 @@ type StatsClient struct { // NewStatsClient returns a new instance of StatsClient. func NewStatsClient(host string) (*StatsClient, error) { - c, err := statsd.NewBuffered(host, BufferLen) + c, err := statsd.NewBuffered(host, bufferLen) if err != nil { return nil, err } From 8aa24e2c389c836c959d035544e5d7e1390c81d3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:28 -0500 Subject: [PATCH 154/166] Unexport statsd.Prefix --- statsd/statsd.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/statsd/statsd.go b/statsd/statsd.go index 0bb369fac..232404a54 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -26,8 +26,8 @@ import ( // statsD defailt host is "127.0.0.1:8125" const ( - // Prefix is appended to each metric event name - Prefix = "pilosa." + // prefix is appended to each metric event name + prefix = "pilosa." // bufferLen Stats lient buffer size. bufferLen = 1024 @@ -80,7 +80,7 @@ func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { // Count tracks the number of times something occurs per second. func (c *StatsClient) Count(name string, value int64, rate float64) { - if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil { + if err := c.client.Count(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } @@ -88,35 +88,35 @@ func (c *StatsClient) Count(name string, value int64, rate float64) { // CountWithCustomTags tracks the number of times something occurs per second with custom tags. func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { tags := append(c.tags, t...) - if err := c.client.Count(Prefix+name, value, tags, rate); err != nil { + if err := c.client.Count(prefix+name, value, tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // Gauge sets the value of a metric. func (c *StatsClient) Gauge(name string, value float64, rate float64) { - if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil { + if err := c.client.Gauge(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Gauge error: %s", err) } } // Histogram tracks statistical distribution of a metric. func (c *StatsClient) Histogram(name string, value float64, rate float64) { - if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil { + if err := c.client.Histogram(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Histogram error: %s", err) } } // Set tracks number of unique elements. func (c *StatsClient) Set(name string, value string, rate float64) { - if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil { + if err := c.client.Set(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Set error: %s", err) } } // Timing tracks timing information for a metric. func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { - if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil { + if err := c.client.Timing(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Timing error: %s", err) } } From 9236df38e9d350743d703bec5a1447723bfae25e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:34 -0500 Subject: [PATCH 155/166] Unexport statsd.StatsClient --- statsd/statsd.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/statsd/statsd.go b/statsd/statsd.go index 232404a54..7a9ae6c14 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -34,44 +34,44 @@ const ( ) // Ensure client implements interface. -var _ pilosa.StatsClient = &StatsClient{} +var _ pilosa.StatsClient = &statsClient{} -// StatsClient represents a StatsD implementation of pilosa.StatsClient. -type StatsClient struct { +// statsClient represents a StatsD implementation of pilosa.statsClient. +type statsClient struct { client *statsd.Client tags []string logger pilosa.Logger } // NewStatsClient returns a new instance of StatsClient. -func NewStatsClient(host string) (*StatsClient, error) { +func NewStatsClient(host string) (*statsClient, error) { c, err := statsd.NewBuffered(host, bufferLen) if err != nil { return nil, err } - return &StatsClient{ + return &statsClient{ client: c, logger: pilosa.NopLogger, }, nil } // Open no-op -func (c *StatsClient) Open() {} +func (c *statsClient) Open() {} // Close closes the connection to the agent. -func (c *StatsClient) Close() error { +func (c *statsClient) Close() error { return c.client.Close() } // Tags returns a sorted list of tags on the client. -func (c *StatsClient) Tags() []string { +func (c *statsClient) Tags() []string { return c.tags } // WithTags returns a new client with additional tags appended. -func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { - return &StatsClient{ +func (c *statsClient) WithTags(tags ...string) pilosa.StatsClient { + return &statsClient{ client: c.client, tags: unionStringSlice(c.tags, tags), logger: c.logger, @@ -79,14 +79,14 @@ func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { } // Count tracks the number of times something occurs per second. -func (c *StatsClient) Count(name string, value int64, rate float64) { +func (c *statsClient) Count(name string, value int64, rate float64) { if err := c.client.Count(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // CountWithCustomTags tracks the number of times something occurs per second with custom tags. -func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { +func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { tags := append(c.tags, t...) if err := c.client.Count(prefix+name, value, tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Count error: %s", err) @@ -94,35 +94,35 @@ func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64 } // Gauge sets the value of a metric. -func (c *StatsClient) Gauge(name string, value float64, rate float64) { +func (c *statsClient) Gauge(name string, value float64, rate float64) { if err := c.client.Gauge(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Gauge error: %s", err) } } // Histogram tracks statistical distribution of a metric. -func (c *StatsClient) Histogram(name string, value float64, rate float64) { +func (c *statsClient) Histogram(name string, value float64, rate float64) { if err := c.client.Histogram(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Histogram error: %s", err) } } // Set tracks number of unique elements. -func (c *StatsClient) Set(name string, value string, rate float64) { +func (c *statsClient) Set(name string, value string, rate float64) { if err := c.client.Set(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Set error: %s", err) } } // Timing tracks timing information for a metric. -func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { +func (c *statsClient) Timing(name string, value time.Duration, rate float64) { if err := c.client.Timing(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Timing error: %s", err) } } // SetLogger sets the logger for client. -func (c *StatsClient) SetLogger(logger pilosa.Logger) { +func (c *statsClient) SetLogger(logger pilosa.Logger) { c.logger = logger } From 66df2cf126cb584014ce82345dbe7d24cbfd4403 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:40 -0500 Subject: [PATCH 156/166] Unexport test.BufferLogger --- test/logger.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/logger.go b/test/logger.go index b4a0079b1..60af32acb 100644 --- a/test/logger.go +++ b/test/logger.go @@ -20,20 +20,20 @@ import ( "io/ioutil" ) -// BufferLogger represents a test Logger that holds log messages +// bufferLogger represents a test Logger that holds log messages // in a buffer for review. -type BufferLogger struct { +type bufferLogger struct { buf *bytes.Buffer } // NewBufferLogger returns a new instance of BufferLogger. -func NewBufferLogger() *BufferLogger { - return &BufferLogger{ +func NewBufferLogger() *bufferLogger { + return &bufferLogger{ buf: &bytes.Buffer{}, } } -func (b *BufferLogger) Printf(format string, v ...interface{}) { +func (b *bufferLogger) Printf(format string, v ...interface{}) { s := fmt.Sprintf(format, v...) _, err := b.buf.WriteString(s) if err != nil { @@ -41,8 +41,8 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) { } } -func (b *BufferLogger) Debugf(format string, v ...interface{}) {} +func (b *bufferLogger) Debugf(format string, v ...interface{}) {} -func (b *BufferLogger) ReadAll() ([]byte, error) { +func (b *bufferLogger) ReadAll() ([]byte, error) { return ioutil.ReadAll(b.buf) } From 3aa759bff799446a54adfa75022e2085731c1a74 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:46 -0500 Subject: [PATCH 157/166] Unexport test.Command.Stdin --- test/pilosa.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index d56723e89..9e1a9a58b 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -36,7 +36,7 @@ type Command struct { commandOptions []server.CommandOption - Stdin bytes.Buffer + stdin bytes.Buffer Stdout bytes.Buffer Stderr bytes.Buffer } @@ -59,7 +59,7 @@ func NewCommand(opts ...server.CommandOption) *Command { m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true - m.Command.Stdin = &m.Stdin + m.Command.Stdin = &m.stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr From 44e0659934a7230dabad47741027d3d04fdad856 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:52 -0500 Subject: [PATCH 158/166] Unexport test.Command.Stdout --- test/pilosa.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 9e1a9a58b..f6150a59a 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -37,7 +37,7 @@ type Command struct { commandOptions []server.CommandOption stdin bytes.Buffer - Stdout bytes.Buffer + stdout bytes.Buffer Stderr bytes.Buffer } @@ -60,7 +60,7 @@ func NewCommand(opts ...server.CommandOption) *Command { m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true m.Command.Stdin = &m.stdin - m.Command.Stdout = &m.Stdout + m.Command.Stdout = &m.stdout m.Command.Stderr = &m.Stderr if testing.Verbose() { From 39504f8ded8e131f3c2be8a7f8e762e30288d2d9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:03:58 -0500 Subject: [PATCH 159/166] Unexport test.Command.Stderr --- test/pilosa.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index f6150a59a..9fb3152c5 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -38,7 +38,7 @@ type Command struct { stdin bytes.Buffer stdout bytes.Buffer - Stderr bytes.Buffer + stderr bytes.Buffer } func OptAllowedOrigins(origins []string) server.CommandOption { @@ -61,7 +61,7 @@ func NewCommand(opts ...server.CommandOption) *Command { m.Config.Cluster.Disabled = true m.Command.Stdin = &m.stdin m.Command.Stdout = &m.stdout - m.Command.Stderr = &m.Stderr + m.Command.Stderr = &m.stderr if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) From 9ddb881bedd41647530f4a20577e916da8957603 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:04:04 -0500 Subject: [PATCH 160/166] Unexport test.Field.Close --- test/field.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/field.go b/test/field.go index 7a0439026..e0ad00442 100644 --- a/test/field.go +++ b/test/field.go @@ -49,8 +49,8 @@ func MustOpenField(opts pilosa.FieldOption) *Field { return f } -// Close closes the field and removes the underlying data. -func (f *Field) Close() error { +// close closes the field and removes the underlying data. +func (f *Field) close() error { defer os.RemoveAll(f.Path()) return f.Field.Close() } @@ -77,7 +77,7 @@ func (f *Field) Reopen() error { // Ensure field can set its cache func TestField_SetCacheSize(t *testing.T) { f := MustOpenField(pilosa.OptFieldTypeDefault()) - defer f.Close() + defer f.close() cacheSize := uint32(100) // Set & retrieve field cache size. From 48730e722fc716dc586690643abf7099b25faaf5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:04:09 -0500 Subject: [PATCH 161/166] Unexport test.Field.Reopen --- test/field.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/field.go b/test/field.go index e0ad00442..140a8de11 100644 --- a/test/field.go +++ b/test/field.go @@ -55,8 +55,8 @@ func (f *Field) close() error { return f.Field.Close() } -// Reopen closes the index and reopens it. -func (f *Field) Reopen() error { +// reopen closes the index and reopens it. +func (f *Field) reopen() error { var err error if err := f.Field.Close(); err != nil { return err @@ -88,7 +88,7 @@ func TestField_SetCacheSize(t *testing.T) { } // Reload field and verify that it is persisted. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } else if q := f.CacheSize(); q != cacheSize { t.Fatalf("unexpected field cache size (reopen): %d", q) From 66771b6ccdec6970162adc665c39b5854e998c65 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:04:15 -0500 Subject: [PATCH 162/166] Unexport test.MustOpenField --- test/field.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/field.go b/test/field.go index 140a8de11..40e55f4c4 100644 --- a/test/field.go +++ b/test/field.go @@ -40,8 +40,8 @@ func NewField(opts pilosa.FieldOption) *Field { return &Field{Field: field} } -// MustOpenField returns a new, opened field at a temporary path. Panic on error. -func MustOpenField(opts pilosa.FieldOption) *Field { +// mustOpenField returns a new, opened field at a temporary path. Panic on error. +func mustOpenField(opts pilosa.FieldOption) *Field { f := NewField(opts) if err := f.Open(); err != nil { panic(err) @@ -76,7 +76,7 @@ func (f *Field) reopen() error { // Ensure field can set its cache func TestField_SetCacheSize(t *testing.T) { - f := MustOpenField(pilosa.OptFieldTypeDefault()) + f := mustOpenField(pilosa.OptFieldTypeDefault()) defer f.close() cacheSize := uint32(100) From 8e026493a407264552d75f8117875001b9ed32e4 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:04:21 -0500 Subject: [PATCH 163/166] Unexport test.NewCommand --- test/pilosa.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 9fb3152c5..e1022ce8b 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -48,8 +48,8 @@ func OptAllowedOrigins(origins []string) server.CommandOption { } } -// NewCommand returns a new instance of Main with a temporary data directory and random port. -func NewCommand(opts ...server.CommandOption) *Command { +// newCommand returns a new instance of Main with a temporary data directory and random port. +func newCommand(opts ...server.CommandOption) *Command { path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) @@ -73,7 +73,7 @@ func NewCommand(opts ...server.CommandOption) *Command { // NewCommandNode returns a new instance of Command with clustering enabled. func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { - m := NewCommand(opts...) + m := newCommand(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator return m @@ -81,7 +81,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { // MustRunCommand returns a new, running Main. Panic on error. func MustRunCommand() *Command { - m := NewCommand() + m := newCommand() m.Config.Metric.Diagnostics = false // Disable diagnostics. if err := m.Start(); err != nil { panic(err) From 17d212444c406c012d8bb27f059849eff9f8ebc7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:04:29 -0500 Subject: [PATCH 164/166] Unexport test.NewField --- test/field.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/field.go b/test/field.go index 40e55f4c4..bb5048575 100644 --- a/test/field.go +++ b/test/field.go @@ -27,8 +27,8 @@ type Field struct { *pilosa.Field } -// NewField returns a new instance of Field d/0. -func NewField(opts pilosa.FieldOption) *Field { +// newField returns a new instance of Field d/0. +func newField(opts pilosa.FieldOption) *Field { path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) @@ -42,7 +42,7 @@ func NewField(opts pilosa.FieldOption) *Field { // mustOpenField returns a new, opened field at a temporary path. Panic on error. func mustOpenField(opts pilosa.FieldOption) *Field { - f := NewField(opts) + f := newField(opts) if err := f.Open(); err != nil { panic(err) } From c004ffba3e91190d194ab32a3c2a57f01155509f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:04:37 -0500 Subject: [PATCH 165/166] Unexport test.NewIndex --- test/index.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/index.go b/test/index.go index 9b7c1684b..ae2b997a9 100644 --- a/test/index.go +++ b/test/index.go @@ -26,8 +26,8 @@ type Index struct { *pilosa.Index } -// NewIndex returns a new instance of Index. -func NewIndex() *Index { +// newIndex returns a new instance of Index. +func newIndex() *Index { path, err := ioutil.TempDir("", "pilosa-index-") if err != nil { panic(err) @@ -41,7 +41,7 @@ func NewIndex() *Index { // MustOpenIndex returns a new, opened index at a temporary path. Panic on error. func MustOpenIndex() *Index { - index := NewIndex() + index := newIndex() if err := index.Open(); err != nil { panic(err) } From 50d57ec229d871daf3191a198c820010793d8f04 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jul 2018 22:46:49 -0500 Subject: [PATCH 166/166] Unexport server.DefaultDiagnosticsInterval (in release tag) --- server/release.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/release.go b/server/release.go index 54d00b244..d988f4f81 100644 --- a/server/release.go +++ b/server/release.go @@ -20,5 +20,5 @@ package server import "time" -// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. -const DefaultDiagnosticsInterval = 1 * time.Hour +// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. +const defaultDiagnosticsInterval = 1 * time.Hour