From 8021fc389b241f5c4941305dfdba0839358e4685 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 8 Jun 2018 15:48:23 -0500 Subject: [PATCH 01/10] un-export (some) Cluster methods --- api.go | 18 +- cluster.go | 422 +++++++++----------- cluster_internal_test.go | 470 +++++++++++++++++++++- cluster_test.go | 494 ------------------------ executor.go | 8 +- executor_test.go | 66 ++-- fragment.go | 4 +- holder.go | 4 +- holder_test.go | 6 +- http/client_test.go | 11 +- server.go | 22 +- stats_test.go | 8 +- test/cluster.go | 378 +----------------- utils_test.go => utils_internal_test.go | 16 +- 14 files changed, 753 insertions(+), 1174 deletions(-) delete mode 100644 cluster_test.go rename utils_test.go => utils_internal_test.go (97%) diff --git a/api.go b/api.go index 3edb1b915..36af91b0f 100644 --- a/api.go +++ b/api.go @@ -291,7 +291,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return ErrClusterDoesNotOwnSlice } @@ -327,7 +327,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) return nil, errors.Wrap(err, "validating api method") } - return api.Cluster.SliceNodes(indexName, slice), nil + return api.Cluster.sliceNodes(indexName, slice), nil } // MarshalFragment returns an object which can write the specified fragment's data @@ -681,7 +681,7 @@ func (api *API) LongQueryTime() time.Duration { func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return nil, nil, ErrClusterDoesNotOwnSlice } @@ -709,15 +709,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return nil, nil, errors.Wrap(err, "validating api method") } - oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) - newNode = api.Cluster.NodeByID(id) + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.nodeByID(id) if newNode == nil { return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") } // If the new coordinator is this node, do the SetCoordinator directly. if newNode.ID == api.LocalID() { - return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + return oldNode, newNode, api.Cluster.setCoordinator(newNode) } // Send the set-coordinator message to new node. @@ -739,13 +739,13 @@ func (api *API) RemoveNode(id string) (*Node, error) { return nil, errors.Wrap(err, "validating api method") } - removeNode := api.Cluster.nodeByID(id) + removeNode := api.Cluster.unprotectedNodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") } // Start the resize process (similar to NodeJoin) - err := api.Cluster.NodeLeave(removeNode) + err := api.Cluster.nodeLeave(removeNode) if err != nil { return removeNode, errors.Wrap(err, "calling node leave") } @@ -758,7 +758,7 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "validating api method") } - err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + err := api.Cluster.completeCurrentJob(resizeJobStateAborted) return errors.Wrap(err, "complete current job") } diff --git a/cluster.go b/cluster.go index a5574e2cd..28feb80a3 100644 --- a/cluster.go +++ b/cluster.go @@ -49,14 +49,14 @@ const ( NodeStateLoading = "LOADING" NodeStateReady = "READY" - // ResizeJob states. - ResizeJobStateRunning = "RUNNING" + // resizeJob states. + resizeJobStateRunning = "RUNNING" // Final states. - ResizeJobStateDone = "DONE" - ResizeJobStateAborted = "ABORTED" + resizeJobStateDone = "DONE" + resizeJobStateAborted = "ABORTED" - ResizeJobActionAdd = "ADD" - ResizeJobActionRemove = "REMOVE" + resizeJobActionAdd = "ADD" + resizeJobActionRemove = "REMOVE" ) // Node represents a node in the cluster. @@ -255,8 +255,8 @@ type Cluster struct { joined bool mu sync.RWMutex - jobs map[int64]*ResizeJob - currentJob *ResizeJob + jobs map[int64]*resizeJob + currentJob *resizeJob // Close management wg sync.WaitGroup @@ -279,7 +279,7 @@ func NewCluster() *Cluster { EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel - jobs: make(map[int64]*ResizeJob), + jobs: make(map[int64]*resizeJob), closing: make(chan struct{}), joining: make(chan struct{}), @@ -289,27 +289,27 @@ func NewCluster() *Cluster { } } -// Coordinator returns the coordinator node. -func (c *Cluster) CoordinatorNode() *Node { - return c.nodeByID(c.Coordinator) +// coordinatorNode returns the coordinator node. +func (c *Cluster) coordinatorNode() *Node { + return c.unprotectedNodeByID(c.Coordinator) } -// IsCoordinator is true if this node is the coordinator. -func (c *Cluster) IsCoordinator() bool { +// isCoordinator is true if this node is the coordinator. +func (c *Cluster) isCoordinator() bool { c.mu.RLock() defer c.mu.RUnlock() - return c.isCoordinator() + return c.unprotectedIsCoordinator() } -func (c *Cluster) isCoordinator() bool { +func (c *Cluster) unprotectedIsCoordinator() bool { return c.Coordinator == c.Node.ID } -// SetCoordinator tells the current node to become the +// setCoordinator tells the current node to become the // Coordinator. In response to this, the current node // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. -func (c *Cluster) SetCoordinator(n *Node) error { +func (c *Cluster) setCoordinator(n *Node) error { c.mu.Lock() // Verify that the new Coordinator value matches // this node. @@ -319,7 +319,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { } // Update IsCoordinator on all nodes (locally). - _ = c.updateCoordinator(n) + _ = c.unprotectedUpdateCoordinator(n) c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.Broadcaster.SendSync( @@ -334,17 +334,17 @@ func (c *Cluster) SetCoordinator(n *Node) error { return c.Broadcaster.SendSync(c.Status()) } -// UpdateCoordinator updates this nodes Coordinator value as well as +// updateCoordinator updates this nodes Coordinator value as well as // changing the corresponding node's IsCoordinator value // to true, and sets all other nodes to false. Returns true if the value // changed. -func (c *Cluster) UpdateCoordinator(n *Node) bool { +func (c *Cluster) updateCoordinator(n *Node) bool { c.mu.Lock() defer c.mu.Unlock() - return c.updateCoordinator(n) + return c.unprotectedUpdateCoordinator(n) } -func (c *Cluster) updateCoordinator(n *Node) bool { +func (c *Cluster) unprotectedUpdateCoordinator(n *Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID @@ -360,9 +360,9 @@ func (c *Cluster) updateCoordinator(n *Node) bool { return changed } -// AddNode adds a node to the Cluster and updates and saves the +// addNode adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddNode(node *Node) error { +func (c *Cluster) addNode(node *Node) error { c.Logger.Printf("add node %s to cluster on %s", node, c.Node) // If the node being added is the coordinator, set it for this node. @@ -387,9 +387,9 @@ func (c *Cluster) AddNode(node *Node) error { return c.saveTopology() } -// RemoveNode removes a node from the Cluster and updates and saves the +// removeNode removes a node from the Cluster and updates and saves the // new topology. -func (c *Cluster) RemoveNode(node *Node) error { +func (c *Cluster) removeNode(node *Node) error { // remove from cluster if !c.removeNodeBasicSorted(node) { return nil @@ -407,8 +407,8 @@ func (c *Cluster) RemoveNode(node *Node) error { return c.saveTopology() } -// NodeIDs returns the list of IDs in the cluster. -func (c *Cluster) NodeIDs() []string { +// nodeIDs returns the list of IDs in the cluster. +func (c *Cluster) nodeIDs() []string { return Nodes(c.Nodes).IDs() } @@ -472,9 +472,9 @@ func (c *Cluster) setState(state string) { } } -func (c *Cluster) SetNodeState(state string) error { - if c.IsCoordinator() { - return c.ReceiveNodeState(c.Node.ID, state) +func (c *Cluster) setNodeState(state string) error { + if c.isCoordinator() { + return c.receiveNodeState(c.Node.ID, state) } // Send node state to coordinator. @@ -484,18 +484,18 @@ func (c *Cluster) SetNodeState(state string) error { } c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) - if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { + if err := c.sendTo(c.coordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } return nil } -// ReceiveNodeState sets node state in Topology in order for the +// receiveNodeState sets node state in Topology in order for the // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. -func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { - if !c.IsCoordinator() { +func (c *Cluster) receiveNodeState(nodeID string, state string) error { + if !c.isCoordinator() { return nil } @@ -515,11 +515,6 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { return nil } -// localNode is not being used. -//func (c *Cluster) localNode() *Node { -// return c.NodeByURI(c.URI) -//} - // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ @@ -529,14 +524,14 @@ func (c *Cluster) Status() *internal.ClusterStatus { } } -func (c *Cluster) NodeByID(id string) *Node { +func (c *Cluster) nodeByID(id string) *Node { c.mu.RLock() defer c.mu.RUnlock() - return c.nodeByID(id) + return c.unprotectedNodeByID(id) } -// nodeByID returns a node reference by ID. -func (c *Cluster) nodeByID(id string) *Node { +// unprotectedNodeByID returns a node reference by ID. +func (c *Cluster) unprotectedNodeByID(id string) *Node { for _, n := range c.Nodes { if n.ID == id { return n @@ -558,7 +553,7 @@ func (c *Cluster) nodePositionByID(nodeID string) int { // addNodeBasicSorted adds a node to the cluster, sorted by id. // Returns a pointer to the node and true if the node was added. func (c *Cluster) addNodeBasicSorted(node *Node) bool { - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) if n != nil { return false } @@ -645,7 +640,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { - nodes := c.SliceNodes(idx, i) + nodes := c.sliceNodes(idx, i) for _, n := range nodes { // for each field/view combination: for field, views := range fieldViews { @@ -673,10 +668,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenTo-lenFrom > 1 { return "", "", errors.New("adding more than one node at a time is not supported") } - action = ResizeJobActionAdd + action = resizeJobActionAdd // Determine the node ID that is being added. for _, n := range other.Nodes { - if c.nodeByID(n.ID) == nil { + if c.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -686,10 +681,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenFrom-lenTo > 1 { return "", "", errors.New("removing more than one node at a time is not supported") } - action = ResizeJobActionRemove + action = resizeJobActionRemove // Determine the node ID that is being removed. for _, n := range c.Nodes { - if other.nodeByID(n.ID) == nil { + if other.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -721,7 +716,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // If a node is being removed, however, then it will most likely // require that a replica fragment be the source data. srcCluster := c - if action == ResizeJobActionAdd && c.ReplicaN > 1 { + if action == resizeJobActionAdd && c.ReplicaN > 1 { srcCluster = NewCluster() srcCluster.Nodes = Nodes(c.Nodes).Clone() srcCluster.Hasher = c.Hasher @@ -740,7 +735,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R srcNodesByFrag := make(map[frag]string) for nodeID, frags := range srcFrags { // If a node is being removed, don't consider it as a source. - if action == ResizeJobActionRemove && nodeID == diffNodeID { + if action == resizeJobActionRemove && nodeID == diffNodeID { continue } for _, frag := range frags { @@ -772,7 +767,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R } src := &internal.ResizeSource{ - Node: EncodeNode(c.nodeByID(srcNodeID)), + Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)), Index: idx.Name(), Field: frag.field, View: frag.view, @@ -786,8 +781,8 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return m, nil } -// Partition returns the partition that a slice belongs to. -func (c *Cluster) Partition(index string, slice uint64) int { +// partition returns the partition that a slice belongs to. +func (c *Cluster) partition(index string, slice uint64) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], slice) @@ -798,18 +793,18 @@ func (c *Cluster) Partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.PartitionN)) } -// SliceNodes returns a list of nodes that own a fragment. -func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { - return c.PartitionNodes(c.Partition(index, slice)) +// sliceNodes returns a list of nodes that own a fragment. +func (c *Cluster) sliceNodes(index string, slice uint64) []*Node { + return c.partitionNodes(c.partition(index, slice)) } -// OwnsSlice returns true if a host owns a fragment. -func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { - return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) +// ownsSlice returns true if a host owns a fragment. +func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool { + return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID) } -// PartitionNodes returns a list of nodes that own a partition. -func (c *Cluster) PartitionNodes(partitionID int) []*Node { +// partitionNodes returns a list of nodes that own a partition. +func (c *Cluster) partitionNodes(partitionID int) []*Node { // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. replicaN := c.ReplicaN @@ -831,27 +826,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// OwnsSlices finds the set of slices owned by the node per Index -func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { +// containsSlices is like OwnsSlices, but it includes replicas. +func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) - // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[nodeIndex].URI == uri { - slices = append(slices, i) - } - } - return slices -} - -// ContainsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) + p := c.partition(index, i) // Determine the nodes for partition. - nodes := c.PartitionNodes(p) + nodes := c.partitionNodes(p) for _, n := range nodes { if n.ID == node.ID { slices = append(slices, i) @@ -884,7 +865,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { return int(b) } -func (c *Cluster) Open() error { +func (c *Cluster) open() error { // Cluster always comes up in state STARTING until cluster membership is determined. c.state = ClusterStateStarting @@ -896,7 +877,7 @@ func (c *Cluster) Open() error { c.ID = c.Topology.ClusterID // Only the coordinator needs to consider the .topology file. - if c.IsCoordinator() { + if c.isCoordinator() { err := c.considerTopology() if err != nil { return fmt.Errorf("considerTopology: %v", err) @@ -904,7 +885,7 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. - err := c.AddNode(c.Node) + err := c.addNode(c.Node) if err != nil { return errors.Wrap(err, "adding local node") } @@ -920,7 +901,7 @@ func (c *Cluster) Open() error { } // If not coordinator then wait for ClusterStatus from coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { // In the case where a node has been restarted and memberlist has // not had enough time to determine the node went down/up, then // the coorninator needs to be alerted that this node is back up @@ -945,7 +926,7 @@ func (c *Cluster) Open() error { return nil } -func (c *Cluster) Close() error { +func (c *Cluster) close() error { // Notify goroutines of closing and wait for completion. close(c.closing) c.wg.Wait() @@ -962,14 +943,14 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) allNodesReady() bool { @@ -999,10 +980,10 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { // channel, which is not consumed until the code below. var eg errgroup.Group eg.Go(func() error { - return j.Run() + return j.run() }) - // Wait for the ResizeJob to finish or be aborted. + // Wait for the resizeJob to finish or be aborted. c.Logger.Printf("wait for jobResult") jobResult := <-j.result @@ -1013,18 +994,18 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { c.Logger.Printf("received jobResult: %s", jobResult) switch jobResult { - case ResizeJobStateDone: - if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { + case resizeJobStateDone: + if err := c.completeCurrentJob(resizeJobStateDone); err != nil { return errors.Wrap(err, "completing finished job") } // Add/remove uri to/from the cluster. - if j.action == ResizeJobActionRemove { - return c.RemoveNode(nodeAction.node) - } else if j.action == ResizeJobActionAdd { - return c.AddNode(nodeAction.node) + if j.action == resizeJobActionRemove { + return c.removeNode(nodeAction.node) + } else if j.action == resizeJobActionAdd { + return c.addNode(nodeAction.node) } - case ResizeJobStateAborted: - if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { + case resizeJobStateAborted: + if err := c.completeCurrentJob(resizeJobStateAborted); err != nil { return errors.Wrap(err, "completing aborted job") } } @@ -1045,64 +1026,63 @@ func (c *Cluster) sendTo(node *Node, msg proto.Message) error { return nil } -// ListenForJoins handles cluster-resize events. -func (c *Cluster) ListenForJoins() { - c.wg.Add(1) - go func() { defer c.wg.Done(); c.listenForJoins() }() -} - +// listenForJoins handles cluster-resize events. func (c *Cluster) listenForJoins() { - // When a cluster starts, the state is STARTING. - // We first want to wait for at least one node to join. - // Then we want to clear out the joiningLeavingNodes queue (buffered channel). - // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. - // We use a bool `setNormal` to indicate when at least one node has joined. + c.wg.Add(1) + go func() { + defer c.wg.Done() - var setNormal bool + // When a cluster starts, the state is STARTING. + // We first want to wait for at least one node to join. + // Then we want to clear out the joiningLeavingNodes queue (buffered channel). + // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. + // We use a bool `setNormal` to indicate when at least one node has joined. + var setNormal bool - for { + for { - // Handle all pending joins before changing state back to NORMAL. - select { - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) + // Handle all pending joins before changing state back to NORMAL. + select { + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true + continue + default: + } + + // Only change state to NORMAL if we have successfully added at least one host. + if setNormal { + // Put the cluster back to state NORMAL and broadcast. + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) + } + } + + // Wait for a joining host or a close. + select { + case <-c.closing: + return + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true continue } - setNormal = true - continue - default: } - - // Only change state to NORMAL if we have successfully added at least one host. - if setNormal { - // Put the cluster back to state NORMAL and broadcast. - if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.Logger.Printf("setStateAndBroadcast error: err=%s", err) - } - } - - // Wait for a joining host or a close. - select { - case <-c.closing: - return - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) - continue - } - setNormal = true - continue - } - } + }() } -// generateResizeJob creates a new ResizeJob based on the new node being -// added/removed. It also saves a reference to the ResizeJob in the `jobs` map +// generateResizeJob creates a new resizeJob based on the new node being +// added/removed. It also saves a reference to the resizeJob in the `jobs` map // for future lookup by JobID. -func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { +func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) { c.Logger.Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -1111,7 +1091,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, errors.Wrap(err, "generating job") } - c.Logger.Printf("generated ResizeJob: %d", j.ID) + c.Logger.Printf("generated resizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1125,12 +1105,12 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { return j, nil } -// generateResizeJobByAction returns a ResizeJob with instructions based on +// generateResizeJobByAction returns a resizeJob with instructions based on // the difference between Cluster and a new Cluster with/without uri. -// Broadcaster is associated to the ResizeJob here for use in broadcasting +// Broadcaster is associated to the resizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. -func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { - j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action) +func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) { + j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action) j.Broadcaster = c.Broadcaster // toCluster is a clone of Cluster with the new node added/removed for comparison. @@ -1139,9 +1119,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, toCluster.Hasher = c.Hasher toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN - if nodeAction.action == ResizeJobActionRemove { + if nodeAction.action == resizeJobActionRemove { toCluster.removeNodeBasicSorted(nodeAction.node) - } else if nodeAction.action == ResizeJobActionAdd { + } else if nodeAction.action == resizeJobActionAdd { toCluster.addNodeBasicSorted(nodeAction.node) } @@ -1172,8 +1152,8 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, } instr := &internal.ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.nodeByID(id)), - Coordinator: EncodeNode(c.CoordinatorNode()), + Node: EncodeNode(toCluster.unprotectedNodeByID(id)), + Coordinator: EncodeNode(c.coordinatorNode()), Sources: sources, Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), @@ -1184,28 +1164,28 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, return j, nil } -// CompleteCurrentJob sets the state of the current ResizeJob +// completeCurrentJob sets the state of the current resizeJob // then removes the pointer to currentJob. -func (c *Cluster) CompleteCurrentJob(state string) error { +func (c *Cluster) completeCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() - if !c.isCoordinator() { + if !c.unprotectedIsCoordinator() { return ErrNodeNotCoordinator } if c.currentJob == nil { return ErrResizeNotRunning } - c.currentJob.SetState(state) + c.currentJob.setState(state) c.currentJob = nil return nil } -// FollowResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +// followResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) error { c.Logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. - if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { + if err := c.mergeClusterStatus(instr.ClusterStatus); err != nil { return errors.Wrap(err, "merging cluster status") } @@ -1297,13 +1277,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return nil } -func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *Cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { - j := c.Job(complete.JobID) + j := c.job(complete.JobID) // Abort the job if an error exists in the complete object. if complete.Error != "" { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.New(complete.Error) } @@ -1311,29 +1291,27 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc defer j.mu.Unlock() if j.isComplete() { - return fmt.Errorf("ResizeJob %d is no longer running", j.ID) + return fmt.Errorf("resize job %d is no longer running", j.ID) } // Mark host complete. j.IDs[complete.Node.ID] = true if !j.nodesArePending() { - j.result <- ResizeJobStateDone + j.result <- resizeJobStateDone } return nil } -// Job returns a ResizeJob by id. -func (c *Cluster) Job(id int64) *ResizeJob { +// job returns a resizeJob by id. +func (c *Cluster) job(id int64) *resizeJob { c.mu.RLock() defer c.mu.RUnlock() - return c.job(id) + return c.jobs[id] } -func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } - -type ResizeJob struct { +type resizeJob struct { ID int64 IDs map[string]bool Instructions []*internal.ResizeInstruction @@ -1348,15 +1326,15 @@ type ResizeJob struct { Logger Logger } -// NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { +// newResizeJob returns a new instance of resizeJob. +func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob { // Build a map of uris to track their resize status. // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. ids := make(map[string]bool) - if action == ResizeJobActionRemove { + if action == resizeJobActionRemove { for _, n := range existingNodes { // Exclude the removed node from the map. if n.ID == node.ID { @@ -1364,7 +1342,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } ids[n.ID] = false } - } else if action == ResizeJobActionAdd { + } else if action == resizeJobActionAdd { for _, n := range existingNodes { ids[n.ID] = false } @@ -1372,7 +1350,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { ids[node.ID] = false } - return &ResizeJob{ + return &resizeJob{ ID: rand.Int63(), IDs: ids, action: action, @@ -1381,50 +1359,40 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } } -func (j *ResizeJob) State() string { - j.mu.RLock() - defer j.mu.RUnlock() - return j.state -} - -func (j *ResizeJob) SetState(state string) { +func (j *resizeJob) setState(state string) { j.mu.Lock() - j.setState(state) + if j.state == "" || j.state == resizeJobStateRunning { + j.state = state + } j.mu.Unlock() } -func (j *ResizeJob) setState(state string) { - if j.state == "" || j.state == ResizeJobStateRunning { - j.state = state - } -} - -// Run distributes ResizeInstructions. -func (j *ResizeJob) Run() error { - j.Logger.Printf("run ResizeJob") +// run distributes ResizeInstructions. +func (j *resizeJob) run() error { + j.Logger.Printf("run resizeJob") // Set job state to RUNNING. - j.SetState(ResizeJobStateRunning) + j.setState(resizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.nodesArePending() { - j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") - j.result <- ResizeJobStateDone + j.Logger.Printf("resizeJob contains no pending tasks; mark as done") + j.result <- resizeJobStateDone return nil } - j.Logger.Printf("distribute tasks for ResizeJob") + j.Logger.Printf("distribute tasks for resizeJob") err := j.distributeResizeInstructions() if err != nil { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.Wrap(err, "distributing instructions") } return nil } // isComplete return true if the job is any one of several completion states. -func (j *ResizeJob) isComplete() bool { +func (j *resizeJob) isComplete() bool { switch j.state { - case ResizeJobStateDone, ResizeJobStateAborted: + case resizeJobStateDone, resizeJobStateAborted: return true default: return false @@ -1432,7 +1400,7 @@ func (j *ResizeJob) isComplete() bool { } // nodesArePending returns true if any node is still working on the resize. -func (j *ResizeJob) nodesArePending() bool { +func (j *resizeJob) nodesArePending() bool { for _, complete := range j.IDs { if !complete { return true @@ -1441,9 +1409,9 @@ func (j *ResizeJob) nodesArePending() bool { return false } -func (j *ResizeJob) distributeResizeInstructions() error { +func (j *resizeJob) distributeResizeInstructions() error { j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) - // Loop through the ResizeInstructions in ResizeJob and send to each host. + // Loop through the ResizeInstructions in resizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. @@ -1659,7 +1627,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { case NodeJoin: c.Logger.Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { return nil } return c.nodeJoin(e.Node) @@ -1681,7 +1649,7 @@ func (c *Cluster) nodeJoin(node *Node) error { return errors.New(err) } - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node for agreement") } @@ -1711,13 +1679,13 @@ func (c *Cluster) nodeJoin(node *Node) error { // If the cluster already contains the node, just send it the cluster status. // This is useful in the case where a node is restarted or temporarily leaves // the cluster. - if node := c.nodeByID(node.ID); node != nil { + if node := c.unprotectedNodeByID(node.ID); node != nil { return c.sendTo(node, c.Status()) } // If the holder does not yet contain data, go ahead and add the node. if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1730,16 +1698,16 @@ func (c *Cluster) nodeJoin(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} + c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd} return nil } -// NodeLeave initiates the removal of a node from the cluster. -func (c *Cluster) NodeLeave(node *Node) error { +// nodeLeave initiates the removal of a node from the cluster. +func (c *Cluster) nodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. - if !c.IsCoordinator() { - return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) + if !c.isCoordinator() { + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID) } if c.State() != ClusterStateNormal { @@ -1747,7 +1715,7 @@ func (c *Cluster) NodeLeave(node *Node) error { } // Ensure that node is in the cluster. - if c.nodeByID(node.ID) == nil { + if c.unprotectedNodeByID(node.ID) == nil { return fmt.Errorf("Node is not a member of the cluster: %s", node.ID) } @@ -1757,18 +1725,12 @@ func (c *Cluster) NodeLeave(node *Node) error { } // See if resize job can be generated - _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) - - if err != nil { + if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil { return errors.Wrap(err, "generating job") } - return c.nodeLeave(node) -} - -func (c *Cluster) nodeLeave(node *Node) error { // Get the actual node in the local cluster. - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) // Don't do anything else if the cluster doesn't contain the node. if n == nil { @@ -1777,7 +1739,7 @@ func (c *Cluster) nodeLeave(node *Node) error { // If the holder does not yet contain data, go ahead and remove the node. if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.RemoveNode(n); err != nil { + if err := c.removeNode(n); err != nil { return errors.Wrap(err, "removing node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1790,17 +1752,17 @@ func (c *Cluster) nodeLeave(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} + c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove} return nil } -func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). - if c.isCoordinator() { + if c.unprotectedIsCoordinator() { return nil } @@ -1811,7 +1773,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { // Add all nodes from the coordinator. for _, node := range officialNodes { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } } @@ -1832,7 +1794,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } for _, nodeID := range nodeIDsToRemove { - if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { + if err := c.removeNode(c.unprotectedNodeByID(nodeID)); err != nil { return errors.Wrap(err, "removing node") } } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index f97775b96..1fd238911 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -15,11 +15,15 @@ package pilosa import ( + "bytes" "io/ioutil" + "math/rand" "reflect" "strings" "testing" + "testing/quick" + "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/internal" ) @@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) { { existingNodes: []*Node{node0, node1}, node: node2, - action: ResizeJobActionAdd, + action: resizeJobActionAdd, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, }, { existingNodes: []*Node{node0, node1, node2}, node: node2, - action: ResizeJobActionRemove, + action: resizeJobActionRemove, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, }, } for _, test := range tests { - actual := NewResizeJob(test.existingNodes, test.node, test.action) + actual := newResizeJob(test.existingNodes, test.node, test.action) if err != nil { t.Fatal(err) } @@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) { } } } + +// Ensure the cluster can fairly distribute partitions across the nodes. +func TestCluster_Owners(t *testing.T) { + c := Cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, + }, + Hasher: NewTestModHasher(), + ReplicaN: 2, + } + + // Verify nodes are distributed. + if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } + + // Verify nodes go around the ring. + if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } +} + +// Ensure the partitioner can assign a fragment to a partition. +func TestCluster_Partition(t *testing.T) { + if err := quick.Check(func(index string, slice uint64, partitionN int) bool { + c := NewCluster() + c.PartitionN = partitionN + + partitionID := c.partition(index, slice) + if partitionID < 0 || partitionID >= partitionN { + t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) + } + + return true + }, &quick.Config{ + Values: func(values []reflect.Value, rand *rand.Rand) { + values[0], _ = quick.Value(reflect.TypeOf(""), rand) + values[1] = reflect.ValueOf(uint64(rand.Uint32())) + values[2] = reflect.ValueOf(rand.Intn(1000) + 1) + }, + }); err != nil { + t.Fatal(err) + } +} + +// Ensure the hasher can hash correctly. +func TestHasher(t *testing.T) { + for _, tt := range []struct { + key uint64 + bucket []int + }{ + // Generated from the reference C++ code + {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, + {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, + {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, + } { + for i, v := range tt.bucket { + if got := NewHasher().Hash(tt.key, i+1); got != v { + t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) + } + } + } +} + +// Ensure ContainsSlices can find the actual slice list for node and index. +func TestCluster_ContainsSlices(t *testing.T) { + c := NewTestCluster(5) + c.ReplicaN = 3 + slices := c.containsSlices("test", 10, c.Nodes[2]) + + if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected slices for node's index: %v", slices) + } +} + +func TestCluster_Nodes(t *testing.T) { + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} + + nodes := []*Node{node0, node1, node2} + + t.Run("NodeIDs", func(t *testing.T) { + actual := Nodes(nodes).IDs() + expected := []string{node0.ID, node1.ID, node2.ID} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Filter", func(t *testing.T) { + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("FilterURI", func(t *testing.T) { + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Contains", func(t *testing.T) { + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("Clone", func(t *testing.T) { + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} + +// NEXT: move this test to internal and unexport IsCoordinator +func TestCluster_Coordinator(t *testing.T) { + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + + c1 := *NewCluster() + c1.Node = node1 + c1.Coordinator = node1.ID + c2 := *NewCluster() + c2.Node = node2 + c2.Coordinator = node1.ID + + t.Run("IsCoordinator", func(t *testing.T) { + if !c1.isCoordinator() { + t.Errorf("!IsCoordinator error: %v", c1.Node) + } else if c2.isCoordinator() { + t.Errorf("IsCoordinator error: %v", c2.Node) + } + }) +} + +func TestCluster_Topology(t *testing.T) { + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} + + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} + + t.Run("AddNode", func(t *testing.T) { + err := c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + // add the same host. + err = c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + err = c1.addNode(node2) + if err != nil { + t.Fatal(err) + } + + actual := c1.nodeIDs() + expected := []string{node0.ID, node1.ID, node2.ID} + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("ContainsID", func(t *testing.T) { + if !c1.Topology.ContainsID(node1.ID) { + t.Errorf("!ContainsHost error: %v", node1.ID) + } else if c1.Topology.ContainsID(nodeinvalid.ID) { + t.Errorf("ContainsHost error: %v", nodeinvalid.ID) + } + }) +} + +// Ensure that general cluster functionality works as expected. +func TestCluster_ResizeStates(t *testing.T) { + + t.Run("Single node, no data", func(t *testing.T) { + tc := NewClusterCluster(1) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + node := tc.Clusters[0] + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"some-other-host"}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" + err := tc.Open() + if err == nil || err.Error() != expected { + t.Errorf("did not receive expected error: %s", expected) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, no data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + tc.AddNode(false) + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"node0", "node2"}, + } + tc.WriteTopology(node0.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node is in state STARTING before the other node joins. + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) + } + + // Expect an error by adding a node not in the topology. + expectedError := "host is not in topology: node1" + err := tc.AddNode(false) + if err == nil || err.Error() != expectedError { + t.Errorf("did not receive expected error: %s", expectedError) + } + + tc.AddNode(false) + node2 := tc.Clusters[2] + + // Ensure that node comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, with data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Add Bit Data to node0. + if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { + t.Fatal(err) + } + tc.SetBit("i", "f", "standard", 1, 101, nil) + tc.SetBit("i", "f", "standard", 1, 1300000, nil) + + // Before starting the resize, get the CheckSum to use for + // comparison later. + node0Field := node0.Holder.Field("i", "f") + node0View := node0Field.View("standard") + node0Fragment := node0View.Fragment(1) + node0Checksum := node0Fragment.Checksum() + + // AddNode needs to block until the resize process has completed. + tc.AddNode(false) + node1 := tc.Clusters[1] + + // Ensure that nodes come up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Bits + // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. + node1Field := node1.Holder.Field("i", "f") + node1View := node1Field.View("standard") + node1Fragment := node1View.Fragment(1) + + // Ensure checksums are the same. + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) +} + +// Ensures that coordinator can be changed. +func TestCluster_UpdateCoordinator(t *testing.T) { + t.Run("UpdateCoordinator", func(t *testing.T) { + c := NewTestCluster(2) + + oldNode := c.Nodes[0] + newNode := c.Nodes[1] + + // Update coordinator to the same value. + if c.updateCoordinator(oldNode) { + t.Errorf("did not expect coordinator to change") + } else if c.Coordinator != oldNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) + } + + // Update coordinator to a new value. + if !c.updateCoordinator(newNode) { + t.Errorf("expected coordinator to change") + } else if c.Coordinator != newNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) + } + }) +} diff --git a/cluster_test.go b/cluster_test.go deleted file mode 100644 index a977b3536..000000000 --- a/cluster_test.go +++ /dev/null @@ -1,494 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "math/rand" - "reflect" - "testing" - "testing/quick" - - "github.com/davecgh/go-spew/spew" -) - -// Ensure the cluster can fairly distribute partitions across the nodes. -func TestCluster_Owners(t *testing.T) { - c := Cluster{ - Nodes: []*Node{ - {URI: NewTestURIFromHostPort("serverA", 1000)}, - {URI: NewTestURIFromHostPort("serverB", 1000)}, - {URI: NewTestURIFromHostPort("serverC", 1000)}, - }, - Hasher: NewTestModHasher(), - ReplicaN: 2, - } - - // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } - - // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } -} - -// Ensure the partitioner can assign a fragment to a partition. -func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := NewCluster() - c.PartitionN = partitionN - - partitionID := c.Partition(index, slice) - if partitionID < 0 || partitionID >= partitionN { - t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) - } - - return true - }, &quick.Config{ - Values: func(values []reflect.Value, rand *rand.Rand) { - values[0], _ = quick.Value(reflect.TypeOf(""), rand) - values[1] = reflect.ValueOf(uint64(rand.Uint32())) - values[2] = reflect.ValueOf(rand.Intn(1000) + 1) - }, - }); err != nil { - t.Fatal(err) - } -} - -// Ensure the hasher can hash correctly. -func TestHasher(t *testing.T) { - for _, tt := range []struct { - key uint64 - bucket []int - }{ - // Generated from the reference C++ code - {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, - {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, - {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, - {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, - } { - for i, v := range tt.bucket { - if got := NewHasher().Hash(tt.key, i+1); got != v { - t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) - } - } - } -} - -// Ensure OwnsSlices can find the actual slice list for node and index. -func TestCluster_OwnsSlices(t *testing.T) { - c := NewTestCluster(5) - slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) - - if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -// Ensure ContainsSlices can find the actual slice list for node and index. -func TestCluster_ContainsSlices(t *testing.T) { - c := NewTestCluster(5) - c.ReplicaN = 3 - slices := c.ContainsSlices("test", 10, c.Nodes[2]) - - if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -func TestCluster_Nodes(t *testing.T) { - uri0 := NewTestURIFromHostPort("node0", 0) - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - uri3 := NewTestURIFromHostPort("node3", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - node3 := &Node{ID: "node3", URI: uri3} - - nodes := []*Node{node0, node1, node2} - - t.Run("NodeIDs", func(t *testing.T) { - actual := Nodes(nodes).IDs() - expected := []string{node0.ID, node1.ID, node2.ID} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Filter", func(t *testing.T) { - actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("FilterURI", func(t *testing.T) { - actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Contains", func(t *testing.T) { - actualTrue := Nodes(nodes).Contains(node1) - actualFalse := Nodes(nodes).Contains(node3) - if !reflect.DeepEqual(actualTrue, true) { - t.Errorf("expected: %v, but got: %v", true, actualTrue) - } - if !reflect.DeepEqual(actualFalse, false) { - t.Errorf("expected: %v, but got: %v", false, actualTrue) - } - }) - - t.Run("Clone", func(t *testing.T) { - clone := Nodes(nodes).Clone() - actual := Nodes(clone).URIs() - expected := []URI{uri0, uri1, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) -} - -func TestCluster_Coordinator(t *testing.T) { - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - - c1 := *NewCluster() - c1.Node = node1 - c1.Coordinator = node1.ID - c2 := *NewCluster() - c2.Node = node2 - c2.Coordinator = node1.ID - - t.Run("IsCoordinator", func(t *testing.T) { - if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.Node) - } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.Node) - } - }) -} - -func TestCluster_Topology(t *testing.T) { - c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} - - uri0 := NewTestURIFromHostPort("host0", 0) - uri1 := NewTestURIFromHostPort("host1", 0) - uri2 := NewTestURIFromHostPort("host2", 0) - invalid := NewTestURIFromHostPort("invalid", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} - - t.Run("AddNode", func(t *testing.T) { - err := c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - // add the same host. - err = c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - err = c1.AddNode(node2) - if err != nil { - t.Fatal(err) - } - - actual := c1.NodeIDs() - expected := []string{node0.ID, node1.ID, node2.ID} - - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("ContainsID", func(t *testing.T) { - if !c1.Topology.ContainsID(node1.ID) { - t.Errorf("!ContainsHost error: %v", node1.ID) - } else if c1.Topology.ContainsID(nodeinvalid.ID) { - t.Errorf("ContainsHost error: %v", nodeinvalid.ID) - } - }) -} - -// Ensure that general cluster functionality works as expected. -func TestCluster_ResizeStates(t *testing.T) { - - t.Run("Single node, no data", func(t *testing.T) { - tc := NewClusterCluster(1) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - node := tc.Clusters[0] - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"some-other-host"}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" - err := tc.Open() - if err == nil || err.Error() != expected { - t.Errorf("did not receive expected error: %s", expected) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - tc.AddNode(false) - - node0 := tc.Clusters[0] - node1 := tc.Clusters[1] - - // Ensure that nodes comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"node0", "node2"}, - } - tc.WriteTopology(node0.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node is in state STARTING before the other node joins. - if node0.State() != ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) - } - - // Expect an error by adding a node not in the topology. - expectedError := "host is not in topology: node1" - err := tc.AddNode(false) - if err == nil || err.Error() != expectedError { - t.Errorf("did not receive expected error: %s", expectedError) - } - - tc.AddNode(false) - node2 := tc.Clusters[2] - - // Ensure that node comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node2.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Add Bit Data to node0. - if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { - t.Fatal(err) - } - tc.SetBit("i", "f", "standard", 1, 101, nil) - tc.SetBit("i", "f", "standard", 1, 1300000, nil) - - // Before starting the resize, get the CheckSum to use for - // comparison later. - node0Field := node0.Holder.Field("i", "f") - node0View := node0Field.View("standard") - node0Fragment := node0View.Fragment(1) - node0Checksum := node0Fragment.Checksum() - - // AddNode needs to block until the resize process has completed. - tc.AddNode(false) - node1 := tc.Clusters[1] - - // Ensure that nodes come up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Bits - // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node1Field := node1.Holder.Field("i", "f") - node1View := node1Field.View("standard") - node1Fragment := node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { - t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) -} - -// Ensures that coordinator can be changed. -func TestCluster_UpdateCoordinator(t *testing.T) { - t.Run("UpdateCoordinator", func(t *testing.T) { - c := NewTestCluster(2) - - oldNode := c.Nodes[0] - newNode := c.Nodes[1] - - // Update coordinator to the same value. - if c.UpdateCoordinator(oldNode) { - t.Errorf("did not expect coordinator to change") - } else if c.Coordinator != oldNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) - } - - // Update coordinator to a new value. - if !c.UpdateCoordinator(newNode) { - t.Errorf("expected coordinator to change") - } else if c.Coordinator != newNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) - } - }) -} diff --git a/executor.go b/executor.go index 17fef6165..241b7c8ec 100644 --- a/executor.go +++ b/executor.go @@ -1002,7 +1002,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) @@ -1078,7 +1078,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) @@ -1414,7 +1414,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m loop: for _, slice := range slices { - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop @@ -1444,7 +1444,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} + nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)} } // Start mapping across all primary owners. diff --git a/executor_test.go b/executor_test.go index 221a554e8..3d0d9854e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -38,7 +38,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ @@ -87,7 +87,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ @@ -113,7 +113,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, 4) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { @@ -127,7 +127,7 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "general", 10, 1) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } @@ -145,7 +145,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { @@ -158,7 +158,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } @@ -175,7 +175,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { @@ -189,7 +189,7 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "general", 10, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { @@ -208,7 +208,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { @@ -224,7 +224,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.SetBit("i", "f", 10, SliceWidth+1) hldr.SetBit("i", "f", 10, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { @@ -240,7 +240,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { // set a bit so the view gets created. hldr.SetBit("i", "f", 1, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if n := hldr.Row("i", "f", 11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -284,7 +284,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil { @@ -322,21 +322,21 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) } @@ -359,7 +359,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other bitmaps and fields to test isolation. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } @@ -385,7 +385,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { func TestExecutor_Execute_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -437,7 +437,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.SetBit("i", "f", 1, SliceWidth) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -471,7 +471,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.SetBit("i", "f", 4, 3*SliceWidth+1) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -506,7 +506,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -530,7 +530,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -553,7 +553,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -567,7 +567,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { func TestExecutor_Execute_MinMax(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -662,7 +662,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -733,7 +733,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { func TestExecutor_Execute_BSIGroupRange(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -775,7 +775,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -955,7 +955,7 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1003,7 +1003,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { // Ensure a remote query can return a count. func TestExecutor_Execute_Remote_Count(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1038,7 +1038,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. @@ -1090,7 +1090,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. @@ -1144,7 +1144,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Ensure a remote query can return a top-n query. func TestExecutor_Execute_Remote_TopN(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1213,7 +1213,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) @@ -1229,7 +1229,7 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { targetAttrs := map[string]interface{}{ "foo": "bar", } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // SetColumnAttrs call should exclude the field attribute _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil) diff --git a/fragment.go b/fragment.go index 830033c63..012c62503 100644 --- a/fragment.go +++ b/fragment.go @@ -1740,7 +1740,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice) + nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice) if len(nodes) == 1 { return nil } @@ -1821,7 +1821,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var uris []*URI var pairSets []pairSet - for _, node := range s.Cluster.SliceNodes(f.index, f.slice) { + for _, node := range s.Cluster.sliceNodes(f.index, f.slice) { if s.Node.ID == node.ID { continue } diff --git a/holder.go b/holder.go index b3147d897..e7af65643 100644 --- a/holder.go +++ b/holder.go @@ -619,7 +619,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { + if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) { continue } @@ -799,7 +799,7 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) + containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node) // Get the fragments registered in memory. for _, field := range index.Fields() { diff --git a/holder_test.go b/holder_test.go index b56863bb0..31b50b900 100644 --- a/holder_test.go +++ b/holder_test.go @@ -383,7 +383,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) cluster.Nodes[1].URI = *uri // Create fields on nodes. @@ -456,7 +456,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Ensure holder can clean up orphaned fragments. func TestHolderCleaner_CleanHolder(t *testing.T) { - cluster := test.NewCluster(2) + cluster := pilosa.NewTestCluster(2) // Create a local holder. hldr0 := test.MustOpenHolder() @@ -465,7 +465,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) // Create fields on nodes. for _, hldr := range []*test.Holder{hldr0} { diff --git a/http/client_test.go b/http/client_test.go index 1cc972ff6..851f1140e 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -87,10 +87,17 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} + + // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())` + owns := [][]uint64{ + {1, 3, 4, 8, 10, 13, 17, 19}, + {2, 5, 7, 11, 12, 14, 18}, + {0, 6, 9, 15, 16, 20}, + } + for i, num := range sliceNums { - owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false - for _, ownNum := range owns { + for _, ownNum := range owns[i] { if ownNum == num { ownsNum = true break diff --git a/server.go b/server.go index b1f0f3d10..f330b6e6a 100644 --- a/server.go +++ b/server.go @@ -332,7 +332,7 @@ func (s *Server) Open() error { } // Open Cluster management. - if err := s.Cluster.Open(); err != nil { + if err := s.Cluster.open(); err != nil { return fmt.Errorf("opening Cluster: %v", err) } @@ -340,7 +340,7 @@ func (s *Server) Open() error { if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } - if err := s.Cluster.SetNodeState(NodeStateReady); err != nil { + if err := s.Cluster.setNodeState(NodeStateReady); err != nil { return fmt.Errorf("setting nodeState: %v", err) } @@ -349,7 +349,7 @@ func (s *Server) Open() error { // the cluster without waiting for data to load on the coordinator. Before // this starts, the joins are queued up in the Cluster.joiningLeavingNodes // buffered channel. - s.Cluster.ListenForJoins() + s.Cluster.listenForJoins() // Start background monitoring. s.wg.Add(3) @@ -370,7 +370,7 @@ func (s *Server) Close() error { s.ln.Close() } if s.Cluster != nil { - s.Cluster.Close() + s.Cluster.close() } if s.Holder != nil { s.Holder.Close() @@ -493,26 +493,26 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.ClusterStatus: - err := s.Cluster.MergeClusterStatus(obj) + err := s.Cluster.mergeClusterStatus(obj) if err != nil { return err } case *internal.ResizeInstruction: - err := s.Cluster.FollowResizeInstruction(obj) + err := s.Cluster.followResizeInstruction(obj) if err != nil { return err } case *internal.ResizeInstructionComplete: - err := s.Cluster.MarkResizeInstructionComplete(obj) + err := s.Cluster.markResizeInstructionComplete(obj) if err != nil { return err } case *internal.SetCoordinatorMessage: - s.Cluster.SetCoordinator(DecodeNode(obj.New)) + s.Cluster.setCoordinator(DecodeNode(obj.New)) case *internal.UpdateCoordinatorMessage: - s.Cluster.UpdateCoordinator(DecodeNode(obj.New)) + s.Cluster.updateCoordinator(DecodeNode(obj.New)) case *internal.NodeStateMessage: - err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) + err := s.Cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } @@ -650,7 +650,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) - s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.nodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.NodeID) diff --git a/stats_test.go b/stats_test.go index ddf577918..6644786cf 100644 --- a/stats_test.go +++ b/stats_test.go @@ -95,7 +95,7 @@ func TestStatsCount_TopN(t *testing.T) { // Execute query. called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "TopN" { @@ -124,7 +124,7 @@ func TestStatsCount_Bitmap(t *testing.T) { hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "Bitmap" { @@ -154,7 +154,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) field := e.Holder.Field("d", "f") if field == nil { t.Fatal("field not found") @@ -184,7 +184,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx := e.Holder.Index("d") if idx == nil { t.Fatal("idex not found") diff --git a/test/cluster.go b/test/cluster.go index 6b1efe665..d1ee6af80 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -15,17 +15,10 @@ package test import ( - "bufio" - "bytes" "fmt" "io/ioutil" - "path/filepath" - "sync" - "time" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" ) // NewCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -37,14 +30,14 @@ func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() c.ReplicaN = 1 - c.Hasher = NewModHasher() + c.Hasher = newModHasher() c.Path = path c.Topology = pilosa.NewTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ ID: fmt.Sprintf("node%d", i), - URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), + URI: newURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) } @@ -55,372 +48,19 @@ func NewCluster(n int) *pilosa.Cluster { return c } -// ModHasher represents a simple, mod-based hashing. -type ModHasher struct{} +// modHasher represents a simple, mod-based hashing. +type modHasher struct{} -// NewModHasher returns a new instance of ModHasher with n buckets. -func NewModHasher() *ModHasher { return &ModHasher{} } +// newModHasher returns a new instance of ModHasher with n buckets. +func newModHasher() *modHasher { return &modHasher{} } -func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } +func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } -// ConstHasher represents hash that always returns the same index. -type ConstHasher struct { - i int -} - -// NewConstHasher returns a new instance of ConstHasher that always returns i. -func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } - -func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } - -// NewURI is a test URI creator that intentionally swallows errors. -func NewURI(scheme, host string, port uint16) pilosa.URI { +// newURI is a test URI creator that intentionally swallows errors. +func newURI(scheme, host string, port uint16) pilosa.URI { uri := pilosa.DefaultURI() uri.SetScheme(scheme) uri.SetHost(host) uri.SetPort(port) return *uri } - -func NewURIFromHostPort(host string, port uint16) pilosa.URI { - uri := pilosa.DefaultURI() - uri.SetHost(host) - uri.SetPort(port) - return *uri -} - -// TestCluster represents a cluster of test nodes, each of which -// has a pilosa.Cluster. -type TestCluster struct { - Clusters []*pilosa.Cluster - - common *commonClusterSettings - - mu sync.RWMutex - resizing bool - resizeDone chan struct{} -} - -type commonClusterSettings struct { - Nodes []*pilosa.Node -} - -func (t *TestCluster) CreateIndex(name string) error { - for _, c := range t.Clusters { - if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil { - return err - } - } - return nil -} - -func (t *TestCluster) CreateField(index, field string, opt pilosa.FieldOptions) error { - for _, c := range t.Clusters { - idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) - if err != nil { - return err - } - if _, err := idx.CreateField(field, opt); err != nil { - return err - } - } - return nil -} -func (t *TestCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error { - // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / pilosa.SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Field(index, field) - if f == nil { - return fmt.Errorf("index/field does not exist: %s/%s", index, field) - } - _, err := f.SetBit(view, rowID, colID, x) - if err != nil { - return err - } - } - - return nil -} - -func (t *TestCluster) clusterByID(id string) *pilosa.Cluster { - for _, c := range t.Clusters { - if c.Node.ID == id { - return c - } - } - return nil -} - -// AddNode adds a node to the cluster and (potentially) starts a resize job. -func (t *TestCluster) AddNode(saveTopology bool) error { - id := len(t.Clusters) - - c, err := t.addCluster(id, saveTopology) - if err != nil { - return err - } - - // Send NodeJoin event to coordinator. - if id > 0 { - coord := t.Clusters[0] - ev := &pilosa.NodeEvent{ - Event: pilosa.NodeJoin, - Node: c.Node, - } - - if err := coord.ReceiveEvent(ev); err != nil { - return err - } - - // Wait for the AddNode job to finish. - if c.State() != pilosa.ClusterStateNormal { - t.resizeDone = make(chan struct{}) - t.mu.Lock() - t.resizing = true - t.mu.Unlock() - <-t.resizeDone - } - } - - return nil -} - -// WriteTopology writes the given topology to disk. -func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { - if buf, err := proto.Marshal(top.Encode()); err != nil { - return err - } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { - return err - } - return nil -} - -func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { - - id := fmt.Sprintf("node%d", i) - uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) - - node := &pilosa.Node{ - ID: id, - URI: uri, - } - - // add URI to common - //t.common.NodeIDs = append(t.common.NodeIDs, id) - //sort.Sort(t.common.NodeIDs) - - // add node to common - t.common.Nodes = append(t.common.Nodes, node) - - // create node-specific temp directory - path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) - if err != nil { - return nil, err - } - - // holder - h := pilosa.NewHolder() - h.Path = path - - // cluster - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - c.Path = path - c.Topology = pilosa.NewTopology() - c.Holder = h - c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) - c.Node = node - c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator - c.Broadcaster = t - - // add nodes - if saveTopology { - for _, n := range t.common.Nodes { - c.AddNode(n) - } - } - - // Add this node to the TestCluster. - t.Clusters = append(t.Clusters, c) - - return c, nil -} - -// NewTestCluster returns a new instance of test.Cluster. -func NewTestCluster(n int) *TestCluster { - - tc := &TestCluster{ - common: &commonClusterSettings{}, - } - - // add clusters - for i := 0; i < n; i++ { - _, err := tc.addCluster(i, true) - if err != nil { - panic(err) - } - } - return tc -} - -// SetState sets the state of the cluster on each node. -func (t *TestCluster) SetState(state string) { - for _, c := range t.Clusters { - c.SetState(state) - } -} - -// Open opens all clusters in the test cluster. -func (t *TestCluster) Open() error { - for _, c := range t.Clusters { - if err := c.Open(); err != nil { - return err - } - if err := c.Holder.Open(); err != nil { - return err - } - if err := c.SetNodeState(pilosa.NodeStateReady); err != nil { - return err - } - } - - // Start the listener on the coordinator. - if len(t.Clusters) == 0 { - return nil - } - t.Clusters[0].ListenForJoins() - - return nil -} - -// Close closes all clusters in the test cluster. -func (t *TestCluster) Close() error { - for _, c := range t.Clusters { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// TestCluster implements Broadcaster interface. - -// SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *TestCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: - // Apply the send message to all nodes (except the coordinator). - for _, c := range t.Clusters { - c.MergeClusterStatus(obj) - } - t.mu.RLock() - if obj.State == pilosa.ClusterStateNormal && t.resizing { - close(t.resizeDone) - } - t.mu.RUnlock() - } - - return nil -} - -// SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *TestCluster) SendAsync(pb proto.Message) error { - return nil -} - -// SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: - err := t.FollowResizeInstruction(obj) - if err != nil { - return err - } - case *internal.ResizeInstructionComplete: - coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) - } - return nil -} - -// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - - // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ - JobID: instr.JobID, - Node: instr.Node, - Error: "", - } - - // Stop processing on any error. - if err := func() error { - - // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) - instrNode := pilosa.DecodeNode(instr.Node) - destCluster := t.clusterByID(instrNode.ID) - - // Sync the schema received in the resize instruction. - if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { - return err - } - - for _, src := range instr.Sources { - srcNode := pilosa.DecodeNode(src.Node) - srcCluster := t.clusterByID(srcNode.ID) - - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) - if destFragment == nil { - // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Field(src.Index, src.Field) - v := f.View(src.View) - var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) - if err != nil { - return err - } - } - - buf := bytes.NewBuffer(nil) - - bw := bufio.NewWriter(buf) - br := bufio.NewReader(buf) - - // Get the fragment from source. - if _, err := srcFragment.WriteTo(bw); err != nil { - return err - } - - // Flush the bufio.buf to the io.Writer (buf). - bw.Flush() - - // Write data to destination. - if _, err := destFragment.ReadFrom(br); err != nil { - return err - } - } - - return nil - }(); err != nil { - complete.Error = err.Error() - } - - node := pilosa.DecodeNode(instr.Coordinator) - if err := t.SendTo(node, complete); err != nil { - return err - } - - return nil -} diff --git a/utils_test.go b/utils_internal_test.go similarity index 97% rename from utils_test.go rename to utils_internal_test.go index 6335d087f..4055ef566 100644 --- a/utils_test.go +++ b/utils_internal_test.go @@ -121,7 +121,7 @@ func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64, // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / SliceWidth - nodes := c0.SliceNodes(index, slice) + nodes := c0.sliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) @@ -236,7 +236,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) // add nodes if saveTopology { for _, n := range t.common.Nodes { - c.AddNode(n) + c.addNode(n) } } @@ -273,13 +273,13 @@ func (t *ClusterCluster) SetState(state string) { // Open opens all clusters in the test cluster. func (t *ClusterCluster) Open() error { for _, c := range t.Clusters { - if err := c.Open(); err != nil { + if err := c.open(); err != nil { return err } if err := c.Holder.Open(); err != nil { return err } - if err := c.SetNodeState(NodeStateReady); err != nil { + if err := c.setNodeState(NodeStateReady); err != nil { return err } } @@ -288,7 +288,7 @@ func (t *ClusterCluster) Open() error { if len(t.Clusters) == 0 { return nil } - t.Clusters[0].ListenForJoins() + t.Clusters[0].listenForJoins() return nil } @@ -296,7 +296,7 @@ func (t *ClusterCluster) Open() error { // Close closes all clusters in the test cluster. func (t *ClusterCluster) Close() error { for _, c := range t.Clusters { - err := c.Close() + err := c.close() if err != nil { return err } @@ -310,7 +310,7 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { case *internal.ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { - c.MergeClusterStatus(obj) + c.mergeClusterStatus(obj) } t.mu.RLock() if obj.State == ClusterStateNormal && t.resizing { @@ -337,7 +337,7 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { } case *internal.ResizeInstructionComplete: coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) + go coord.markResizeInstructionComplete(obj) } return nil } From 1211663019b3cf4fd4497094cecbaca3046fcae1 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 13 Jun 2018 15:47:00 -0500 Subject: [PATCH 02/10] add supporting functions for functional query contstruction --- test/querygenerator_test.go | 403 ++++++++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 test/querygenerator_test.go diff --git a/test/querygenerator_test.go b/test/querygenerator_test.go new file mode 100644 index 000000000..4702e6b91 --- /dev/null +++ b/test/querygenerator_test.go @@ -0,0 +1,403 @@ +package test + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/pilosa/pilosa/pql" +) + +type Args map[string]interface{} + +type Calls []*pql.Call + +func PQL(calls ...*pql.Call) *pql.Query { + return &pql.Query{Calls: calls} +} + +func Row(frame string, row int) *pql.Call { + return &pql.Call{ + Name: "Row", + Args: Args{ + "frame": frame, + "row": row, + }, + } +} + +func mutationArgs(args ...interface{}) Args { + rargs := make(Args) + for _, arg := range args { + switch v := arg.(type) { + case int: + rargs["column"] = v + case string: + if strings.Contains(v, "=") { + parts := strings.Split(v, "=") + rargs["frame"] = parts[0] + i, _ := strconv.ParseInt(parts[1], 10, 64) + rargs["value"] = i + } else { + rargs["timestamp"] = v + } + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs +} + +func Set(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Set", Args: mutationArgs(args...)} +} + +func Clear(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} +} + +func magic(args ...interface{}) (Args, Calls) { + var ( + rargs Args + calls Calls + ) + + for _, arg := range args { + switch v := arg.(type) { + case Args: + rargs = v + case []*pql.Call: + calls = append(calls, v...) + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs, calls +} +func Count(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Count", Args: kvargs, Children: children} +} + +func Union(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Union", Args: kvargs, Children: children} +} + +func Intersect(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} +} + +func Difference(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Difference", Args: kvargs, Children: children} +} +func Xor(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Xor", Args: kvargs, Children: children} +} + +func Between(frame string, min, max int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.BETWEEN, + "Value": []int{min, max}, + }, + } +} +func Lt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LT, + "Value": column, + }, + } +} +func Lte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LTE, + "Value": column, + }, + } +} +func Gt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GT, + "Value": column, + }, + } +} + +func Gte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GTE, + "Value": column, + }, + } +} +func CompareCall(a, b *pql.Call) bool { + if a.Name != b.Name { + return false + } + for k, i := range a.Args { + switch v := i.(type) { + case []int: + bside := b.Args[k] + for j := range v { + if v[j] != bside.([]int)[j] { + return false + } + + } + default: + if b.Args[k] != i { + return false + } + } + } + + if len(a.Children) == len(b.Children) { + for i := range a.Children { + if !CompareCall(a.Children[i], b.Children[i]) { + return false + } + } + } else { + return false + } + return true +} + +func Compare(a, b *pql.Query) bool { + for i := range a.Calls { + if !CompareCall(a.Calls[i], b.Calls[i]) { + return false + } + + } + return true +} + +func TestPQL_Generator(t *testing.T) { + t.Run("pql.Query generator", func(t *testing.T) { + for _, u := range []struct { + pql string + calc *pql.Query + exp *pql.Query + }{ + { + pql: "Union(Row(aaa=10),Row(bbb=9))", + calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Row(aaa=10),Row(bbb=9))", + calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Difference(Row(aaa=10),Row(bbb=9))", + calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Difference", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Range(bbb > 20)", + calc: PQL(Gt("bbb", 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.GT, + "Value": 20, + }, + }, + }, + }, + }, + { + pql: "Range(10 < bbb < 20)", + calc: PQL(Between("bbb", 10, 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.BETWEEN, + "Value": []int{10, 20}, + }, + }, + }, + }, + }, + { + pql: "Set(10, aaa=9)", + calc: PQL(Set(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: "Clear(10, aaa=10)", + calc: PQL(Clear(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Clear", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: `Set(10, aaa=10, "2017-03-02T03:00")`, + calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + "timestamp": "2017-03-02T03:00", + }, + }, + }, + }, + }, + { + pql: `Count(Row(aaa=10))`, + calc: PQL(Count(Row("aaa", 10))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Count", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))", + calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 12}, + }, + }, + }, + }, + }, + }, + } { + + if !Compare(u.calc, u.exp) { + t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql) + } + } + }) + +} From fe167ea78c026f32ecf71a2e99748ac199c9fa64 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 11 Jun 2018 13:49:00 -0500 Subject: [PATCH 03/10] un-export some top-level functions --- cluster.go | 4 +- executor.go | 2 +- field.go | 16 +++---- gossip/gossip.go | 21 ++++++++- index.go | 4 +- pilosa.go | 68 ++------------------------- pilosa_internal_test.go | 43 +++++++++++++++++ pilosa_test.go | 48 ------------------- server.go | 12 ++--- server/server_test.go | 16 ------- server_internal_test.go | 35 ++++++++++++++ server_test.go | 14 ++++++ stats.go | 6 +-- statsd/statsd.go | 38 ++++++++++++++- time.go | 28 +++++------ time_test.go => time_internal_test.go | 58 +++++++++++------------ view.go | 4 +- 17 files changed, 219 insertions(+), 198 deletions(-) create mode 100644 pilosa_internal_test.go create mode 100644 server_internal_test.go rename time_test.go => time_internal_test.go (68%) diff --git a/cluster.go b/cluster.go index 28feb80a3..8462d91e7 100644 --- a/cluster.go +++ b/cluster.go @@ -943,14 +943,14 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) + return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - return StringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) + return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) allNodesReady() bool { diff --git a/executor.go b/executor.go index 241b7c8ec..03c636215 100644 --- a/executor.go +++ b/executor.go @@ -752,7 +752,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} - for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { + for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { f := e.Holder.Fragment(index, field, view, slice) if f == nil { continue diff --git a/field.go b/field.go index 8c7683cd1..d03da2c22 100644 --- a/field.go +++ b/field.go @@ -82,7 +82,7 @@ func OptFieldFieldOptions(o FieldOptions) FieldOption { // NewField returns a new instance of field. func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { - err := ValidateName(name) + err := validateName(name) if err != nil { return nil, err } @@ -645,7 +645,7 @@ func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { // SetBit sets a bit on a view within the field. func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. - if !IsValidView(name) { + if !isValidView(name) { return false, ErrInvalidView } @@ -668,7 +668,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed } // If a timestamp is specified then set bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { return changed, errors.Wrapf(err, "creating view %s", subname) @@ -687,7 +687,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed // ClearBit clears a bit within the field. func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. - if !IsValidView(name) { + if !isValidView(name) { return false, ErrInvalidView } @@ -710,7 +710,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // If a timestamp is specified then clear bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { return changed, errors.Wrapf(err, "creating view %s", subname) @@ -899,7 +899,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro if timestamp == nil { standard = []string{ViewStandard} } else { - standard = ViewsByTime(ViewStandard, *timestamp, q) + standard = viewsByTime(ViewStandard, *timestamp, q) // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. standard = append(standard, ViewStandard) @@ -1233,8 +1233,8 @@ const ( CacheTypeNone = "none" ) -// IsValidCacheType returns true if v is a valid cache type. -func IsValidCacheType(v string) bool { +// isValidCacheType returns true if v is a valid cache type. +func isValidCacheType(v string) bool { switch v { case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: return true diff --git a/gossip/gossip.go b/gossip/gossip.go index 2da6e3e4c..dfcb2f759 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" "log" + "net" "strconv" "strings" "sync" @@ -213,7 +214,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = pilosa.HostToIP(host) + conf.AdvertiseAddr = hostToIP(host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult @@ -580,3 +581,21 @@ type Config struct { Nodes int `toml:"nodes"` ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` } + +// hostToIP converts host to an IP4 address based on net.LookupIP(). +func hostToIP(host string) string { + // if host is not an IP addr, check net.LookupIP() + if net.ParseIP(host) == nil { + hosts, err := net.LookupIP(host) + if err != nil { + return host + } + for _, h := range hosts { + // this restricts pilosa to IP4 + if h.To4() != nil { + return h.String() + } + } + } + return host +} diff --git a/index.go b/index.go index 80a5c2eb8..46ca48e47 100644 --- a/index.go +++ b/index.go @@ -53,7 +53,7 @@ type Index struct { // NewIndex returns a new instance of Index. func NewIndex(path, name string) (*Index, error) { - err := ValidateName(name) + err := validateName(name) if err != nil { return nil, errors.Wrap(err, "validating name") } @@ -295,7 +295,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, e func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { if name == "" { return nil, errors.New("field name required") - } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { + } else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) { return nil, ErrInvalidCacheType } diff --git a/pilosa.go b/pilosa.go index a87bab4fc..c18fb5c0c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,9 +16,7 @@ package pilosa import ( "errors" - "net" "regexp" - "strings" "github.com/pilosa/pilosa/internal" ) @@ -108,26 +106,16 @@ func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// ValidateName ensures that the name is a valid format. -func ValidateName(name string) error { +// validateName ensures that the name is a valid format. +func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { return ErrName } return nil } -// StringInSlice checks for substring a in the slice. -func StringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -// StringSlicesAreEqual determines if two string slices are equal. -func StringSlicesAreEqual(a, b []string) bool { +// stringSlicesAreEqual determines if two string slices are equal. +func stringSlicesAreEqual(a, b []string) bool { if a == nil && b == nil { return true @@ -150,54 +138,6 @@ func StringSlicesAreEqual(a, b []string) bool { return true } -// SliceDiff returns the difference between two uint64 slices. -func SliceDiff(a, b []uint64) []uint64 { - m := make(map[uint64]uint64) - - for _, y := range b { - m[y]++ - } - - var ret []uint64 - for _, x := range a { - if m[x] > 0 { - m[x]-- - continue - } - ret = append(ret, x) - } - - return ret -} - -// ContainsSubstring checks to see if substring a is contained in any string in the slice. -func ContainsSubstring(a string, list []string) bool { - for _, b := range list { - if strings.Contains(b, a) { - return true - } - } - return false -} - -// HostToIP converts host to an IP4 address based on net.LookupIP(). -func HostToIP(host string) string { - // if host is not an IP addr, check net.LookupIP() - if net.ParseIP(host) == nil { - hosts, err := net.LookupIP(host) - if err != nil { - return host - } - for _, h := range hosts { - // this restricts pilosa to IP4 - if h.To4() != nil { - return h.String() - } - } - } - return host -} - // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go new file mode 100644 index 000000000..139d5c2b7 --- /dev/null +++ b/pilosa_internal_test.go @@ -0,0 +1,43 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "testing" +) + +func TestValidateName(t *testing.T) { + names := []string{ + "a", "ab", "ab1", "b-c", "d_e", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, name := range names { + if validateName(name) != nil { + t.Fatalf("Should be valid index name: %s", name) + } + } +} + +func TestValidateNameInvalid(t *testing.T) { + names := []string{ + "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, name := range names { + if validateName(name) == nil { + t.Fatalf("Should be invalid index name: %s", name) + } + } +} diff --git a/pilosa_test.go b/pilosa_test.go index 41b0d7098..1c98686e5 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -22,54 +22,6 @@ import ( _ "github.com/pilosa/pilosa/test" ) -func TestValidateName(t *testing.T) { - names := []string{ - "a", "ab", "ab1", "b-c", "d_e", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, name := range names { - if pilosa.ValidateName(name) != nil { - t.Fatalf("Should be valid index name: %s", name) - } - } -} - -func TestValidateNameInvalid(t *testing.T) { - names := []string{ - "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, name := range names { - if pilosa.ValidateName(name) == nil { - t.Fatalf("Should be invalid index name: %s", name) - } - } -} - -func TestStringInSlice(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "localhost:10101" - if !pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s in %v", substr, list) - } - substr = "10101" - if pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s not in %v", substr, list) - } -} - -func TestContainsSubstring(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "10101" - if !pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s contained in %v", substr, list) - } - substr = "4000" - if pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s in not contained in %v", substr, list) - } -} - func TestAddressWithDefaults(t *testing.T) { tests := []struct { addr string diff --git a/server.go b/server.go index f330b6e6a..c94bcbca0 100644 --- a/server.go +++ b/server.go @@ -659,7 +659,7 @@ func (s *Server) monitorDiagnostics() { // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } @@ -716,7 +716,7 @@ func (s *Server) monitorRuntime() { // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() // Open File handles. if err == nil { s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) @@ -732,8 +732,8 @@ func (s *Server) monitorRuntime() { } } -// CountOpenFiles on operating systems that support lsof. -func CountOpenFiles() (int, error) { +// countOpenFiles on operating systems that support lsof. +func countOpenFiles() (int, error) { switch runtime.GOOS { case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks @@ -747,9 +747,9 @@ func CountOpenFiles() (int, error) { return len(lines), nil case "windows": // TODO: count open file handles on windows - return 0, errors.New("CountOpenFiles() on Windows is not supported") + return 0, errors.New("countOpenFiles() on Windows is not supported") default: - return 0, errors.New("CountOpenFiles() on this OS is not supported") + return 0, errors.New("countOpenFiles() on this OS is not supported") } } diff --git a/server/server_test.go b/server/server_test.go index a8cc2f8f4..971795156 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "math/rand" "reflect" - "runtime" "sort" "strings" "testing" @@ -263,21 +262,6 @@ func tempMkdir(t *testing.T) string { return dir } -// Ensure the file handle count is working -func TestCountOpenFiles(t *testing.T) { - // Windows is not supported yet - if runtime.GOOS == "windows" { - t.Skip("Skipping unsupported CountOpenFiles test on Windows.") - } - count, err := pilosa.CountOpenFiles() - if err != nil { - t.Errorf("CountOpenFiles failed: %s", err) - } - if count == 0 { - t.Error("CountOpenFiles returned invalid value 0.") - } -} - func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 cluster := test.MustRunMainWithCluster(t, clusterSize) diff --git a/server_internal_test.go b/server_internal_test.go new file mode 100644 index 000000000..e64f80a0f --- /dev/null +++ b/server_internal_test.go @@ -0,0 +1,35 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "runtime" + "testing" +) + +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + if runtime.GOOS == "windows" { + t.Skip("Skipping unsupported countOpenFiles test on Windows.") + } + count, err := countOpenFiles() + if err != nil { + t.Errorf("countOpenFiles failed: %s", err) + } + if count == 0 { + t.Error("countOpenFiles returned invalid value 0.") + } +} diff --git a/server_test.go b/server_test.go index 6cbbe191e..2f1003592 100644 --- a/server_test.go +++ b/server_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/stats.go b/stats.go index 313708fbc..23c3e0bc8 100644 --- a/stats.go +++ b/stats.go @@ -110,7 +110,7 @@ func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient { return &ExpvarStatsClient{ m: m, - tags: UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), } } @@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error { return nil } -// UnionStringSlice returns a sorted set of tags which combine a & b. -func UnionStringSlice(a, b []string) []string { +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { // Sort both sets first. sort.Strings(a) sort.Strings(b) diff --git a/statsd/statsd.go b/statsd/statsd.go index 0cec63718..81cfbb2b4 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,6 +15,7 @@ package statsd import ( + "sort" "time" "github.com/DataDog/datadog-go/statsd" @@ -72,7 +73,7 @@ func (c *StatsClient) Tags() []string { func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { return &StatsClient{ client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), logger: c.logger, } } @@ -124,3 +125,38 @@ func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { func (c *StatsClient) SetLogger(logger pilosa.Logger) { c.logger = logger } + +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { + // Sort both sets first. + sort.Strings(a) + sort.Strings(b) + + // Find size of largest slice. + n := len(a) + if len(b) > n { + n = len(b) + } + + // Exit if both sets are empty. + if n == 0 { + return nil + } + + // Iterate over both in order and merge. + other := make([]string, 0, n) + for len(a) > 0 || len(b) > 0 { + if len(a) == 0 { + other, b = append(other, b[0]), b[1:] + } else if len(b) == 0 { + other, a = append(other, a[0]), a[1:] + } else if a[0] < b[0] { + other, a = append(other, a[0]), a[1:] + } else if b[0] < a[0] { + other, b = append(other, b[0]), b[1:] + } else { + other, a, b = append(other, a[0]), a[1:], b[1:] + } + } + return other +} diff --git a/time.go b/time.go index 517292071..def889304 100644 --- a/time.go +++ b/time.go @@ -79,8 +79,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } -// ViewByTimeUnit returns the view name for time with a given quantum unit. -func ViewByTimeUnit(name string, t time.Time, unit rune) string { +// viewByTimeUnit returns the view name for time with a given quantum unit. +func viewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': return fmt.Sprintf("%s_%s", name, t.Format("2006")) @@ -95,11 +95,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string { } } -// ViewsByTime returns a list of views for a given timestamp. -func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { +// viewsByTime returns a list of views for a given timestamp. +func viewsByTime(name string, t time.Time, q TimeQuantum) []string { a := make([]string, 0, len(q)) for _, unit := range q { - view := ViewByTimeUnit(name, t, unit) + view := viewByTimeUnit(name, t, unit) if view == "" { continue } @@ -108,8 +108,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { return a } -// ViewsByTimeRange returns a list of views to traverse to query a time range. -func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +// viewsByTimeRange returns a list of views to traverse to query a time range. +func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -127,7 +127,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextDayGTE(t, end) { break } else if t.Hour() != 0 { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) continue } @@ -138,7 +138,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextMonthGTE(t, end) { break } else if t.Day() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) continue } @@ -148,7 +148,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextYearGTE(t, end) { break } else if t.Month() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) continue } @@ -164,16 +164,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string // Walk back down from largest units to smallest units. for t.Before(end) { if hasYear && nextYearGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'Y')) + results = append(results, viewByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) } else if hasMonth && nextMonthGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) } else if hasDay && nextDayGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) } else if hasHour { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) } else { break diff --git a/time_test.go b/time_internal_test.go similarity index 68% rename from time_test.go rename to time_internal_test.go index 3ab652455..2920685fc 100644 --- a/time_test.go +++ b/time_internal_test.go @@ -12,28 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "reflect" "testing" "time" - - "github.com/pilosa/pilosa" ) // Ensure string can be parsed into time quantum. func TestParseTimeQuantum(t *testing.T) { t.Run("OK", func(t *testing.T) { - if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + if q, err := ParseTimeQuantum("YMDH"); err != nil { t.Fatalf("unexpected error: %s", err) - } else if q != pilosa.TimeQuantum("YMDH") { + } else if q != TimeQuantum("YMDH") { t.Fatalf("unexpected quantum: %#v", q) } }) t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { - if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum { t.Fatalf("unexpected error: %s", err) } }) @@ -44,22 +42,22 @@ func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" { + if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" { t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" { + if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" { t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" { + if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" { t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { t.Fatalf("unexpected name: %s", s) } }) @@ -70,14 +68,14 @@ func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH")) + a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { t.Fatalf("unexpected names: %+v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D")) + a := viewsByTime("F", ts, mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000102"}) { t.Fatalf("unexpected names: %+v", a) } @@ -87,82 +85,82 @@ func TestViewsByTime(t *testing.T) { // Ensure sets of fields can be returned for a given time range. func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YM", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("M", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("DH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { t.Fatalf("unexpected fields: %#v", a) } }) t.Run("H", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { t.Fatalf("unexpected fields: %#v", a) } }) } -// DefaultTimeLayout is the time layout used by the tests. -const DefaultTimeLayout = "2006-01-02 15:04" +// defaultTimeLayout is the time layout used by the tests. +const defaultTimeLayout = "2006-01-02 15:04" -// MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) time.Time { - v, err := time.Parse(DefaultTimeLayout, value) +// mustParseTime parses value using DefaultTimeLayout. Panic on error. +func mustParseTime(value string) time.Time { + v, err := time.Parse(defaultTimeLayout, value) if err != nil { panic(err) } return v } -// MustParseTimeQuantum parses v into a time quantum. Panic on error. -func MustParseTimeQuantum(v string) pilosa.TimeQuantum { - q, err := pilosa.ParseTimeQuantum(v) +// mustParseTimeQuantum parses v into a time quantum. Panic on error. +func mustParseTimeQuantum(v string) TimeQuantum { + q, err := ParseTimeQuantum(v) if err != nil { panic(err) } diff --git a/view.go b/view.go index a7aaf0cb8..baef603d3 100644 --- a/view.go +++ b/view.go @@ -34,8 +34,8 @@ const ( viewBSIGroupPrefix = "bsig_" ) -// IsValidView returns true if name is valid. -func IsValidView(name string) bool { +// isValidView returns true if name is valid. +func isValidView(name string) bool { return name == ViewStandard } From fba865fc6cc5c76d21cd8af21705617f19b114ab Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Jun 2018 11:24:25 -0500 Subject: [PATCH 04/10] Remove more net/http references --- api.go | 2 -- cluster.go | 4 ---- fragment.go | 6 ++---- handler.go | 6 +++--- holder.go | 15 ++++++--------- holder_test.go | 10 ++++------ http/handler.go | 12 ++++++++++++ server.go | 26 ++------------------------ server/server.go | 2 -- 9 files changed, 29 insertions(+), 54 deletions(-) diff --git a/api.go b/api.go index 36af91b0f..ab4948588 100644 --- a/api.go +++ b/api.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "strconv" "strings" "time" @@ -45,7 +44,6 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster - RemoteClient *http.Client Logger Logger } diff --git a/cluster.go b/cluster.go index 28feb80a3..dccac02a1 100644 --- a/cluster.go +++ b/cluster.go @@ -21,7 +21,6 @@ import ( "hash/fnv" "io/ioutil" "math/rand" - "net/http" "os" "path/filepath" "sort" @@ -264,9 +263,6 @@ type Cluster struct { Logger Logger - // - RemoteClient *http.Client - InternalClient InternalClient } diff --git a/fragment.go b/fragment.go index 012c62503..ab50117fc 100644 --- a/fragment.go +++ b/fragment.go @@ -25,7 +25,6 @@ import ( "hash" "io" "io/ioutil" - "net/http" "os" "sort" "sync" @@ -1719,9 +1718,8 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *Cluster Closing <-chan struct{} } diff --git a/handler.go b/handler.go index f9c3435af..7c2b76a3f 100644 --- a/handler.go +++ b/handler.go @@ -2,7 +2,7 @@ package pilosa import ( "encoding/json" - "net/http" + "net" ) // QueryRequest represent a request to process a query. @@ -61,13 +61,13 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { } type Handler interface { - http.Handler + Serve(ln net.Listener, closing <-chan struct{}) GetAPI() *API } type NopHandler struct{} -func (n *NopHandler) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {} +func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {} func (n *NopHandler) GetAPI() *API { return nil diff --git a/holder.go b/holder.go index e7af65643..8dac3329b 100644 --- a/holder.go +++ b/holder.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io/ioutil" - "net/http" "os" "path" "path/filepath" @@ -563,9 +562,8 @@ func (h *Holder) logStartup() error { type HolderSyncer struct { Holder *Holder - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *Cluster // Stats Stats StatsClient @@ -755,11 +753,10 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ - Fragment: frag, - Node: s.Node, - Cluster: s.Cluster, - Closing: s.Closing, - RemoteClient: s.RemoteClient, + Fragment: frag, + Node: s.Node, + Cluster: s.Cluster, + Closing: s.Closing, } if err := fs.syncFragment(); err != nil { return errors.Wrap(err, "syncing fragment") diff --git a/holder_test.go b/holder_test.go index 31b50b900..72a10b815 100644 --- a/holder_test.go +++ b/holder_test.go @@ -362,7 +362,6 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { client := http.GetHTTPClient(nil) httpClient := http.NewInternalClientFromURI(uri, client) cluster.InternalClient = httpClient - cluster.RemoteClient = client // Create a local holder. hldr0 := test.MustOpenHolder() @@ -419,11 +418,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Set up syncer. syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - Node: cluster.Nodes[0], - Cluster: cluster, - RemoteClient: http.GetHTTPClient(nil), - Stats: pilosa.NopStatsClient, + Holder: hldr0.Holder, + Node: cluster.Nodes[0], + Cluster: cluster, + Stats: pilosa.NopStatsClient, } if err := syncer.SyncHolder(); err != nil { diff --git a/http/handler.go b/http/handler.go index df5108826..41a5faf55 100644 --- a/http/handler.go +++ b/http/handler.go @@ -117,6 +117,18 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) { return handler, nil } +func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) { + server := &http.Server{Handler: h} + go func() { + <-closing + server.Close() + }() + err := server.Serve(ln) + if err != nil && err.Error() != "http: Server closed" { + h.Logger.Printf("HTTP handler terminated with error: %s\n", err) + } +} + func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") diff --git a/server.go b/server.go index f330b6e6a..b1ce4bfaa 100644 --- a/server.go +++ b/server.go @@ -19,7 +19,6 @@ import ( "fmt" "log" "net" - "net/http" "os" "os/exec" "path/filepath" @@ -63,7 +62,6 @@ type Server struct { Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper - remoteClient *http.Client systemInfo SystemInfo gcNotifier GCNotifier NewAttrStore func(string) AttrStore @@ -162,15 +160,6 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } -// TODO: Remove RemoteClient -func OptServerRemoteClient(c *http.Client) ServerOption { - return func(s *Server) error { - s.remoteClient = c - s.Cluster.RemoteClient = c - return nil - } -} - func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { s.executor = NewExecutor(OptExecutorInternalQueryClient(c)) @@ -313,18 +302,8 @@ func (s *Server) Open() error { // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster - // Serve HTTP. - go func() { - server := &http.Server{Handler: s.handler} - go func() { - <-s.closing - server.Close() - }() - err := server.Serve(s.ln) - if err != nil && err.Error() != "http: Server closed" { - s.logger.Printf("HTTP handler terminated with error: %s\n", err) - } - }() + // Serve handler. + go s.handler.Serve(s.ln, s.closing) // Start the BroadcastReceiver. if err := s.BroadcastReceiver.Start(s); err != nil { @@ -424,7 +403,6 @@ func (s *Server) monitorAntiEntropy() { syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.RemoteClient = s.remoteClient syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. diff --git a/server/server.go b/server/server.go index 42b547d43..d034a4aa8 100644 --- a/server/server.go +++ b/server/server.go @@ -216,7 +216,6 @@ func (m *Command) SetupServer() error { } c := http.GetHTTPClient(TLSConfig) - api.RemoteClient = c m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), @@ -235,7 +234,6 @@ func (m *Command) SetupServer() error { pilosa.OptServerStatsClient(statsClient), pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), - pilosa.OptServerRemoteClient(c), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), ) From 7d91261968f8bc9a3165da581100e1ae4b0a1f23 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 11 Jun 2018 17:54:31 -0500 Subject: [PATCH 05/10] un-export some package level constants --- attr.go | 26 +++++------ broadcast.go | 90 +++++++++++++++++++-------------------- cache.go | 6 +-- executor.go | 16 +++---- field.go | 8 ++-- fragment.go | 30 ++++++------- fragment_internal_test.go | 2 +- holder.go | 6 +-- view.go | 2 +- view_internal_test.go | 2 +- 10 files changed, 93 insertions(+), 95 deletions(-) diff --git a/attr.go b/attr.go index 168ac0052..e73f303af 100644 --- a/attr.go +++ b/attr.go @@ -24,10 +24,10 @@ import ( // Attribute data type enum. const ( - AttrTypeString = 1 - AttrTypeInt = 2 - AttrTypeBool = 3 - AttrTypeFloat = 4 + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 ) // AttrStore represents an interface for handling row/column attributes. @@ -165,19 +165,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr { pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.Type = AttrTypeString + pb.Type = attrTypeString pb.StringValue = value case float64: - pb.Type = AttrTypeFloat + pb.Type = attrTypeFloat pb.FloatValue = value case uint64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = int64(value) case int64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = value case bool: - pb.Type = AttrTypeBool + pb.Type = attrTypeBool pb.BoolValue = value } return pb @@ -186,13 +186,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr { // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { switch attr.Type { - case AttrTypeString: + case attrTypeString: return attr.Key, attr.StringValue - case AttrTypeInt: + case attrTypeInt: return attr.Key, attr.IntValue - case AttrTypeBool: + case attrTypeBool: return attr.Key, attr.BoolValue - case AttrTypeFloat: + case attrTypeFloat: return attr.Key, attr.FloatValue default: return attr.Key, nil diff --git a/broadcast.go b/broadcast.go index 6b26b5d58..9b894fea2 100644 --- a/broadcast.go +++ b/broadcast.go @@ -120,21 +120,21 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error { // Broadcast message types. const ( - MessageTypeCreateSlice = iota - MessageTypeCreateIndex - MessageTypeDeleteIndex - MessageTypeCreateField - MessageTypeDeleteField - MessageTypeCreateView - MessageTypeDeleteView - MessageTypeClusterStatus - MessageTypeResizeInstruction - MessageTypeResizeInstructionComplete - MessageTypeSetCoordinator - MessageTypeUpdateCoordinator - MessageTypeNodeState - MessageTypeRecalculateCaches - MessageTypeNodeEvent + messageTypeCreateSlice = iota + messageTypeCreateIndex + messageTypeDeleteIndex + messageTypeCreateField + messageTypeDeleteField + messageTypeCreateView + messageTypeDeleteView + messageTypeClusterStatus + messageTypeResizeInstruction + messageTypeResizeInstructionComplete + messageTypeSetCoordinator + messageTypeUpdateCoordinator + messageTypeNodeState + messageTypeRecalculateCaches + messageTypeNodeEvent ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -142,35 +142,35 @@ func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { case *internal.CreateSliceMessage: - typ = MessageTypeCreateSlice + typ = messageTypeCreateSlice case *internal.CreateIndexMessage: - typ = MessageTypeCreateIndex + typ = messageTypeCreateIndex case *internal.DeleteIndexMessage: - typ = MessageTypeDeleteIndex + typ = messageTypeDeleteIndex case *internal.CreateFieldMessage: - typ = MessageTypeCreateField + typ = messageTypeCreateField case *internal.DeleteFieldMessage: - typ = MessageTypeDeleteField + typ = messageTypeDeleteField case *internal.CreateViewMessage: - typ = MessageTypeCreateView + typ = messageTypeCreateView case *internal.DeleteViewMessage: - typ = MessageTypeDeleteView + typ = messageTypeDeleteView case *internal.ClusterStatus: - typ = MessageTypeClusterStatus + typ = messageTypeClusterStatus case *internal.ResizeInstruction: - typ = MessageTypeResizeInstruction + typ = messageTypeResizeInstruction case *internal.ResizeInstructionComplete: - typ = MessageTypeResizeInstructionComplete + typ = messageTypeResizeInstructionComplete case *internal.SetCoordinatorMessage: - typ = MessageTypeSetCoordinator + typ = messageTypeSetCoordinator case *internal.UpdateCoordinatorMessage: - typ = MessageTypeUpdateCoordinator + typ = messageTypeUpdateCoordinator case *internal.NodeStateMessage: - typ = MessageTypeNodeState + typ = messageTypeNodeState case *internal.RecalculateCaches: - typ = MessageTypeRecalculateCaches + typ = messageTypeRecalculateCaches case *internal.NodeEventMessage: - typ = MessageTypeNodeEvent + typ = messageTypeNodeEvent default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -187,35 +187,35 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { var m proto.Message switch typ { - case MessageTypeCreateSlice: + case messageTypeCreateSlice: m = &internal.CreateSliceMessage{} - case MessageTypeCreateIndex: + case messageTypeCreateIndex: m = &internal.CreateIndexMessage{} - case MessageTypeDeleteIndex: + case messageTypeDeleteIndex: m = &internal.DeleteIndexMessage{} - case MessageTypeCreateField: + case messageTypeCreateField: m = &internal.CreateFieldMessage{} - case MessageTypeDeleteField: + case messageTypeDeleteField: m = &internal.DeleteFieldMessage{} - case MessageTypeCreateView: + case messageTypeCreateView: m = &internal.CreateViewMessage{} - case MessageTypeDeleteView: + case messageTypeDeleteView: m = &internal.DeleteViewMessage{} - case MessageTypeClusterStatus: + case messageTypeClusterStatus: m = &internal.ClusterStatus{} - case MessageTypeResizeInstruction: + case messageTypeResizeInstruction: m = &internal.ResizeInstruction{} - case MessageTypeResizeInstructionComplete: + case messageTypeResizeInstructionComplete: m = &internal.ResizeInstructionComplete{} - case MessageTypeSetCoordinator: + case messageTypeSetCoordinator: m = &internal.SetCoordinatorMessage{} - case MessageTypeUpdateCoordinator: + case messageTypeUpdateCoordinator: m = &internal.UpdateCoordinatorMessage{} - case MessageTypeNodeState: + case messageTypeNodeState: m = &internal.NodeStateMessage{} - case MessageTypeRecalculateCaches: + case messageTypeRecalculateCaches: m = &internal.RecalculateCaches{} - case MessageTypeNodeEvent: + case messageTypeNodeEvent: m = &internal.NodeEventMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) diff --git a/cache.go b/cache.go index ec9ade91b..e6a7dc283 100644 --- a/cache.go +++ b/cache.go @@ -27,8 +27,8 @@ import ( ) const ( - // ThresholdFactor is used to calculate the threshold for new items entering the cache - ThresholdFactor = 1.1 + // thresholdFactor is used to calculate the threshold for new items entering the cache + thresholdFactor = 1.1 ) // Cache represents a cache of counts. @@ -158,7 +158,7 @@ type RankCache struct { func NewRankCache(maxEntries uint32) *RankCache { return &RankCache{ maxEntries: maxEntries, - thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), + thresholdBuffer: int(thresholdFactor * float64(maxEntries)), entries: make(map[uint64]uint64), stats: NopStatsClient, } diff --git a/executor.go b/executor.go index 03c636215..fd07bd82a 100644 --- a/executor.go +++ b/executor.go @@ -25,13 +25,13 @@ import ( "github.com/pkg/errors" ) -// DefaultField is the field used if one is not specified. +// defaultField is the field used if one is not specified. const ( - DefaultField = "general" + defaultField = "general" - // MinThreshold is the lowest count to use in a Top-N operation when + // defaultMinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. - MinThreshold = 1 + defaultMinThreshold = 1 columnLabel = "col" rowLabel = "row" @@ -588,7 +588,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca // Set default field. if field == "" { - field = DefaultField + field = defaultField } f := e.Holder.Fragment(index, field, ViewStandard, slice) @@ -597,7 +597,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca } if minThreshold <= 0 { - minThreshold = MinThreshold + minThreshold = defaultMinThreshold } if tanimotoThreshold > 100 { @@ -646,7 +646,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. // Fetch field & row label based on argument. field, _ := c.Args["field"].(string) if field == "" { - field = DefaultField + field = defaultField } f := e.Holder.Field(index, field) if f == nil { @@ -700,7 +700,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Parse field, use default if unset. field, _ := c.Args["field"].(string) if field == "" { - field = DefaultField + field = defaultField } // Retrieve column label. diff --git a/field.go b/field.go index d03da2c22..2c7598037 100644 --- a/field.go +++ b/field.go @@ -33,10 +33,10 @@ import ( const ( DefaultFieldType = FieldTypeSet - DefaultCacheType = CacheTypeRanked + defaultCacheType = CacheTypeRanked // Default ranked field cache - DefaultCacheSize = 50000 + defaultCacheSize = 50000 ) // Field types. @@ -101,8 +101,8 @@ func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { options: FieldOptions{ Type: DefaultFieldType, - CacheType: DefaultCacheType, - CacheSize: DefaultCacheSize, + CacheType: defaultCacheType, + CacheSize: defaultCacheSize, }, Logger: NopLogger, diff --git a/fragment.go b/fragment.go index 012c62503..d06676170 100644 --- a/fragment.go +++ b/fragment.go @@ -48,22 +48,20 @@ const ( // SliceWidth is the number of column IDs in a slice. SliceWidth = 1048576 - // SnapshotExt is the file extension used for an in-process snapshot. - SnapshotExt = ".snapshotting" + // snapshotExt is the file extension used for an in-process snapshot. + snapshotExt = ".snapshotting" - // CopyExt is the file extension used for the temp file used while copying. - CopyExt = ".copying" + // copyExt is the file extension used for the temp file used while copying. + copyExt = ".copying" - // CacheExt is the file extension for persisted cache ids. - CacheExt = ".cache" + // cacheExt is the file extension for persisted cache ids. + cacheExt = ".cache" // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 -) -const ( - // DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - DefaultFragmentMaxOpN = 2000 + // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. + defaultFragmentMaxOpN = 2000 ) // Fragment represents the intersection of a field and slice in an index. @@ -120,18 +118,18 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment { field: field, view: view, slice: slice, - CacheType: DefaultCacheType, - CacheSize: DefaultCacheSize, + CacheType: defaultCacheType, + CacheSize: defaultCacheSize, Logger: NopLogger, - MaxOpN: DefaultFragmentMaxOpN, + MaxOpN: defaultFragmentMaxOpN, stats: NopStatsClient, } } // cachePath returns the path to the fragment's cache data. -func (f *Fragment) cachePath() string { return f.path + CacheExt } +func (f *Fragment) cachePath() string { return f.path + cacheExt } // Open opens the underlying storage. func (f *Fragment) Open() error { @@ -1432,7 +1430,7 @@ func (f *Fragment) snapshot() error { defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. - snapshotPath := f.path + SnapshotExt + snapshotPath := f.path + snapshotExt file, err := os.Create(snapshotPath) if err != nil { return fmt.Errorf("create snapshot file: %s", err) @@ -1636,7 +1634,7 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) { func (f *Fragment) readStorageFromArchive(r io.Reader) error { // Create a temporary file to copy into. - path := f.path + CopyExt + path := f.path + copyExt file, err := os.Create(path) if err != nil { return errors.Wrap(err, "creating directory") diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e7a77dafb..6c733ba4c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1245,7 +1245,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string) file.Close() if cacheType == "" { - cacheType = DefaultCacheType + cacheType = defaultCacheType } f := NewFragment(file.Name(), index, field, view, slice) diff --git a/holder.go b/holder.go index e7af65643..cdffd9c8d 100644 --- a/holder.go +++ b/holder.go @@ -34,8 +34,8 @@ import ( ) const ( - // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. - DefaultCacheFlushInterval = 1 * time.Minute + // defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. + defaultCacheFlushInterval = 1 * time.Minute // FileLimit is the maximum open file limit (ulimit -n) to automatically set. FileLimit = 262144 // (512^2) @@ -84,7 +84,7 @@ func NewHolder() *Holder { NewAttrStore: NewNopAttrStore, - CacheFlushInterval: DefaultCacheFlushInterval, + CacheFlushInterval: defaultCacheFlushInterval, Logger: NopLogger, } diff --git a/view.go b/view.go index baef603d3..b21c3f15a 100644 --- a/view.go +++ b/view.go @@ -73,7 +73,7 @@ func NewView(path, index, field, name string, cacheSize uint32) *View { name: name, cacheSize: cacheSize, - cacheType: DefaultCacheType, + cacheType: defaultCacheType, fragments: make(map[uint64]*Fragment), broadcaster: NopBroadcaster, diff --git a/view_internal_test.go b/view_internal_test.go index d0e8bfdd1..48df030db 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -26,7 +26,7 @@ func mustOpenView(index, field, name string) *View { panic(err) } - v := NewView(path, index, field, name, DefaultCacheSize) + v := NewView(path, index, field, name, defaultCacheSize) if err := v.open(); err != nil { panic(err) } From b36b3b16ffe398ca01f7fb98022edfc79a56b26e Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 14 Jun 2018 09:12:05 -0500 Subject: [PATCH 06/10] moved to pilosa core --- test/querygenerator_test.go => querygenerator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/querygenerator_test.go => querygenerator_test.go (99%) diff --git a/test/querygenerator_test.go b/querygenerator_test.go similarity index 99% rename from test/querygenerator_test.go rename to querygenerator_test.go index 4702e6b91..4641f5ed0 100644 --- a/test/querygenerator_test.go +++ b/querygenerator_test.go @@ -1,4 +1,4 @@ -package test +package pilosa_test import ( "fmt" From aa218905037e77c12383ca7b81fa78117f946a9f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 14 Jun 2018 10:02:36 -0500 Subject: [PATCH 07/10] moved to internal/test --- internal/test/querygenerator.go | 190 ++++++++++++++++++ .../test/querygenerator_test.go | 186 +---------------- 2 files changed, 191 insertions(+), 185 deletions(-) create mode 100644 internal/test/querygenerator.go rename querygenerator_test.go => internal/test/querygenerator_test.go (57%) diff --git a/internal/test/querygenerator.go b/internal/test/querygenerator.go new file mode 100644 index 000000000..32720cb68 --- /dev/null +++ b/internal/test/querygenerator.go @@ -0,0 +1,190 @@ +package test + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pilosa/pilosa/pql" +) + +type Args map[string]interface{} + +type Calls []*pql.Call + +func PQL(calls ...*pql.Call) *pql.Query { + return &pql.Query{Calls: calls} +} + +func Row(frame string, row int) *pql.Call { + return &pql.Call{ + Name: "Row", + Args: Args{ + "frame": frame, + "row": row, + }, + } +} + +func mutationArgs(args ...interface{}) Args { + rargs := make(Args) + for _, arg := range args { + switch v := arg.(type) { + case int: + rargs["column"] = v + case string: + if strings.Contains(v, "=") { + parts := strings.Split(v, "=") + rargs["frame"] = parts[0] + i, _ := strconv.ParseInt(parts[1], 10, 64) + rargs["value"] = i + } else { + rargs["timestamp"] = v + } + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs +} + +func Set(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Set", Args: mutationArgs(args...)} +} + +func Clear(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} +} + +func magic(args ...interface{}) (Args, Calls) { + var ( + rargs Args + calls Calls + ) + + for _, arg := range args { + switch v := arg.(type) { + case Args: + rargs = v + case []*pql.Call: + calls = append(calls, v...) + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs, calls +} +func Count(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Count", Args: kvargs, Children: children} +} + +func Union(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Union", Args: kvargs, Children: children} +} + +func Intersect(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} +} + +func Difference(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Difference", Args: kvargs, Children: children} +} +func Xor(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Xor", Args: kvargs, Children: children} +} + +func Between(frame string, min, max int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.BETWEEN, + "Value": []int{min, max}, + }, + } +} +func Lt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LT, + "Value": column, + }, + } +} +func Lte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LTE, + "Value": column, + }, + } +} +func Gt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GT, + "Value": column, + }, + } +} + +func Gte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GTE, + "Value": column, + }, + } +} +func CompareCall(a, b *pql.Call) bool { + if a.Name != b.Name { + return false + } + for k, i := range a.Args { + switch v := i.(type) { + case []int: + bside := b.Args[k] + for j := range v { + if v[j] != bside.([]int)[j] { + return false + } + + } + default: + if b.Args[k] != i { + return false + } + } + } + + if len(a.Children) == len(b.Children) { + for i := range a.Children { + if !CompareCall(a.Children[i], b.Children[i]) { + return false + } + } + } else { + return false + } + return true +} + +func Compare(a, b *pql.Query) bool { + for i := range a.Calls { + if !CompareCall(a.Calls[i], b.Calls[i]) { + return false + } + + } + return true +} diff --git a/querygenerator_test.go b/internal/test/querygenerator_test.go similarity index 57% rename from querygenerator_test.go rename to internal/test/querygenerator_test.go index 4641f5ed0..7af764602 100644 --- a/querygenerator_test.go +++ b/internal/test/querygenerator_test.go @@ -1,194 +1,10 @@ -package pilosa_test +package test import ( - "fmt" - "strconv" - "strings" "testing" - "github.com/pilosa/pilosa/pql" ) -type Args map[string]interface{} - -type Calls []*pql.Call - -func PQL(calls ...*pql.Call) *pql.Query { - return &pql.Query{Calls: calls} -} - -func Row(frame string, row int) *pql.Call { - return &pql.Call{ - Name: "Row", - Args: Args{ - "frame": frame, - "row": row, - }, - } -} - -func mutationArgs(args ...interface{}) Args { - rargs := make(Args) - for _, arg := range args { - switch v := arg.(type) { - case int: - rargs["column"] = v - case string: - if strings.Contains(v, "=") { - parts := strings.Split(v, "=") - rargs["frame"] = parts[0] - i, _ := strconv.ParseInt(parts[1], 10, 64) - rargs["value"] = i - } else { - rargs["timestamp"] = v - } - default: - fmt.Printf("wat %T!\n", v) - } - } - - return rargs -} - -func Set(args ...interface{}) *pql.Call { - return &pql.Call{Name: "Set", Args: mutationArgs(args...)} -} - -func Clear(args ...interface{}) *pql.Call { - return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} -} - -func magic(args ...interface{}) (Args, Calls) { - var ( - rargs Args - calls Calls - ) - - for _, arg := range args { - switch v := arg.(type) { - case Args: - rargs = v - case []*pql.Call: - calls = append(calls, v...) - default: - fmt.Printf("wat %T!\n", v) - } - } - - return rargs, calls -} -func Count(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Count", Args: kvargs, Children: children} -} - -func Union(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Union", Args: kvargs, Children: children} -} - -func Intersect(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} -} - -func Difference(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Difference", Args: kvargs, Children: children} -} -func Xor(args ...*pql.Call) *pql.Call { - kvargs, children := magic(args) - return &pql.Call{Name: "Xor", Args: kvargs, Children: children} -} - -func Between(frame string, min, max int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.BETWEEN, - "Value": []int{min, max}, - }, - } -} -func Lt(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.LT, - "Value": column, - }, - } -} -func Lte(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.LTE, - "Value": column, - }, - } -} -func Gt(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.GT, - "Value": column, - }, - } -} - -func Gte(frame string, column int) *pql.Call { - return &pql.Call{ - Name: "Range", - Args: Args{ - "Op": pql.GTE, - "Value": column, - }, - } -} -func CompareCall(a, b *pql.Call) bool { - if a.Name != b.Name { - return false - } - for k, i := range a.Args { - switch v := i.(type) { - case []int: - bside := b.Args[k] - for j := range v { - if v[j] != bside.([]int)[j] { - return false - } - - } - default: - if b.Args[k] != i { - return false - } - } - } - - if len(a.Children) == len(b.Children) { - for i := range a.Children { - if !CompareCall(a.Children[i], b.Children[i]) { - return false - } - } - } else { - return false - } - return true -} - -func Compare(a, b *pql.Query) bool { - for i := range a.Calls { - if !CompareCall(a.Calls[i], b.Calls[i]) { - return false - } - - } - return true -} func TestPQL_Generator(t *testing.T) { t.Run("pql.Query generator", func(t *testing.T) { From 060254e0d60b9a50aa757c75b94e1ecc43e6e433 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 15 Jun 2018 16:44:56 -0600 Subject: [PATCH 08/10] Key-to-ID Translation This commit adds id-to-key translation to make it easier for users to provide non-integer identifiers for rows & columns. --- Gopkg.lock | 14 +- api.go | 13 + ctl/server.go | 3 + executor.go | 112 +++++ executor_test.go | 126 ++++- field.go | 14 + holder.go | 8 +- http/handler.go | 52 ++ http/translator.go | 87 ++++ http/translator_test.go | 134 +++++ index.go | 22 +- inmem/translator.go | 215 ++++++++ inmem/translator_test.go | 132 +++++ internal/private.pb.go | 349 +++++++------ internal/private.proto | 2 + internal/public.pb.go | 35 +- mock/mock.go | 14 + mock/translator.go | 38 ++ pilosa.go | 3 + pql/ast.go | 33 ++ row.go | 10 + server.go | 31 +- server/config.go | 5 + server/server.go | 7 + statik/statik.go | 10 + test/executor.go | 2 + translate.go | 1006 ++++++++++++++++++++++++++++++++++++++ translate_test.go | 565 +++++++++++++++++++++ 28 files changed, 2829 insertions(+), 213 deletions(-) create mode 100644 http/translator.go create mode 100644 http/translator_test.go create mode 100644 inmem/translator.go create mode 100644 inmem/translator_test.go create mode 100644 mock/mock.go create mode 100644 mock/translator.go create mode 100644 statik/statik.go create mode 100644 translate.go create mode 100644 translate_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 0b4a8e9ea..b3fc9be2b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -70,6 +70,18 @@ packages = ["proto"] revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" +[[projects]] + name = "github.com/google/go-cmp" + packages = [ + "cmp", + "cmp/cmpopts", + "cmp/internal/diff", + "cmp/internal/function", + "cmp/internal/value" + ] + revision = "3af367b6b30c263d47e8895973edcca9a49cf029" + version = "v0.2.0" + [[projects]] name = "github.com/gorilla/context" packages = ["."] @@ -304,6 +316,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "325d0fb217ec7f1509186ff947e184f6c8e65941f06000eb110180e65816b1a4" + inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5" solver-name = "gps-cdcl" solver-version = 1 diff --git a/api.go b/api.go index ab4948588..7adff1d1b 100644 --- a/api.go +++ b/api.go @@ -44,6 +44,7 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster + TranslateStore TranslateStore Logger Logger } @@ -124,6 +125,18 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return resp, errors.Wrap(err, "reading column attrs") } + + // Translate column attributes, if necessary. + if api.TranslateStore != nil { + for _, col := range resp.ColumnAttrSets { + v, err := api.TranslateStore.TranslateColumnToString(req.Index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + resp.ColumnAttrSets = columnAttrSets } return resp, nil diff --git a/ctl/server.go b/ctl/server.go index a4c272a46..ad1ac5df5 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + // Translation + flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.") + // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") diff --git a/executor.go b/executor.go index fd07bd82a..a99753023 100644 --- a/executor.go +++ b/executor.go @@ -50,6 +50,9 @@ type Executor struct { // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int + + // Stores key/id translation data. + TranslateStore TranslateStore } // ExecutorOption is a functional option type for pilosa.Executor @@ -83,6 +86,11 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic return nil, ErrIndexRequired } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { return nil, ErrTooManyWrites @@ -93,6 +101,29 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic opt = &ExecOptions{} } + // Translate query keys to ids, if necessary. + for i := range q.Calls { + if err := e.translateCall(index, idx, q.Calls[i]); err != nil { + return nil, err + } + } + + results, err := e.execute(ctx, index, q, slices, opt) + if err != nil { + return nil, err + } + + // Translate response objects from ids to keys, if necessary. + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Don't bother calculating slices for query types that don't require it. needsSlices := needsSlices(q.Calls) @@ -1559,6 +1590,78 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu } } +func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { + // Translate column key. + if idx.Keys() { + if value := callArgString(c, "col"); value != "" { + ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) + if err != nil { + return err + } + c.Args["col"] = ids[0] + } + } + + // Translate row key, if field is specified & key exists. + if fieldName := callArgString(c, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + if value := callArgString(c, "row"); value != "" { + ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) + if err != nil { + return err + } + c.Args["row"] = ids[0] + } + } + } + + // Translate child calls. + for _, child := range c.Children { + if err := e.translateCall(index, idx, child); err != nil { + return err + } + } + + return nil +} + +func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { + switch result := result.(type) { + case *Row: + if idx.Keys() { + other := &Row{Attrs: result.Attrs} + for _, segment := range result.Segments() { + for _, col := range segment.Columns() { + key, err := e.TranslateStore.TranslateColumnToString(index, col) + if err != nil { + return nil, err + } + other.Keys = append(other.Keys, key) + } + } + return other, nil + } + + case []Pair: + if fieldName := callArgString(call, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + other := make([]Pair, len(result)) + for i := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID) + if err != nil { + return nil, err + } + other[i] = Pair{Key: key, Count: result[i].Count} + } + return other, nil + } + } + } + return result, nil +} + // errSliceUnavailable is a marker error if no nodes are available. var errSliceUnavailable = errors.New("slice unavailable") @@ -1670,3 +1773,12 @@ func (vc *ValCount) Larger(other ValCount) ValCount { Count: vc.Count, } } + +func callArgString(call *pql.Call, key string) string { + value, ok := call.Args[key] + if !ok { + return "" + } + s, _ := value.(string) + return s +} diff --git a/executor_test.go b/executor_test.go index 3d0d9854e..164133ecb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -22,6 +22,8 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -101,6 +103,35 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set bits. + if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ + `SetBit(field=f, row="bar", col="foo")`+"\n"+ + `SetBit(field=f, row="baz", col="foo")`+"\n"+ + `SetBit(field=f, row="bar", col="bat")`+"\n"+ + `SetBit(field=f, row="bbb", col="aaa")`+"\n", + ), nil, nil); err != nil { + t.Fatal(err) + } + + if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(results, []interface{}{ + &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, + }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { + t.Fatal(diff) + } + }) } // Ensure a difference query can be executed. @@ -383,36 +414,36 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + t.Run("ID", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row=0, col=0) - SetBit(field=f, row=0, col=1) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(field=f, row=10, col=0) - SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=other, row=0, col=0) - `), nil, nil); err != nil { - t.Fatal(err) - } + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=1) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(field=f, row=10, col=0) + SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=other, row=0, col=0) + `), nil, nil); err != nil { + t.Fatal(err) + } - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - t.Run("Standard", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ @@ -422,6 +453,46 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row="foo", col="a") + SetBit(field=f, row="foo", col="b") + SetBit(field=f, row="foo", col="c") + SetBit(field=f, row="foo", col="d") + SetBit(field=f, row="foo", col="e") + SetBit(field=f, row="bar", col="a") + SetBit(field=f, row="bar", col="b") + SetBit(field=f, row="baz", col="b") + SetBit(field=other, row="foo", col="a") + `), nil, nil); err != nil { + t.Fatal(err) + } + + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(result, []interface{}{ + []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + }); diff != "" { + t.Fatal(diff) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { @@ -1213,6 +1284,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { diff --git a/field.go b/field.go index 2c7598037..d9c88c924 100644 --- a/field.go +++ b/field.go @@ -282,6 +282,7 @@ func (f *Field) loadMeta() error { f.options.Min = pb.Min f.options.Max = pb.Max f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) + f.options.Keys = pb.Keys return nil } @@ -317,6 +318,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = 0 f.options.Max = 0 f.options.TimeQuantum = "" + f.options.Keys = opt.Keys case FieldTypeInt: f.options.Type = opt.Type f.options.CacheType = CacheTypeNone @@ -324,6 +326,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = opt.Min f.options.Max = opt.Max f.options.TimeQuantum = "" + f.options.Keys = opt.Keys // Create new bsiGroup. bsig := &bsiGroup{ @@ -345,6 +348,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = 0 f.options.Min = 0 f.options.Max = 0 + f.options.Keys = opt.Keys // Set the time quantum. if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { f.Close() @@ -378,6 +382,13 @@ func (f *Field) Close() error { return nil } +// Keys returns true if the field uses string keys. +func (f *Field) Keys() bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Keys +} + // bsiGroup returns a bsiGroup by name. func (f *Field) bsiGroup(name string) *bsiGroup { f.mu.RLock() @@ -1038,6 +1049,7 @@ type FieldOptions struct { Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Keys bool `json:"keys,omitempty"` } // Validate ensures that FieldOption values are valid. @@ -1075,6 +1087,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Min: o.Min, Max: o.Max, TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, } } @@ -1089,6 +1102,7 @@ func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { Min: options.Min, Max: options.Max, TimeQuantum: TimeQuantum(options.TimeQuantum), + Keys: options.Keys, } } diff --git a/holder.go b/holder.go index 7cb285d78..316ca1244 100644 --- a/holder.go +++ b/holder.go @@ -111,7 +111,8 @@ func (h *Holder) Open() error { } for _, fi := range fis { - if !fi.IsDir() { + // Skip files or hidden directories. + if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { continue } @@ -338,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { return nil, errors.Wrap(err, "creating") } + index.keys = opt.Keys + if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") + } else if err := index.saveMeta(); err != nil { + return nil, errors.Wrap(err, "meta") } // Update options. - h.indexes[index.Name()] = index return index, nil diff --git a/http/handler.go b/http/handler.go index 41a5faf55..bc09def3b 100644 --- a/http/handler.go +++ b/http/handler.go @@ -203,6 +203,8 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET") + router.Use(handler.queryArgValidator) return router } @@ -1184,6 +1186,56 @@ func (h *Handler) GetAPI() *pilosa.API { type defaultClusterMessageResponse struct{} +// TranslateStoreBufferSize is the buffer size used for streaming data. +const TranslateStoreBufferSize = 65536 + +func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64) + + rc, err := h.API.TranslateStore.Reader(r.Context(), offset) + if err == pilosa.ErrNotImplemented { + http.Error(w, err.Error(), http.StatusNotImplemented) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer rc.Close() + + // Ensure reader is closed when the client disconnects. + go func() { <-r.Context().Done(); rc.Close() }() + + // Flush header so client can continue. + w.WriteHeader(http.StatusOK) + if w, ok := w.(http.Flusher); ok { + w.Flush() + } + + // Copy from reader to client until store or client disconnect. + buf := make([]byte, TranslateStoreBufferSize) + for { + // Read from store. + n, err := rc.Read(buf) + if err == io.EOF { + return + } else if err != nil { + h.Logger.Printf("http: translate store read error: %s", err) + return + } else if n == 0 { + continue + } + + // Write to response & flush. + if _, err := w.Write(buf[:n]); err != nil { + h.Logger.Printf("http: translate store response write error: %s", err) + return + } else if w, ok := w.(http.Flusher); ok { + w.Flush() + } + } +} + type queryValidationSpec struct { required []string args map[string]struct{} diff --git a/http/translator.go b/http/translator.go new file mode 100644 index 000000000..3ca9b840d --- /dev/null +++ b/http/translator.go @@ -0,0 +1,87 @@ +package http + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + + "github.com/pilosa/pilosa" +) + +// Ensure implementation implements inteface. +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +// TranslateStore represents an implementation of TranslateStore that +// communicates over HTTP. This is used with the TranslateHandler. +type TranslateStore struct { + URL string +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore(rawurl string) *TranslateStore { + return &TranslateStore{URL: rawurl} +} + +// TranslateColumnsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateColumnToString is not currently implemented. +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// TranslateRowsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateRowToString is not currently implemented. +func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// Reader returns a reader that can stream data from a remote store. +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + // Generate remote URL. + u, err := url.Parse(s.URL) + if err != nil { + return nil, err + } + u.Path = "/translate/data" + u.RawQuery = (url.Values{ + "offset": {strconv.FormatInt(off, 10)}, + }).Encode() + + // Connect a stream to the remote server. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + + // Connect a stream to the remote server. + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err) + } + + // Handle error codes or return body as stream. + switch resp.StatusCode { + case http.StatusOK: + return resp.Body, nil + case http.StatusNotImplemented: + resp.Body.Close() + return nil, pilosa.ErrNotImplemented + default: + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body)) + } +} diff --git a/http/translator_test.go b/http/translator_test.go new file mode 100644 index 000000000..3378ddc58 --- /dev/null +++ b/http/translator_test.go @@ -0,0 +1,134 @@ +package http_test + +import ( + "context" + "io" + "io/ioutil" + "net/http/httptest" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/mock" + "github.com/pilosa/pilosa/test" +) + +func TestTranslateStore_Reader(t *testing.T) { + // Ensure client can connect and stream the translate store data. + t.Run("OK", func(t *testing.T) { + t.Run("ServerDisconnect", func(t *testing.T) { + var mrc mock.ReadCloser + var readN int + mrc.ReadFunc = func(p []byte) (int, error) { + readN++ + switch readN { + case 1: + copy(p, []byte("foo")) + return 3, nil + case 2: + copy(p, []byte("barbaz")) + return 6, nil + case 3: + return 0, io.EOF + default: + t.Fatal("unexpected read") + return 0, nil + } + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + // Setup handler on test server. + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + if off != 100 { + t.Fatalf("unexpected off: %d", off) + } + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + // Connect to server and stream all available data. + store := http.NewTranslateStore(s.URL) + rc, err := store.Reader(context.Background(), 100) + if err != nil { + t.Fatal(err) + } else if data, err := ioutil.ReadAll(rc); err != nil { + t.Fatal(err) + } else if string(data) != `foobarbaz` { + t.Fatalf("unexpected data: %q", data) + } else if err := rc.Close(); err != nil { + t.Fatal(err) + } + + if !closeInvoked { + t.Fatal("expected server close") + } + }) + + // Ensure server closes store reader if client disconnects. + t.Run("ClientDisconnect", func(t *testing.T) { + // Setup mock so that Read() hangs. + done := make(chan struct{}) + + var mrc mock.ReadCloser + mrc.ReadFunc = func(p []byte) (int, error) { + <-done + return 0, io.EOF + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + defer close(done) + + // Connect to server and begin streaming. + ctx, cancel := context.WithCancel(context.Background()) + store := http.NewTranslateStore(s.URL) + if _, err := store.Reader(ctx, 0); err != nil { + t.Fatal(err) + } + + // Cancel the context and check if server is closed. + cancel() + time.Sleep(100 * time.Millisecond) + if !closeInvoked { + t.Fatal("expected server-side close") + } + }) + }) + + // Ensure client is notified if the server doesn't support streaming replication. + t.Run("ErrNotImplemented", func(t *testing.T) { + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return nil, pilosa.ErrNotImplemented + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + _, err := http.NewTranslateStore(s.URL).Reader(context.Background(), 0) + if err != pilosa.ErrNotImplemented { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/index.go b/index.go index 46ca48e47..68c48c829 100644 --- a/index.go +++ b/index.go @@ -33,6 +33,7 @@ type Index struct { mu sync.RWMutex path string name string + keys bool // use string keys // Fields by name. fields map[string]*Field @@ -80,6 +81,9 @@ func (i *Index) Name() string { return i.name } // Path returns the path the index was initialized with. func (i *Index) Path() string { return i.path } +// Keys returns true if the index uses string keys. +func (i *Index) Keys() bool { return i.keys } + // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } @@ -164,18 +168,17 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. + i.keys = pb.Keys return nil } -// NOTE: Until we introduce new attributes to store in the index .meta file, -// we don't need to actually write the file. The code related to index.options -// and the index meta file are left in place for future use. -/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{}) + buf, err := proto.Marshal(&internal.IndexMeta{ + Keys: i.keys, + }) if err != nil { return errors.Wrap(err, "marshalling") } @@ -187,7 +190,6 @@ func (i *Index) saveMeta() error { return nil } -*/ // Close closes the index and its fields. func (i *Index) Close() error { @@ -407,11 +409,15 @@ func encodeIndex(d *Index) *internal.Index { } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct{} +type IndexOptions struct { + Keys bool `json:"keys"` +} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{} + return &internal.IndexMeta{ + Keys: i.Keys, + } } // hasTime returns true if a contains a non-nil time. diff --git a/inmem/translator.go b/inmem/translator.go new file mode 100644 index 000000000..b620b2d03 --- /dev/null +++ b/inmem/translator.go @@ -0,0 +1,215 @@ +package inmem + +import ( + "context" + "io" + "sync" + + "github.com/pilosa/pilosa" +) + +// Ensure type implements interface. +var _ pilosa.TranslateStore = &TranslateStore{} + +// TranslateStore is an in-memory storage engine for translating string-to-uint64 values. +type TranslateStore struct { + mu sync.RWMutex + + cols map[string]*translateIndex + rows map[frameKey]*translateIndex +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore() *TranslateStore { + return &TranslateStore{ + cols: make(map[string]*translateIndex), + rows: make(map[frameKey]*translateIndex), + } +} + +// Reader returns an error because it is not supported by the inmem store. +func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + return nil, pilosa.ErrReplicationNotSupported +} + +// TranslateColumnsToUint64 converts value to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.cols[index] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.rows[key] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +type frameKey struct { + index string + frame string +} + +type translateIndex struct { + seq uint64 + lookup map[string]uint64 + reverse map[uint64]string +} + +func newTranslateIndex() *translateIndex { + return &translateIndex{ + lookup: make(map[string]uint64), + reverse: make(map[uint64]string), + } +} diff --git a/inmem/translator_test.go b/inmem/translator_test.go new file mode 100644 index 000000000..d4d232566 --- /dev/null +++ b/inmem/translator_test.go @@ -0,0 +1,132 @@ +package inmem_test + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/pilosa/pilosa/inmem" +) + +func TestTranslateStore_TranslateColumn(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func TestTranslateStore_TranslateRow(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/1000) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/private.pb.go b/internal/private.pb.go index 1e2b34f38..c3dadb455 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -61,6 +60,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -68,6 +68,13 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *IndexMeta) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type FieldOptions struct { Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` @@ -75,6 +82,7 @@ type FieldOptions struct { Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *FieldOptions) Reset() { *m = FieldOptions{} } @@ -124,6 +132,13 @@ func (m *FieldOptions) GetTimeQuantum() string { return "" } +func (m *FieldOptions) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } @@ -966,6 +981,16 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Keys { + dAtA[i] = 0x18 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -1017,6 +1042,16 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } + if m.Keys { + dAtA[i] = 0x58 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -2105,24 +2140,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2135,6 +2152,9 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l + if m.Keys { + n += 2 + } return n } @@ -2162,6 +2182,9 @@ func (m *FieldOptions) Size() (n int) { if m.Max != 0 { n += 1 + sovPrivate(uint64(m.Max)) } + if m.Keys { + n += 2 + } return n } @@ -2675,6 +2698,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2869,6 +2912,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3485,51 +3548,14 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3539,31 +3565,69 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.Standard[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.Standard[mapkey] = mapvalue } + m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -6617,69 +6681,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1011 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35, - 0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a, - 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x1b, 0x60, 0x10, 0x09, 0xe0, 0x49, 0x7a, - 0xeb, 0xc1, 0xdd, 0xb5, 0xda, 0x51, 0x66, 0xc7, 0xc3, 0x8c, 0x27, 0xc9, 0xf6, 0xc0, 0x15, 0x2e, - 0xdc, 0x11, 0x67, 0xfe, 0x18, 0x8e, 0xfc, 0x09, 0x28, 0xfc, 0x23, 0xc8, 0x9f, 0x3d, 0x8f, 0x64, - 0x37, 0x4d, 0x15, 0x7a, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b, - 0xb1, 0x95, 0xe5, 0x52, 0x49, 0xd2, 0x8b, 0x53, 0x25, 0xf2, 0x94, 0x27, 0x74, 0x19, 0xfa, 0x61, - 0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8a, 0x45, 0x32, 0xf9, 0x3e, - 0x53, 0xb1, 0x4c, 0x0b, 0xf2, 0x01, 0xf4, 0x77, 0xf9, 0xf8, 0x85, 0x38, 0x9c, 0x65, 0x22, 0xf0, - 0x86, 0xce, 0x66, 0x9f, 0x35, 0x8c, 0x5a, 0x1a, 0xc5, 0x2f, 0x45, 0xe0, 0x0f, 0x9d, 0xcd, 0x01, - 0x6b, 0x18, 0x64, 0x08, 0xcb, 0x87, 0xf1, 0x54, 0xfc, 0x58, 0xf2, 0x54, 0x95, 0xd3, 0xa0, 0x83, - 0xd6, 0x6d, 0x16, 0x21, 0xe0, 0xa3, 0xe3, 0x1e, 0x8a, 0xf0, 0x4c, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, - 0xa0, 0x3f, 0x74, 0x36, 0x3d, 0xa6, 0x8f, 0xc8, 0xe1, 0x67, 0x01, 0x58, 0x0e, 0x3f, 0xa3, 0x14, - 0x56, 0xc2, 0x69, 0x26, 0x73, 0xc5, 0x44, 0x91, 0xc9, 0xb4, 0x40, 0xab, 0xbd, 0x3c, 0x0f, 0x1c, - 0x74, 0xa4, 0x8f, 0xf4, 0x67, 0x58, 0xdd, 0x49, 0xe4, 0xf8, 0x78, 0xc4, 0x15, 0x67, 0xe2, 0xa7, - 0x52, 0x14, 0x8a, 0xbc, 0x03, 0x1d, 0xac, 0xd5, 0xea, 0x19, 0x42, 0x73, 0xb1, 0xe6, 0xc0, 0x35, - 0x5c, 0x24, 0x34, 0x17, 0xed, 0xb1, 0x6a, 0x9f, 0x19, 0x42, 0x73, 0xa3, 0x24, 0x1e, 0x9b, 0x6a, - 0x7d, 0x66, 0x08, 0x5d, 0xc7, 0x93, 0x58, 0x9c, 0xda, 0x12, 0xf1, 0x4c, 0x43, 0x58, 0x6b, 0xc5, - 0xb7, 0x69, 0xae, 0x43, 0x97, 0xc9, 0xd3, 0x70, 0x54, 0x04, 0xce, 0xd0, 0xdb, 0xf4, 0x99, 0xa5, - 0xb0, 0x91, 0x32, 0x29, 0xa7, 0xa9, 0x16, 0xb9, 0x28, 0x6a, 0x18, 0xf4, 0x0e, 0x74, 0xb0, 0xab, - 0xba, 0xca, 0xc6, 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xf4, 0xf7, 0xf9, 0x19, 0xa6, 0x51, 0x90, 0xc7, - 0xd0, 0x8b, 0x14, 0x4f, 0x27, 0x3c, 0x9f, 0xa0, 0xd2, 0xf2, 0xc3, 0x8f, 0xb6, 0xaa, 0x41, 0x6f, - 0xd5, 0x6a, 0x5b, 0x95, 0xce, 0x5e, 0xaa, 0xf2, 0x19, 0xab, 0x4d, 0xde, 0xff, 0x02, 0x06, 0x17, - 0x44, 0x3a, 0xde, 0xb1, 0x98, 0x55, 0x5d, 0x3d, 0x16, 0x33, 0x5d, 0xff, 0x09, 0x4f, 0x4a, 0x81, - 0xbd, 0xf2, 0x99, 0x21, 0x3e, 0x77, 0x3f, 0x75, 0xe8, 0x36, 0x90, 0xdd, 0x5c, 0x70, 0x25, 0x30, - 0xc8, 0xbe, 0x28, 0x0a, 0xfe, 0x5c, 0x5c, 0xdd, 0x71, 0xd3, 0x45, 0xb7, 0xd5, 0x45, 0x7a, 0x1f, - 0xc8, 0x48, 0x24, 0x42, 0x09, 0x8b, 0xc7, 0x57, 0x78, 0xa0, 0x51, 0x15, 0xed, 0x7a, 0x5d, 0x72, - 0x0f, 0x7c, 0x0d, 0x6e, 0x0c, 0xb6, 0xfc, 0xf0, 0x76, 0xd3, 0x91, 0x1a, 0xf7, 0x0c, 0x15, 0x68, - 0x52, 0x39, 0x45, 0x04, 0x5c, 0x5b, 0xc2, 0x02, 0xd0, 0xdc, 0xb7, 0xa1, 0x3c, 0x0c, 0xb5, 0xde, - 0x84, 0x6a, 0x2f, 0x95, 0x8d, 0xb6, 0x5d, 0x95, 0x7b, 0xd3, 0x68, 0xf4, 0xa9, 0xe5, 0x6a, 0xfc, - 0x1d, 0xf0, 0xa9, 0xb0, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, 0x2d, - 0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc4, 0x94, 0x93, 0x8f, - 0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, 0xfc, - 0x17, 0xe6, 0x74, 0x0f, 0xba, 0x18, 0xbd, 0x08, 0xfc, 0xcb, 0x6e, 0x90, 0xcf, 0xac, 0x98, 0xee, - 0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, 0x37, 0xb2, 0x50, - 0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0x1f, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, 0xfa, 0x14, 0xfc, - 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, 0x88, 0xee, 0x6d, - 0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, 0x57, 0xca, 0x7c, - 0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, 0xed, 0x3e, 0x52, - 0x5c, 0xd5, 0x80, 0x5f, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, 0x7a, 0xd5, 0x04, - 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, 0x0e, 0x06, 0xcc, - 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, - 0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, 0xf9, 0x05, 0xa9, - 0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, 0xdf, 0x8e, 0x89, - 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, 0x18, 0x75, 0x7f, - 0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, 0x61, 0x84, 0xf4, - 0x09, 0xf4, 0x76, 0xa2, 0xf0, 0xeb, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, 0x8d, 0x3b, 0xff, - 0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, 0xaf, 0xc4, 0x4d, - 0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xdf, 0xa4, 0xd3, 0x3f, - 0x5c, 0x58, 0x63, 0xa2, 0x88, 0x5f, 0x8a, 0x30, 0x2d, 0x54, 0x5e, 0x8e, 0xf5, 0x82, 0x6b, 0xfb, - 0x6f, 0xe5, 0x33, 0xdb, 0x6d, 0x8f, 0x19, 0xe2, 0x75, 0xc0, 0x44, 0x1e, 0xc0, 0xf2, 0xe5, 0x05, - 0x98, 0x57, 0x6d, 0xab, 0x90, 0x07, 0xb0, 0x14, 0xc9, 0x32, 0xd7, 0x48, 0x32, 0xeb, 0xdd, 0xba, - 0x74, 0x4c, 0x66, 0x46, 0xcc, 0x2a, 0xb5, 0x16, 0x94, 0x3a, 0xaf, 0x86, 0x12, 0x79, 0x7c, 0x09, - 0x4a, 0x41, 0x17, 0x0d, 0xde, 0x6b, 0x0c, 0x2e, 0x88, 0xd9, 0x45, 0x6d, 0xfa, 0xab, 0x03, 0xb7, - 0xda, 0x29, 0xbc, 0xd6, 0x6e, 0xd4, 0x13, 0x71, 0x17, 0x4e, 0xc4, 0x5b, 0x34, 0x11, 0xbf, 0x99, - 0x48, 0xf3, 0xca, 0x75, 0xda, 0xaf, 0xdc, 0x31, 0xdc, 0x99, 0x1b, 0xd3, 0xae, 0x9c, 0x66, 0x1a, - 0x0f, 0xff, 0x63, 0x5c, 0xfa, 0xd6, 0xc8, 0x73, 0x3b, 0xa8, 0x3e, 0x33, 0x04, 0xfd, 0x0c, 0xde, - 0x8d, 0x84, 0x6a, 0x0d, 0xa9, 0x42, 0xdb, 0x10, 0xbc, 0x03, 0x71, 0x7a, 0x45, 0xf9, 0x5a, 0x44, - 0xbf, 0x84, 0xe0, 0x28, 0x9b, 0x70, 0x25, 0x6e, 0x64, 0xbd, 0x03, 0xbd, 0x43, 0x99, 0xc9, 0x44, - 0x3e, 0x9f, 0x5d, 0xb3, 0xf5, 0x01, 0x2c, 0x99, 0x2b, 0xd2, 0x7c, 0x7c, 0xfa, 0xac, 0x22, 0xe9, - 0x6d, 0x0d, 0xe8, 0x31, 0x4f, 0xc6, 0x65, 0xa2, 0xd3, 0xd0, 0x3f, 0xa0, 0x62, 0x67, 0xf5, 0xaf, - 0xf3, 0x0d, 0xe7, 0xef, 0xf3, 0x0d, 0xe7, 0x9f, 0xf3, 0x0d, 0xe7, 0xf7, 0x7f, 0x37, 0xde, 0x7a, - 0xd6, 0xc5, 0x1f, 0xed, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x97, 0xf0, 0x12, 0xfd, 0xe2, - 0x0a, 0x00, 0x00, + // 1028 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, + 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, + 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, + 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, + 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, + 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, + 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, + 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, + 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, + 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, + 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, + 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, + 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, + 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, + 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, + 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, + 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, + 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, + 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, + 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, + 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, + 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, + 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, + 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, + 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, + 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, + 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, + 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, + 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, + 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, + 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, + 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, + 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, + 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, + 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, + 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, + 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, + 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, + 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, + 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, + 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, + 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, + 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, + 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, + 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, + 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, + 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, + 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, + 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, + 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, + 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, + 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, + 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, + 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, + 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, + 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, + 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, + 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, + 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, + 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, + 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, + 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, + 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, + 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, + 0x0a, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 9cab31828..23bb4886c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package internal; message IndexMeta { + bool Keys = 3; } message FieldOptions { @@ -12,6 +13,7 @@ message FieldOptions { int64 Min = 9; int64 Max = 10; string TimeQuantum = 5; + bool Keys = 11; } message ImportResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 76266d2db..3cb6fa270 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import encoding_binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -799,7 +800,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 } return i, nil } @@ -1235,24 +1237,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2333,15 +2317,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex diff --git a/mock/mock.go b/mock/mock.go new file mode 100644 index 000000000..46469c2f9 --- /dev/null +++ b/mock/mock.go @@ -0,0 +1,14 @@ +package mock + +type ReadCloser struct { + ReadFunc func(p []byte) (int, error) + CloseFunc func() error +} + +func (rc *ReadCloser) Read(p []byte) (int, error) { + return rc.ReadFunc(p) +} + +func (rc *ReadCloser) Close() error { + return rc.CloseFunc() +} diff --git a/mock/translator.go b/mock/translator.go new file mode 100644 index 000000000..3f815b89f --- /dev/null +++ b/mock/translator.go @@ -0,0 +1,38 @@ +package mock + +import ( + "context" + "io" + + "github.com/pilosa/pilosa" +) + +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +type TranslateStore struct { + TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error) + TranslateColumnToStringFunc func(index string, values uint64) (string, error) + TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error) + TranslateRowToStringFunc func(index, frame string, values uint64) (string, error) + ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error) +} + +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return s.TranslateColumnsToUint64Func(index, values) +} + +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return s.TranslateColumnToStringFunc(index, values) +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return s.TranslateRowsToUint64Func(index, frame, values) +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + return s.TranslateRowToStringFunc(index, frame, value) +} + +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + return s.ReaderFunc(ctx, off) +} diff --git a/pilosa.go b/pilosa.go index c18fb5c0c..bff167a7d 100644 --- a/pilosa.go +++ b/pilosa.go @@ -61,6 +61,8 @@ var ( ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") + + ErrNotImplemented = errors.New("not implemented") ) // ApiMethodNotAllowedError wraps an error value indicating that a particular @@ -83,6 +85,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // Can have a set of attributes attached to it. type ColumnAttrSet struct { ID uint64 `json:"id"` + Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..2d3f59e58 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -40,6 +40,23 @@ func (q *Query) WriteCallN() int { return n } +// HasKeys returns true if any call in the query uses keys and requires translation to ids. +func (q *Query) HasKeys() bool { + for _, call := range q.Calls { + if call.Args["col"] != nil { + if _, ok := call.Args["col"].(string); ok { + return true + } + } + if call.Args["row"] != nil { + if _, ok := call.Args["row"].(string); ok { + return true + } + } + } + return false +} + // String returns a string representation of the query. func (q *Query) String() string { a := make([]string, len(q.Calls)) @@ -100,6 +117,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +// StringArg is for reading the value at key from call.Args as a string. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. An error is returned if the value is not a string. +func (c *Call) StringArg(key string) (string, bool, error) { + val, ok := c.Args[key] + if !ok { + return "", false, nil + } + switch tval := val.(type) { + case string: + return tval, true, nil + default: + return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval) + } +} + // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/row.go b/row.go index b7643a975..a59bd73e0 100644 --- a/row.go +++ b/row.go @@ -27,6 +27,9 @@ import ( type Row struct { segments []RowSegment + // String keys translated to/from segment columns. + Keys []string + // Attributes associated with the row. Attrs map[string]interface{} } @@ -166,6 +169,11 @@ func (r *Row) ClearBit(i uint64) (changed bool) { return s.ClearBit(i) } +// Segments returns a list of all segments in the row. +func (r *Row) Segments() []RowSegment { + return r.segments +} + // segment returns a segment for a given slice. // Returns nil if segment does not exist. func (r *Row) segment(slice uint64) *RowSegment { @@ -241,8 +249,10 @@ func (r *Row) MarshalJSON() ([]byte, error) { var o struct { Attrs map[string]interface{} `json:"attrs"` Columns []uint64 `json:"columns"` + Keys []string `json:"keys,omitempty"` } o.Columns = r.Columns() + o.Keys = r.Keys o.Attrs = r.Attrs if o.Attrs == nil { diff --git a/server.go b/server.go index cbc79a86c..4a453cef5 100644 --- a/server.go +++ b/server.go @@ -52,10 +52,11 @@ type Server struct { closing chan struct{} // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - executor *Executor + Holder *Holder + Cluster *Cluster + TranslateFile *TranslateFile + diagnostics *DiagnosticsCollector + executor *Executor // External handler Handler @@ -75,6 +76,8 @@ type Server struct { diagnosticInterval time.Duration maxWritesPerRequest int + primaryTranslateStore TranslateStore + defaultClient InternalClient dataDir string } @@ -169,6 +172,13 @@ func OptServerInternalClient(c InternalClient) ServerOption { } } +func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { + return func(s *Server) error { + s.primaryTranslateStore = store + return nil + } +} + func OptServerStatsClient(sc StatsClient) ServerOption { return func(s *Server) error { s.Holder.Stats = sc @@ -241,6 +251,14 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder + // Initialize translation database. + s.TranslateFile = NewTranslateFile() + s.TranslateFile.Path = filepath.Join(path, "keys") + s.TranslateFile.PrimaryTranslateStore = s.primaryTranslateStore + if err := s.TranslateFile.Open(); err != nil { + return nil, err + } + // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) @@ -259,8 +277,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Holder = s.Holder s.executor.Node = node s.executor.Cluster = s.Cluster + s.executor.TranslateStore = s.TranslateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest s.handler.GetAPI().Executor = s.executor + s.handler.GetAPI().TranslateStore = s.TranslateFile return s, nil } @@ -354,6 +374,9 @@ func (s *Server) Close() error { if s.Holder != nil { s.Holder.Close() } + if s.TranslateFile != nil { + s.TranslateFile.Close() + } return nil } diff --git a/server/config.go b/server/config.go index 6c7db569a..1b74b177b 100644 --- a/server/config.go +++ b/server/config.go @@ -78,6 +78,11 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` + // Translation config supports translation store replication. + Translation struct { + PrimaryURL string `toml:"primary-url"` + } + AntiEntropy struct { Interval toml.Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/server/server.go b/server/server.go index d034a4aa8..0c4184d60 100644 --- a/server/server.go +++ b/server/server.go @@ -217,6 +217,12 @@ func (m *Command) SetupServer() error { c := http.GetHTTPClient(TLSConfig) + // Setup connection to primary store if this is a replica. + var primaryTranslateStore pilosa.TranslateStore + if m.Config.Translation.PrimaryURL != "" { + primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL) + } + m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), @@ -235,6 +241,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), + pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), ) return errors.Wrap(err, "new server") diff --git a/statik/statik.go b/statik/statik.go new file mode 100644 index 000000000..54ef98b8f --- /dev/null +++ b/statik/statik.go @@ -0,0 +1,10 @@ +package statik + +import ( + "github.com/rakyll/statik/fs" +) + +func init() { + data := "PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00assets/chevron-down.png\x89PNG\x0d\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x00\xc8\x00\x00\x00\xc8\x08\x06\x00\x00\x00\xadX\xae\x9e\x00\x00\x0e\x0eIDATx\xda\xed\xddy\x90\x14\xd5\x1d\x07\xf0\xc7\xce\xd5=}\xce\xec\x1c;\xb33\xb3;3{\xc0\x9e\xec\x01\xcb.\xbb\xec1\xbb\xa8A\xa3$h\xc5#\x1e \xb95\xa5Dc*\x95C<\"\xa5\xd1T\x02\xc6Jb\x89\xe6\x1f\xe3\x91hb\"\xa8\x89g\x8c\xa6\"\xc6\x8ax!`\x8c ^ \x88r\xaf\xc9\xef\x07\xa31D\x84\x85\xdd\x9973\xdfOUWQ\xcbL\xf7{\xef\xf7\xeb\xee7\xef\xf5!\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe4\x8d\x93\x16\xc3\x8cu\xfby\xa1\x7f\xeb\xb48\xd0,p\x10\x8e\xfd\xf2\xc6\xc8\xe6RQ\xd1\\V\xbc\xd3\xa1X\x0b\xed\x9a\xcc\xb7yq(\xf6\x99n_\xba\x89\xfe\xcf\x83\x1c\x80\x03\xf0p\x8eP\xae\xcc\x7f?o\xca<\xd6\xe78\x978\xa7\x8a\xa5\x92\x15j\xb0a\x9eQ9m\xb9Z^\xb7\xdeW;\xf2o^\xd4@\xfdsf|\xc62oE\xdb1\xf4\x19\x1fr\x01\xf6\xe3\xe3\xdc\xa0\x1c\xb9\x96r\xe5\x85\x0f\xf2\xa6\xbc\xee\xa5\xbd\xb9D9\xc5\xb9U\xe8\x95\xacR}\xa9\xaf\x1bU=\xcf\xbc_\xc1\xfd\x17\xb3\xba\xef/\xde\xc0\xe4\x85\xf4\xd90r\x02\xb2\xc2\x9c\x13\x9c\x1b\x07\xca\x1b\xce)\xd5\x97\xbe\x90s\xac\x10+\xa8*v\xb2\xca\xa1\x96/;P\x05\xf7_\x9cZ\xf8\"::T\xa2\xcbU\xda]*\xce\x01\xce\x85C\xcd\x1b\xca\xb1\xa5\x8a/\x95\xa0\xef*\x85RIEx\xb4\xd9\xde\xd0\x94{\x0f\xb5\x92\xef/\xde\x8a\xe6[\x85jt\xd3:\xdc\xc8\x95\x92\xe3\xa6\xd8\xf7p\x0e\x8c9o(\xd7(\xe7F\ne'9\xdf\xa8\x9a\xf9\x8a]32:\xd6\x8a\xf2w\xcc\xea\xde\xf5\xb4\x8e3\x91/%\xe7,\x8e\xfd\xe1\xe6\x0d\xe7\x1c\xad\xe3<\x99+\x18sx\xfd\xd7\xd9\xe9\xc1\xcdv\xcd\xf0\x9e\xb1V\xf2\xbf\x95\x1d\xdeC\xeb\xd8\xe4\xd4\x02W\xd2:#\xc8\x9b\xa2\x17\xa1X_\xc51\x1f\x87\xbc\xd9\xcc9\xc8\xb9([%\x8f\xd3\xc2\xcd\xbf\xb1\x92\xb36\x1dn\x05\xf7_\xacd\xff\x9bz\xb8\xf5\x97\xb4\xee\xa3\x91CE\xebh\x8e1\xc7z\xfc\xf2f\xd6&\xceEZ\xf7\xb12T\xb0\x8c\x96\x0b\x8d\xea\x9e\xa7\xecd\xff\xd6\xf1\xaa\xe4\x07G\x85T\xff\xdbf\xf5\xcc'\xb9\xdb\x96\xdd\x16\x14\x07\x8e\xe5\"\x8e-\xc7x\xdc\xf3\x86r\x91s\x92\xb6qA\xbe\xf2f\x12-\x8dn\xbb\xfab\xbd\xb2}-\x9d\xde\xde\x1b\xefJ~\xe8\xd4\xf9\x1emc\x0do\x8b\xb6\xd9\x94\xdd6\x14&\x8e]s6o^\xccA\xde\xac\xcd\xe6Mc.\xf3\x86\xf7\xc8\x11\xad\xa2e\xb9\x11\x9f\xbea\xa2*\xf8\x7f\xe3\xde\xb4-\xda\xe6\x8d\xb4\xed\x13\x04.Q)D|y\xc8\\\x8a\xe1My\xc8\x9b\xe5\x9c\xb3\xb9:\x9b\xd4z\xc3M+\xa9\xdf\xf8N\xae*\xf9\xa1\xfe\xe5\xbbj\xa8\xe1a\xe12\xe6S9\xca\x91s\x05\xa3\x9ccF\xb1{\x84c\x98\xfb\xbc\xe9\x7f\xc7[\xd1\xb4\x82\xca\x91\xceEeO\xceu\x05\xf7_\x94@\xed:\x8f]\xcdCz\xad\xa2\x08/\\+\xb2\xb3F+\xc7\x8ac\x96\xef\xbc\xa1\xb2\x9c\x94\x8bJ\x9f\x91\xef\x8a\xf2\xa2G\xdb7\xba\xcd\x04\x9f:3\x02\xb3\xef2\xe2\x98\x0cs\x8c8V2\xe4\x0c\x95\xe7\xb4\\T\xfdR*[\x1dr\xfb\x88\xd5s[r\x9b\xca\x12_=\xd6\xf9\x00\xe7\x1c\xe7^\xc1\x1ca<\xbe\xf41z\xb4\xfdf:\xfdJ1\x14l\xa7\x07\xff\xa9'\xba~.\x9cJ\xaf\xc0\xec\xfb\xe1\xf5\x10\xa8\xed\xb8\x0d\xb9-%\xe9Fo\xe7\x1c\xe3\\\x93\xb1Ku(\xda\x8dD\xcfO\xadT\xff\x1aY\x8e6\xdeH\xcb\x1djp\xf2\xa7\x05n\xc6\x1a\x8b\x08\xb7\x19\xb7\x9d4\xbd\x82T\xff\x0b\x9c[\x9cc\x85\xde\xb8 \xad\xb2c\x91Y\xdd\xfb'Y\x1a\xd7\x8cM_\xa5\x84\x1b.\xe2\xee\x02r\xff\xa0&s[q\x9bI\x13?\xca%\xce)\xce\xadbid\x8f\xd3\xe3\x9fm\xa5\x06\x1e\xb3k2[\xe48=\x0fn\xf1X\x89\x1f\xbb\xb50\xff\xa8\xf3b?\xf8?^n\x1bj\xa3\xeb\xb8\xad\xe4\x18t\xc9l\xe1\x1c\xe2\\\x12Ez\xfd]\xbd\x91\xec\xbb\xc5J\x0fm\x90\xe5h\xa4\xf8k\xeeq\x9a\x91\xa3\xa8l~\xec\x13\x1f\xf0s\x9b(\xfe\xf4\xbd\x12\xcdmm\xe0\xdc)\x85\x81\x16\x9f\x1an9\x97\x8e\x06odg\xbf\xf3>\xebj\xc4\xa7\xaf+s\x1b\x17e\x87\x82K\xf9\x8eE\xae\xbb\x8f\xdb\x82\xdbD\x92\xab#\xf8\x92\x917\xd4p\xeb9\xa56T?\x85*\xbe\xd5JgFe\xd8I\xf6^\x0e=\xc9\xc9\x0f\x890Kx\x071\xa9\x0d~%\xc9\x1c\xd6{Vjh\x94s\x84\x7f\x07\x95b0\x1c\x1e#Y\xef\xb1\x93?SC\x0d\xb2\xfc.\xd9A\xdd\x8a\x87\xa9l\xc7\x8b\xd2\xba*\x98\xebz<\xd5\xfd\x11n\x03\x19b\xc19\xe11\xa2?\xe1\x1c\x11%|\x85\xb6C\xa8\xfe\xb8C\xf1\x9d\xea\xd4C\xaf\xc8ry\xb4\x11\xebz\x8e\xca\xb6\x84\x16\xb5\x04b\xc0u\\\xc2u\x96\xe5\xf6\x05\xce\x05\xca\x89S(7b\x02\xb7/\xec\xed\xf7Z\xc2\xe9\x99\xed\xd2\xc3\xbf7\xabfn\xcb\xff\xe9}h\xb7\x91\x98\xb1\x81\x82t\x13\x95\x8dG\xb9\x8a\xf1\xf2y\xaeS3\xd7\x91\xeb\xcau\xce\xfb\xf0-\xc5\x9er\xe0>\xca\x85\x91\xbd9\x81'\xd8\xfc\x0f~\x1eo\x1fu\xb9\xae\xf2\x86\x9b\xa4\x18\xe52b\xd3\xb6\xa8\x81\xfa;\xa9\\\xa7\x17\xd9P0\xd7\xe5t\xae\x1b\xd7Q\x8a \\\x8a9\xc7\x9es@\xe0V\x85\x8f=\xaa\xd5\xba\x8c\xd8\x02\xc5\x9fz^\x86\x1f\xeftT\xe3;\xd2VQ\xb9.+\x92\xa1`\xae\xc3e\\'\xae\x9b\x0c?\xc6)\xd6\xcfr\xcc9\xf6\x027\xbb\x1d\x12>\x82,\xd0\"\xad+(\x88\x9b\xa4\xf8\xd1X\x9e\xde\xa8\xf8R\x97;\xfd\xe9\x1eQ\x98\x93T\x8a\x12\xe9\xea\xe5:p]\xa4\x98\x11\xa7\xd8r\x8c\xa9l\xf3q\xd68\xbc\xb3IF\x0dMY\xaaWv\xbe,\xc5U\xa3\x95\x1d\xdb\xf4X\xe7\xedf\xa2\xfb\xf4l\x1f\xb9PX\xf4;\xe3\x0c39\xeb\x0e\xae\x83\x1cm\xd9\xf92\xc7\x96\xca6\x84\xb3\xc6\x91\xa9t*\xfe\xefP\x80\x9f\xb1\xd2C;$\x18\xe5\x1a\xb5\xd3C/\xe9\xd1\xf6/ 5\x1a\x97|\x94\xc5\xc1e\xe4\xb2r\x99%\x99\x11\xdf\xc1\xb1\xa4\x98~K\xe0\x8e\xcfq\x1d\xe9\x9aO\x0d\xfb\x90\x95\xec\x7fK\x9a\xfb\x10\xa2m\x17k\xc1\x86\xa9\x92\x0e\x07\xab\\6=\xd2\xb6X\x9a\xcbE(v\x1cC*\xdbY\x18\xa1\x9a\x18\xd3\x94\xf2\xba\xe5\xd4\xd0\x1b\xe5\xb9\xb2t\xe6\n\x8f?9[\xb2Q.\xaf\xc7_}\x14\x95m\xa5<;\xc7\xc0\x9b\x14\xbb\x1b8\x86H\xe3\x89\xee6\x08\xf1M_:\xf3\x8e];\xbc[\x8e\xcb\"Fv\n\xb7>O\x88j%\xcfG\xc6I{\xcb\xe0\xd6O\xe42I\xd16\x1c#\x8a\x15\x95\xed\x1b\x02\x93~9\xdcI\x1c\x8e\xe3\x8d\xf8\xf4?\xcbr\xdd\x10-\xbb\xd4P\xe3\xd5\"\x87O\xef\xfb\x08I5\xd8x\x0d\x97E\x96\xeb\xdb(F\x8fS\xac\x8e\xc3\xce\x91\x87aK\xbe_A-\xaf[&K7\xc2\xae\xc9\xbcmT\xf5\xf2\xb5\\#yh\x8f\xd9\xbcm.\x83,\xedA\xb1Y\xea\xd6\xc3\xfcN\x17\x05\xe9\x9a\xaf\xf9\x125V\xe9\xb1\x13_\xb5\xd2\x83\xafK\xb2\x93l\xb7R\x03k\xca\xbc\xe5_\x16\xb9\xb92\xd8\xe4m\xf16y\xdb\x92\\\xf4\xf9:\xc5\xe4\\\xe1\x8d\xf3(\x15\xee\xff\x97@P 5\x9f\xaa'\xbaWH\xf3\xe3\xbd\xaa\xf7E-\xdcr\x0d\x95\xadc\x02\xeb\xdd\xc9\xdb\xe0m\xc9Ro\xa3\xaa\xfbn\x8e\x05\xc7\x04i)\x17C\x89\xb4\xf6j\x91\xf6k$\x1a\xd6|\xd5H\xcc\xf8-\x95m\xde\x04\xd4w\x9e\x11\x9f\xf1;\xde\x864O4\x8c\xb4_\xcd1\xe0X \x1d\xa5\x9d/\xf1\xa4\x8dD\xcf\x12:\xcd\xaf\x91\xa4\xcb\xb5K\x8b\xb4>\xe62bgS\xf9\xe2\xe3P\xc78\xadk!\xaf\x93\x1f\xce,I\x97\x8a\x9f0\xb2\x84\xda>\x85\xf9\x8d\x02\xe9r\xe9\x89\x19\xe7\xd0\xd1u\xa5<\x93\x8a\xed\xcf{\x83\x0d|\xc1c\xf7a&\x11\x7f\xa7\x9b\xd6q9\xad\xeb\x05\x89\xce\x92+\xf5\xaa\xee\xaf\x08\x89\x9e\x83\x0b\x878\xca\xa5T4\x0eZ\xa9\xfe\xdb\xcdD\xcf\xdbv:3*\xc1\xe3j6\xa9\xe5\xb5\xb7 \xa7\x87\xdf\xfd>\x96\x89E/\x7f\x87\xbf\xcb\xeb\xc8\xffC\xf82\xa3Fl\xda[Vj\xe06\xa5\xa2i\x00\xa3T\x85-\xa2\xf8jn\xd2+;\x9e\xa5\xe4\x92b\xf2\xccm'V\xb9\xed\xd8\xdcl\x97k\xd2A\xce\x1aq\xfe,}\xe7II\x9eI\xb5\x93\xdb\xd2\xa5\x85\xaf\xa7\xb2\x85\x91^\xc5\x81\xdf\x11\xb1P\xf1\xa7\x1f5\xaaz\xa4\x18\n\xf5V4\xff\xc3\xa9\x07/\xa1rM\x11\x1f=\x89\xc6\x7f\x9bB\x9f\xb9\x94?+\xc7\x08U\xcf6j\xc3?R\xb9\x16d\xdb\x14\x8a\x08_R]/\x1c\xca\x83vr`\xab\x0c].\xea\xfe\xed\x98\xe4R\xff N\x97Z\xd1z\x85\x11\xefZ]*;\x07\xd5\xf5i5\xd2\xfa=\x81\xdba\xe1\x10\xc5\x14\x7f\xf2l\xbd\xb2\x83\xdf\xe2\xba\xa7Xw\x0c\xae\x9b\x1e\xebxH\xf1\xa7\xf8\xb9T1\x84\x1d\xc6j\x8eY5\xf3\x1e+5\xf0j\xd1u\xa9\xa8N\xfcP\x07\xaa\xe31\x083\x1c\x89\x16o\xb4\xe3\x87\xd4G_[<;\xc7\xac\x17\xb5\xca\x0e\x9e\xf8kFxa<\x98.#r\xb6\x9d\x1a\\O\xdd\x92\x1d\x05\xdc\xa5\xda\xc1up\x19Q\xeeR\x99\x08+\x8c\xb7\x99fr\xd6\xe3\xbe\x9a\xcc\xb6\x82\xdbA\xa8\xcc\\v\xb1\xef~\x14\x80 \x93\xd0\"m\xdf/\xb8\xc9\xbfH\x1b\xbfZ \x8e\xf0\xc1D+\x13\x81z\xc3\xe1\x0d|\xb2Pv\x0e*\xeb\x1c.\xb3\xc0C\xa2!\x87\xbcZ\xb0\xe1h\xfa\xf1~\x9f\xbc\x93\x7f\xfd\xf7j\xa1&\xd9\x1e\x8b\n%D\xf1VL\x9b\xa6\xc7g\\\xa2E;\xb6\xcb\xb2cx\xc3M[\xf5x\xf7bodz\xa7(\xccw\x99@\x11q(\xbe\x86\x84\xc7W\xfb\x05\xb7\x9dXMg\x94]y\xbc\\d\x97\xdb\x8e\xff\x8d\x9fz\xa2\xfa\x9bd\x7f5\x03\x94\x18\xbe\x99\xe8\x145P\xf3\x0b=\xda\x96\xf3\xf7\xfe\xe9\x95m\x9bi\xdb7S\x19>C\x8b\x86p\x80\x8c\xf8E\xa4\xddn+~\xb9\x1a\x9c\x92\xb3\xd9w\xda\xd6\x06\xda\xe6\xa5\xb4\xed.\x81W\x99A\x01\xf0 Q\xb6X\xaf\xec|\xd2\xac\xee\x9b\xb09\x13\xab\xba\xef]\xda\xc6*\xda\xd6wi\x9b6\x9a\x1d\n\xedl\xf2Yo\xb0\xf1\xd7f\xa2{\xdc\xdf\x8aE\xeb\xdc\xe4\x0d5\xf2+\xaaO\xcbn\x0b\xa0\xe0\xf0\xbcC\xa3K\x0b\xde`V\xf5\xbcj\xa73{\xc6\xe1 #{x]\xb4N~.U\x83\xc0\xdc\x06\x14\x01\xfe]p\x99Y5s\xed\x91\xbc\x88\x94\xbe\xbb\x9d\xd7A\xeb\xe2\xdba\xf1Z\x01(*|o\xf7\xb1j\xb0\x9e_r\xb3k\xec\x17\x1afv\xa9\xc1\xc9\x0f\xd2:\xe6\xa0)\xa1X9\x84\xd0\x83en\xed\xc6\xb1\xee \xf4\x9d\xeb\xf9\xbb\x02s\x1bP\x02g\x92r\xa7\xea_d$\xba\x0f\xfa\xe2\x1b\xfa\xcc\x1a\x97\xea?\x8f\xbf#\xf0\x84\x11(!\x01\x8f\xafj\x8e7\xd8p\xd7\x01/\x19 5\xdeI\x9f\xf9Dv\xe7\x00(9\x9aK\xabl\xd5BM\x8b= 0) {\n output_string += `
\n
\n Just getting started? Try this:
\n :create index test
\n :use test
\n :create frame foo
\n SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit\n `\n }\n }\n }\n }\n\n\n var markup =`\n
\n
\n
\n
\n
Input
\n       \n Source: ${res.indexname}\n
\n
\n ${res.input}\n
\n
\n
\n
\n
output
\n       \n ${res.querytime_ms} ms\n
\n
\n ${output_string}\n
\n
Expand
\n \n
\n
\n
\n
\n \n Export raw .csv\n
\n
\n
\n `\n node.innerHTML = markup;\n this.output.insertBefore(node, this.output.firstChild);\n\n // Expand when overflow\n var element = this.output.firstChild.getElementsByClassName(result_class)[0];\n var expand = this.output.firstChild.getElementsByClassName(\"expand\")[0];\n if (element.clientHeight < element.scrollHeight) {\n expand.style.display = 'block';\n } else {\n expand.style.display = 'none';\n }\n expand.onclick = function () {\n element.style.height = element.scrollHeight + \"px\";\n expand.style.display = 'none';\n return false;\n };\n }\n\n populate_index_dropdown() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/schema')\n var select = document.getElementById('index-dropdown')\n\n xhr.onload = function() {\n var schema = JSON.parse(xhr.responseText)\n for(var i=0; i 0) {\n select.value = 1;\n }\n }\n xhr.send(null)\n }\n\n}\n\nfunction populate_version() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/version')\n var node = document.getElementById('server-version')\n\n xhr.onload = function() {\n var version = JSON.parse(xhr.responseText)['version']\n var version_major_minor = /(v\\d+\\.\\d+)/.exec(version)[0]\n var doc_link = document.getElementById('nav-documentation')\n doc_link.onclick = function() {\n window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/')\n }\n node.innerHTML = version\n }\n xhr.send(null)\n}\n\nfunction handle_nav_click(e) {\n // e.id = \"nav-xxx\"\n name = e.id.substring(4)\n set_active_pane_by_name(name)\n window.location.hash = name\n}\n\nfunction set_active_pane_by_name(name) {\n // toggle the nav buttons\n document.getElementsByClassName(\"nav-active\")[0].classList.remove(\"nav-active\")\n document.getElementById(\"nav-\" + name).classList.add(\"nav-active\")\n\n // toggle the main interface content divs\n document.getElementsByClassName(\"interface-active\")[0].classList.remove(\"interface-active\")\n document.getElementById('interface-' + name).classList.add(\"interface-active\")\n\n // hack hack\n switch(name) {\n case \"cluster\":\n update_cluster_status()\n break\n case \"documentation\":\n open_external_docs()\n break\n }\n}\n\n\nfunction update_cluster_status() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/status')\n status_node = document.getElementById('status')\n xhr.onload = function() {\n var status = JSON.parse(xhr.responseText)\n render_status(status)\n }\n xhr.send(null)\n}\n\nfunction render_status(status) {\n // render node table\n var nodes_div = document.getElementById(\"status-nodes\")\n while (nodes_div.firstChild) {\n nodes_div.removeChild(nodes_div.firstChild);\n }\n\n var nodes = status[\"status\"][\"Nodes\"]\n table = document.createElement(\"table\")\n tbody = document.createElement(\"tbody\")\n table.appendChild(tbody)\n var caption = document.createElement(\"caption\")\n caption.innerHTML = \"(\" + nodes.length + \")\"\n table.appendChild(caption)\n\n var header = document.createElement('tr')\n markup = `Host\n State`\n header.innerHTML = markup\n tbody.appendChild(header)\n for(var n=0; n${nodes[n][\"Host\"]}\n ${nodes[n][\"State\"]}`\n row.innerHTML = markup\n tbody.appendChild(row)\n }\n nodes_div.appendChild(table)\n\n // render index tables\n var indexes_div = document.getElementById(\"status-indexes\")\n while (indexes_div.firstChild) {\n indexes_div.removeChild(indexes_div.firstChild);\n }\n\n var indexes = nodes[0][\"Indexes\"] // TODO currently comes from only node 0\n for(var n=0; nName\n Cache Type\n Cache Size`\n header.innerHTML = markup\n tbody.appendChild(header)\n\n var frames = indexes[n][\"Frames\"]\n if(frames) {\n for(var m=0; m${frames[m][\"Name\"]}\n ${frames[m][\"Meta\"][\"CacheType\"]}\n ${frames[m][\"Meta\"][\"CacheSize\"]}`\n tbody.appendChild(row)\n }\n }\n indexes_div.appendChild(table)\n }\n\n // render slice tables\n // TODO enable when Slices element is present in status response\n /*\n var slices_div = document.getElementById(\"status-slices\")\n data = \"\"\n for(var n=0; n\"\n }\n }\n slices_div.innerHTML = data\n */\n\n}\n\nfunction open_external_docs() {\n window.open(\"https://www.pilosa.com/docs\");\n}\n\nfunction check_anchor_uri() {\n var pane_names = {\"console\": 0, \"cluster\": 0, \"documentation\": 0}\n var anchor = window.location.hash.substr(1);\n if(anchor in pane_names) {\n set_active_pane_by_name(anchor)\n }\n}\n\nDate.prototype.today = function () {\n return this.getFullYear() +\"/\"+ (((this.getMonth()+1) < 10)?\"0\":\"\") + (this.getMonth()+1) +\"/\"+ ((this.getDate() < 10)?\"0\":\"\") + this.getDate();\n}\n\nDate.prototype.timeNow = function () {\n return ((this.getHours() < 10)?\"0\":\"\") + this.getHours() +\":\"+ ((this.getMinutes() < 10)?\"0\":\"\") + this.getMinutes() +\":\"+ ((this.getSeconds() < 10)?\"0\":\"\") + this.getSeconds();\n}\n\npopulate_version()\n\n\nclass Autocompleter {\n constructor(input, output) {\n this.input = input\n this.output = output\n this.keyword_map = this.static_keywords\n this.init_dynamic_keywords()\n }\n\n get static_keywords() {\n return {\n // keyword: length of substring that comes after cursor\n \"SetBit()\": 1,\n \"ClearBit()\": 1,\n \"SetRowAttrs()\": 1,\n \"SetColumnAttrs()\": 1,\n \"Bitmap()\": 1,\n \"Union()\": 1,\n \"Intersect()\": 1,\n \"Difference()\": 1,\n \"Count()\": 1,\n \"Range()\": 1,\n \"TopN()\": 1,\n \"frame=\": 0,\n }\n }\n\n complete() {\n var completer = this\n // extract word fragment ending at cursor. a word fragment:\n // - starts with last nonalpha character before cursor (or beginning of string)\n // - ends at cursor\n var word_start = completer.input.selectionEnd-1\n while(word_start>0) {\n var c = completer.input.value.charCodeAt(word_start)\n if(!((c>64 && c<91) || (c>96 && c<123))) {\n word_start++\n break\n }\n word_start--\n }\n var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)\n\n // check for keyword match and insert if exactly one match\n var matches = []\n for(var keyword in this.keyword_map) {\n if(keyword.startsWith(input_word)){\n matches.push(keyword)\n }\n }\n if(matches.length > 1) {\n // completer.output.innerHTML = whatever\n }\n\n if(matches.length == 1) {\n // completer.output.innerHTML = \"\"\n var cursor_pos = completer.input.selectionEnd\n var completion = matches[0].substring(input_word.length)\n var before = completer.input.value.substring(0, cursor_pos)\n var after = completer.input.value.substring(cursor_pos)\n completer.input.value = before + completion + after\n var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]\n completer.input.setSelectionRange(new_pos, new_pos)\n }\n }\n\n init_dynamic_keywords() {\n // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list\n }\n\n add_keyword() {\n // call when index or frame created in webui\n }\n\n remove_keyword() {\n // call when index or frame deleted in webui\n // issue: if e.g. multiple indexes have same frame, removing one removes all.\n // solution: maintain count. requires more elaborate representation of keywords.\n }\n}\n\nvar input = document.getElementById('query')\nvar output = document.getElementById('outputs')\nvar button = document.getElementById('query-btn')\nvar autocomplete_output = document.getElementById('autocomplete-container')\n\nautocompleter = new Autocompleter(input, autocomplete_output)\nrepl = new REPL(input, output, button, autocompleter)\nrepl.populate_index_dropdown()\nrepl.bind_events()\n\ninput.focus()\n\ncheck_anchor_uri()\n\nfunction isJSON(str) {\n try {\n JSON.parse(str)\n } catch (e) {\n return false\n }\n return true\n}\n\nfunction parse_query(query, indexname) {\n var keys = query.replace(/\\s+/g, \" \").split(\" \");\n var command = keys[0];\n var command_type = keys[1];\n var command_name = keys[2];\n var option_str = keys.slice(3, keys.length)\n var options = parse_options(option_str);\n if (command !== \":use\") {\n if (!command_name){\n return {}\n }\n }\n\n var parsed_query = {};\n parsed_query[\"command\"] = command.substr(1, command.length);\n parsed_query[\"command_name\"] = command_name;\n switch (command) {\n case \":create\":\n parsed_query[\"request\"] = \"POST\";\n if(Object.keys(options).length === 0) {\n parsed_query[\"data\"] = \"\";\n } else {\n var opts = {\"options\":{}};\n for (var o in options) {\n opts.options[o] = options[o]\n }\n parsed_query[\"data\"] = JSON.stringify(opts);\n }\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n break\n }\n break;\n case \":delete\":\n parsed_query[\"request\"] = \"DELETE\";\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n }\n break;\n case \":use\":\n parsed_query[\"command_name\"] = keys[1];\n break;\n default:\n return {}\n }\n return parsed_query;\n}\n\nfunction parse_options(option_str) {\n var int_keys = [\"cacheSize\"];\n var bool_keys = [\"inverseEnabled\"];\n var options = {};\n for (var i = 0; i < option_str.length; i++) {\n var parts = option_str[i].split('=');\n if (int_keys.indexOf(parts[0]) !== -1 ){\n options[parts[0]] = Number(parts[1])\n } else if (bool_keys.indexOf(parts[0]) !== -1){\n options[parts[0]] = (parts[1] == \"true\")\n } else {\n options[parts[0]] = parts[1]\n }\n }\n return options;\n}PK\x07\x08\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-cluster-active.svgnav_cluster_1\nPK\x07\x08\xc1J\xead \x02\x00\x00 \x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-cluster.svgnav_cluster_1PK\x07\x08\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-console-active.svgnav_consolePK\x07\x08\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-console.svgnav_console\nPK\x07\x08\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00assets/nav-documentation-active.svgdocumentation\nPK\x07\x08\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00assets/nav-documentation.svgdocumentationPK\x07\x08\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00assets/nav_item1.svgnav_item1PK\x07\x08+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xa0~\xe6J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00assets/style.css*{\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box;\n}\nbody{\n font-family: sans-serif;\n background-color: #fbfcfd;\n margin: 0;\n color: #102445;\n}\nh2{\n margin-bottom: 30px;\n}\n\nh5{\n text-transform: uppercase;\n letter-spacing: 2px;\n line-height: 1.21;\n margin: 0;\n}\na{\n line-height: 1.38;\n letter-spacing: 0.2px;\n text-decoration: none;\n color: #102445;\n}\n\n\na:hover{\n color: #1db598;\n}\n\ntextarea{\n width: 100%;\n margin-bottom: 10px;\n border-radius: 2px;\n background-color: #fbfcfd;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n outline: none;\n padding: 30px;\n}\n\n\nselect{\n /*-webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n background: url(\"img/chevron-down.png\") no-repeat calc(100% - 10px) !important;*/\n border-radius: 3px;\n background-color: #fbfcfd;\n width: 187px;\n height: 50px;\n border: solid 1.5px #e4eff4;\n font-size: 18px;\n font-weight: bold;\n line-height: 1.39;\n letter-spacing: 0.2px;\n color: #102445;\n padding: 10.5px;\n\n}\n\nbutton{\n width: 165px;\n height: 50px;\n border-radius: 3px;\n background-color: #1db598;\n outline: none;\n border: none;\n font-size: 16px;\n color: white;\n}\n\nem{\n font-style: normal;\n opacity: 0.5;\n font-size: 14px;\n font-weight: 500;\n letter-spacing: 0.2px;\n color: #102445;\n}\n\n.header{\n height: 92px;\n display: flex;\n align-items: center;\n justify-content: space-between;\n width: 90%;\n margin: auto;\n}\n\n.container{\n display: flex;\n height:100%;\n min-height: 100vh;\n}\n.nav{\n color: white;\n display: flex;\n flex-direction: column;\n width: 150px;\n background: #3c5f8d;\n}\n\n.nav-item{\n height:150px;\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n border-bottom: 3px solid #2a4871;\n cursor: pointer;\n}\n\n.nav-active{\n background: #f2f7f9;\n font-weight: bold;\n color: #1db598;\n}\n\n.nav-item > .nav-image {\n display: flex;\n}\n\n.nav-item > .nav-image-active {\n display: none;\n}\n\n.nav-active > .nav-image {\n display: none;\n}\n\n.nav-active > .nav-image-active {\n display: flex;\n}\n\n\n.interface{\n display: none;\n flex: 1;\n flex-direction: column;\n align-items: center;\n background: #f2f7f9;\n}\n\n.interface-active{\n display: flex;\n}\n\n.query{\n margin-bottom: 30px;\n}\n.query,\n.output-container,\n.status-container{\n width: 75%;\n}\n\n.output{\n margin-bottom: 30px;\n}\n\n.input-controls{\n display: flex;\n justify-content: flex-end;\n}\n\n.tabs{\n display: flex;\n background: #eaf2f6;\n}\n.active-tab{\n background: white;\n font-weight: bold;\n color: #1db598;\n\n}\n\n.tab{\n height:60px;\n width: 100px;\n border-top-right-radius: 5px;\n display: flex;\n align-items: center;\n justify-content: center;\n visibility: visible;\n cursor: pointer;\n\n}\n\n.pane{\n background: white;\n padding: 30px;\n display: none;\n}\n\n.active{\n display: block;\n}\n\n.result-io-header{\n display: flex;\n align-items: center;\n margin-bottom: 15px;\n}\n\n.result-input,\n.result-output,\n.result-error{\n height: 60px;\n border-radius: 2px;\n background-color: #fafafa;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n color: #102445;\n padding: 15px;\n margin-bottom: 15px;\n word-break: break-all;\n overflow-wrap: break-word;\n overflow:hidden;\n}\n\n\n.result-output{\n background-color: #edf9f7;\n border-left: solid 4px #1db598;\n}\n\n.result-error{\n background-color: #fbf1f0;\n border-left: solid 4px #fa3035;\n color: #fa3035;\n}\n\n.raw{\n height: 253px;\n display: flex;\n align-items: center;\n justify-content: center;\n}\n\n\n.result-table > table {\n border-left: solid 4px #1db598;\n}\n\ntable{\n border: solid 0.5px #e0e0e0;\n width: 100%;\n margin-bottom: 30px;\n /*color:#3c5f8d;*/\n}\ncaption{\n text-align:left;\n font-size: 16px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n text-align: left;\n}\nth{\n font-size: 14px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n color: #102445;\n text-transform: uppercase;\n text-align: left;\n padding: 21px 30px;\n background-color: white;\n}\ntr{\n border: solid 0.5px #e0e0e0;\n background-color: white;\n}\ntr:nth-child(even) {\n background-color: #f2f7f9;\n}\ntd{\n padding: 21px 30px;\n}\n\n.expand {\n text-align: center;\n}\n\n.query h2 {\n display: inline-block;\n}\n\n.query-tooltip {\n position: relative;\n display: inline;\n color: #000;\n margin-left: 5px;\n}\n\n.query-tooltip:hover {\n color: #000;\n}\n\n.query-tooltip-content {\n background-color: rgb(250, 250, 250);\n border: solid 1.5px #e4eff4;\n color: #102445;\n border-radius: 2px;\n padding: 15px;\n margin-bottom: 15px;\n\n position: absolute;\n left: 80px;\n top: -30px;\n z-index: 1;\n}\n\n.query-tooltip-container {\n position: relative;\n visibility: hidden;\n}\n\n.query-tooltip:hover+.query-tooltip-container{\n visibility: visible;\n}\n\n.code{\n font-family: monospace;\n}\n\nPK\x07\x08\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00cz\xbfJ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00index.html\n\n\n \n \n \n \n Pilosa WebUI\n \n\n\n
\n \"\"\n
\n
\n
\n
\n
\n \"\"\n \"\"\n Console\n
\n
\n \"\"\n \"\"\n Cluster Admin\n
\n
\n \"\"\n \"\"\n Documentation\n
\n
\n
\n\n
\n

Query

\n ?\n
\n
\n
PQL
\n
\n SetBit(frame=foo, rowID=0, columnID=0)
\n ClearBit(frame=foo, rowID=0, columnID=0)
\n SetRowAttrs(frame=foo, rowID=0, color=\"blue\")
\n SetColumnAttrs(frame=foo, columnID=0, shape=\"circle\")
\n Bitmap(frame=foo, rowID=0)
\n Range(frame=foo, rowID=0, start=\"2010-01\", end=\"2017-03\")
\n Count(<BITMAP_CALL>)
\n TopN([BITMAP_CALL], frame=foo, n=20)
\n Union([BITMAP_CALL, ...])
\n Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
\n Difference(<BITMAP_CALL>, <BITMAP_CALL>)\n
\n
\n
Special commands
\n
\n :create index test [columnLabel=column]
\n :use test
\n :create frame foo [rowLabel=row]
\n :delete index test
\n :delete frame foo\n
\n
\n <tab>: autocomplete
\n <up>/<down>: history
\n
\n
\n \n
\n
\n \n    \n \n
\n
\n
\n\n
\n

Output

\n
\n \n
\n
\n\n
\n\n
\n
\n

Nodes

\n
\n
\n
\n
\n

Indexes

\n
\n
\n
\n
\n \n
\n\n
\n\n
\n docs!\n
\n\n
\n \n\n\nPK\x07\x08\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3JJ\x1c\xff\xa8G\x0e\x00\x00G\x0e\x00\x00\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00assets/chevron-down.pngPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x96\x84jK\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x8c\x0e\x00\x00assets/main.jsPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc1J\xead \x02\x00\x00 \x02\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x92W\x00\x00assets/nav-cluster-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xe6Y\x00\x00assets/nav-cluster.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81/\\\x00\x00assets/nav-console-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x1a^\x00\x00assets/nav-console.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xfc_\x00\x00assets/nav-documentation-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x819b\x00\x00assets/nav-documentation.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81kd\x00\x00assets/nav_item1.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xa0~\xe6J\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81Of\x00\x00assets/style.cssPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00cz\xbfJ\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xcay\x00\x00index.htmlPK\x05\x06\x00\x00\x00\x00\x0b\x00\x0b\x00\xf2\x02\x00\x00\xf1\x89\x00\x00\x00\x00" + fs.Register(data) +} diff --git a/test/executor.go b/test/executor.go index c04f91eca..c4af65b01 100644 --- a/test/executor.go +++ b/test/executor.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/inmem" "github.com/pilosa/pilosa/pql" ) @@ -42,6 +43,7 @@ func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster + e.TranslateStore = inmem.NewTranslateStore() e.Node = cluster.Nodes[0] return e } diff --git a/translate.go b/translate.go new file mode 100644 index 000000000..398055fdb --- /dev/null +++ b/translate.go @@ -0,0 +1,1006 @@ +package pilosa + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/cespare/xxhash" +) + +const ( + LogEntryTypeInsertColumn = 1 + LogEntryTypeInsertRow = 2 +) + +const ( + DefaultMapSize = 10 * (1 << 30) // 10GB + + DefaultReplicationRetryInterval = 1 * time.Second +) + +const ( + ReplicationBufferSize = 65536 +) + +var ( + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: operation not supported, translate store read only") +) + +// TranslateStore is the storage for translation string-to-uint64 values. +type TranslateStore interface { + TranslateColumnsToUint64(index string, values []string) ([]uint64, error) + TranslateColumnToString(index string, values uint64) (string, error) + + TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) + TranslateRowToString(index, frame string, values uint64) (string, error) + + // Returns a reader from the given offset of the raw data file. + // The returned reader must be closed by the caller when done. + Reader(ctx context.Context, off int64) (io.ReadCloser, error) +} + +// Ensure type implements interface. +var _ TranslateStore = &TranslateFile{} + +// TranslateFile is an on-disk storage engine for translating string-to-uint64 values. +type TranslateFile struct { + mu sync.RWMutex + data []byte + file *os.File + w *bufio.Writer + n int64 + writeNotify chan struct{} + + once sync.Once + wg sync.WaitGroup + closing chan struct{} + + cols map[string]*index + rows map[frameKey]*index + + Path string + MapSize int + + // If non-nil, data is streamed from a primary and this is a read-only store. + PrimaryTranslateStore TranslateStore + + // Delay after attempting to connect to a primary that the store will retry. + ReplicationRetryInterval time.Duration +} + +// NewTranslateFile returns a new instance of TranslateFile. +func NewTranslateFile() *TranslateFile { + return &TranslateFile{ + writeNotify: make(chan struct{}), + closing: make(chan struct{}), + cols: make(map[string]*index), + rows: make(map[frameKey]*index), + + MapSize: DefaultMapSize, + + ReplicationRetryInterval: DefaultReplicationRetryInterval, + } +} + +func (s *TranslateFile) Open() (err error) { + // Open writer & buffered writer. + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { + return err + } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { + return err + } + s.w = bufio.NewWriter(s.file) + + // Memory map data file. + if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.MapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + return err + } + + // Replay the log. + if err := s.replayEntries(); err != nil { + return err + } + + // Stream from primary, if available. + if s.PrimaryTranslateStore != nil { + s.wg.Add(1) + go func() { defer s.wg.Done(); s.monitorReplication() }() + } + + return nil +} + +func (s *TranslateFile) Close() (err error) { + s.once.Do(func() { + close(s.closing) + + if s.file != nil { + if e := s.file.Close(); e != nil && err == nil { + err = e + } + } + if s.data != nil { + if e := syscall.Munmap(s.data); e != nil && err == nil { + err = e + } + } + }) + s.wg.Wait() + return err +} + +// Closing returns a channel that is closed when the store is closed. +func (s *TranslateFile) Closing() <-chan struct{} { + return s.closing +} + +// Size returns the number of bytes in use in the data file. +func (s *TranslateFile) Size() int64 { + s.mu.RLock() + n := s.n + s.mu.RUnlock() + return n +} + +// IsReadOnly returns true if this store is being replicated from a primary store. +func (s *TranslateFile) IsReadOnly() bool { + return s.PrimaryTranslateStore != nil +} + +// WriteNotify returns a channel that is closed when a new entry is written. +func (s *TranslateFile) WriteNotify() <-chan struct{} { + s.mu.RLock() + ch := s.writeNotify + s.mu.RUnlock() + return ch +} + +func (s *TranslateFile) appendEntry(entry *LogEntry) error { + offset := s.n + + // Append entry to the end of the WAL. + n, err := entry.WriteTo(s.w) + if err != nil { + return err + } else if err := s.w.Flush(); err != nil { + return err + } + + // Move position forward. + s.n += n + + // Apply the entry to the current state. + if err := s.applyEntry(entry, offset); err != nil { + return err + } else if err := s.file.Sync(); err != nil { + return err + } + + // Notify others of write update. + close(s.writeNotify) + s.writeNotify = make(chan struct{}) + + return nil +} + +func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { + // Move offset to the start of the id/key pairs. + offset += entry.HeaderSize() + + var idx *index + switch entry.Type { + case LogEntryTypeInsertColumn: + idx = s.col(string(entry.Index)) + + case LogEntryTypeInsertRow: + idx = s.row(string(entry.Index), string(entry.Frame)) + + default: + return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type) + } + + // Insert id/key pairs into index. + for i, id := range entry.IDs { + key := entry.Keys[i] + + // Determine key offset based on ID size. + sz := int64(UvarintSize(id)) + idx.insert(id, offset+sz) + + // Move sequence forward. + if id > idx.seq { + idx.seq = id + } + + // Move offset forward. + offset += sz + int64(UvarintSize(uint64(len(key)))) + int64(len(key)) + } + + return nil +} + +func (s *TranslateFile) replayEntries() error { + // Build a reader from the memory-map data. + fi, err := os.Stat(s.Path) + if err != nil { + return err + } + r := bytes.NewReader(s.data[:fi.Size()]) + + // Iterate over each entry and reapply. + for { + offset := s.n + + var entry LogEntry + if n, err := entry.ReadFrom(r); err == io.EOF { + return nil + } else if err != nil { + return err + } else { + s.n += n + } + + if err := s.applyEntry(&entry, offset); err != nil { + return err + } + } +} + +// monitorReplication is executed in a separate goroutine and continually streams +// from the primary store until this store is closed. +func (s *TranslateFile) monitorReplication() { + // Create context that will cancel on close. + ctx, cancel := context.WithCancel(context.Background()) + go func() { <-s.closing; cancel() }() + + // Keep attempting to replicate until the store closes. + for { + if err := s.replicate(ctx); err != nil { + log.Printf("pilosa: replication error: %s", err) + } + + select { + case <-s.closing: + return + case <-time.After(s.ReplicationRetryInterval): + log.Printf("pilosa: reconnecting to primary replica") + } + } +} + +func (s *TranslateFile) replicate(ctx context.Context) error { + off := s.Size() + + // Connect to remote primary. + log.Printf("pilosa: replicating from offset %d", off) + rc, err := s.PrimaryTranslateStore.Reader(ctx, off) + if err != nil { + return err + } + defer rc.Close() + + // Wrap in bufferred I/O so it implements io.ByteReader. + bufr := bufio.NewReader(rc) + + // Continually read new entries from primary and append to local store. + for { + // Read next available entry. + var entry LogEntry + if _, err := entry.ReadFrom(bufr); err == io.EOF { + return nil + } else if err != nil { + return err + } + + // Write to local store. + if err := s.appendEntry(&entry); err != nil { + return err + } + } +} + +func (s *TranslateFile) col(index string) *index { + idx := s.cols[index] + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + return idx +} + +func (s *TranslateFile) row(index, frame string) *index { + idx := s.rows[frameKey{index, frame}] + if idx == nil { + idx = newIndex(s.data) + s.rows[frameKey{index, frame}] = idx + } + return idx +} + +// TranslateColumnsToUint64 converts values to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertColumn, + Index: []byte(index), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + + ret[i] = v + + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.keyByID(value); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.rows[key] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertRow, + Index: []byte(index), + Frame: []byte(frame), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + ret[i] = v + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.keyByID(id); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +// Reader returns a reader that streams the underlying data file. +func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + rc := NewTranslateFileReader(ctx, s, offset) + if err := rc.Open(); err != nil { + return nil, err + } + return rc, nil +} + +type LogEntry struct { + Type uint8 + Index []byte + Frame []byte + + IDs []uint64 + Keys [][]byte + + // Length of the entry, in bytes. + // This is only populated after ReadFrom() or WriteTo(). + Length uint64 +} + +// HeaderSize returns the number of bytes required for size, type, index, frame, & pair count. +func (e *LogEntry) HeaderSize() int64 { + sz := UvarintSize(e.Length) + // total entry length + 1 + // type + UvarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data + UvarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data + UvarintSize(uint64(len(e.IDs))) // ID/Key pair count + return int64(sz) +} + +// ReadFrom deserializes a LogEntry from r. r must be a ByteReader. +func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) { + br := r.(io.ByteReader) + + // Read the entry length. + if e.Length, err = binary.ReadUvarint(br); err != nil { + return int64(UvarintSize(e.Length)), err + } + + // Slurp entire entry and replace reader. + buf := make([]byte, e.Length) + n, err := io.ReadFull(r, buf) + n64 := int64(n + UvarintSize(e.Length)) + if err != nil { + return n64, err + } + bufr := bytes.NewReader(buf) + br, r = bufr, bufr + + // Read the entry type. + if err := binary.Read(r, binary.BigEndian, &e.Type); err != nil { + return n64, err + } + + // Read index name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Index = nil + } else { + e.Index = make([]byte, sz) + if _, err := io.ReadFull(r, e.Index); err != nil { + return n64, err + } + } + + // Read frame name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Frame = nil + } else { + e.Frame = make([]byte, sz) + if _, err := io.ReadFull(r, e.Frame); err != nil { + return n64, err + } + } + + // Read key count. + if n, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if n == 0 { + e.IDs, e.Keys = nil, nil + } else { + e.IDs, e.Keys = make([]uint64, n), make([][]byte, n) + } + + // Read each id/key pairs. + for i := range e.Keys { + // Read identifier. + if e.IDs[i], err = binary.ReadUvarint(br); err != nil { + return n64, err + } + + // Read key. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz > 0 { + e.Keys[i] = make([]byte, sz) + if _, err := io.ReadFull(r, e.Keys[i]); err != nil { + return n64, err + } + } + } + return n64, nil +} + +// WriteTo serializes a LogEntry to w. +func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) { + var buf bytes.Buffer + b := make([]byte, binary.MaxVarintLen64) + + // Write the entry type. + if err := binary.Write(&buf, binary.BigEndian, e.Type); err != nil { + return 0, err + } + + // Write the index name. + sz := binary.PutUvarint(b, uint64(len(e.Index))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Index); err != nil { + return 0, err + } + + // Write frame name. + sz = binary.PutUvarint(b, uint64(len(e.Frame))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Frame); err != nil { + return 0, err + } + + // Write key count. + sz = binary.PutUvarint(b, uint64(len(e.IDs))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write each id/key pairs. + for i := range e.Keys { + // Write identifier. + sz = binary.PutUvarint(b, e.IDs[i]) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write key. + sz = binary.PutUvarint(b, uint64(len(e.Keys[i]))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Keys[i]); err != nil { + return 0, err + } + } + + // Write buffer size. + e.Length = uint64(buf.Len()) + sz = binary.PutUvarint(b, e.Length) + if n, err := w.Write(b[:sz]); err != nil { + return int64(n), err + } + + // Write buffer. + n, err := buf.WriteTo(w) + return int64(sz) + n, err +} + +// ValidLogEntriesLen returns the maximum length of p that contains valid entries. +func ValidLogEntriesLen(p []byte) (n int) { + r := bytes.NewReader(p) + for { + if sz, err := binary.ReadUvarint(r); err != nil { + return n + } else if off, err := r.Seek(int64(sz), io.SeekCurrent); err != nil { + return n + } else if off > int64(len(p)) { + return n + } else { + n = int(off) + } + } +} + +type frameKey struct { + index string + frame string +} + +const defaultLoadFactor = 90 + +// index represents a two-way index between IDs and keys. +type index struct { + seq uint64 // autoincrement sequence + data []byte // memory-mapped file containing key data + + // RHH hashmap for id-to-offset mapping. + // This is required so we don't need to store key data on the heap. + // https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf + elems []elem // id/offset key pairs + n uint64 // number of inuse elements + mask uint64 // mask applied for modulus + threshold uint64 // threshold when capacity doubles + loadFactor int // factor used to calculate threshold + + // Builtin hashmap for offset-to-id mapping. + offsetsByID map[uint64]int64 +} + +func newIndex(data []byte) *index { + idx := &index{ + data: data, + offsetsByID: make(map[uint64]int64), + + loadFactor: defaultLoadFactor, + } + idx.alloc(pow2(uint64(256))) + return idx +} + +// keyByID returns the key for a given ID, if it exists. +func (idx *index) keyByID(id uint64) ([]byte, bool) { + offset, ok := idx.offsetsByID[id] + if !ok { + return nil, false + } + return idx.lookupKey(offset), true +} + +// idByKey returns the ID for a given key, if it exists. +func (idx *index) idByKey(key []byte) (uint64, bool) { + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + if e := &idx.elems[pos]; e.hash == 0 { + return 0, false + } else if dist > idx.dist(e.hash, pos) { + return 0, false + } else if e.hash == hash && bytes.Equal(idx.lookupKey(e.offset), key) { + return e.id, true + } + + pos = (pos + 1) & idx.mask + dist++ + } +} + +// insert adds the id/offset pair to the index. +// This function will resize the map if it crosses the threshold. +func (idx *index) insert(id uint64, offset int64) { + idx.n++ + + // Add to reverse lookup. + idx.offsetsByID[id] = offset + + // Grow the map if we've run out of slots. + if idx.n > idx.threshold { + elems, capacity := idx.elems, uint64(len(idx.elems)) + idx.alloc(uint64(len(idx.elems) * 2)) + + for i := uint64(0); i < capacity; i++ { + e := &elems[i] + if e.hash == 0 { + continue + } + idx.insertIDbyOffset(e.offset, e.id) + } + } + + // If the key was overwritten then decrement the size. + if overwritten := idx.insertIDbyOffset(offset, id); overwritten { + idx.n-- + } +} + +// insertIDbyOffset writes to the RHH id-by-offset map. +func (idx *index) insertIDbyOffset(offset int64, id uint64) (overwritten bool) { + key := idx.lookupKey(offset) + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + e := &idx.elems[pos] + + // Exit if a matching or empty slot exists. + if e.hash == 0 { + e.hash, e.offset, e.id = hash, offset, id + return false + } else if bytes.Equal(idx.lookupKey(e.offset), key) { + e.hash, e.offset, e.id = hash, offset, id + return true + } + + // Swap if current element has a lower probe distance. + d := idx.dist(e.hash, pos) + if d < dist { + hash, e.hash = e.hash, hash + offset, e.offset = e.offset, offset + id, e.id = e.id, id + dist = d + } + + // Move position forward. + pos = (pos + 1) & idx.mask + dist++ + } +} + +// lookupKey returns the key at the given offset in the memory-mapped file. +func (idx *index) lookupKey(offset int64) []byte { + data := idx.data[offset:] + n, sz := binary.Uvarint(data) + if sz == 0 { + return nil + } + return data[sz : sz+int(n)] +} + +func (idx *index) alloc(capacity uint64) { + idx.elems = make([]elem, capacity) + idx.threshold = (capacity * uint64(idx.loadFactor)) / 100 + idx.mask = uint64(capacity - 1) +} + +func (idx *index) dist(hash, i uint64) uint64 { + return (i + uint64(len(idx.elems)) - (hash & idx.mask)) & idx.mask +} + +type elem struct { + offset int64 + id uint64 + hash uint64 +} + +func (e *elem) reset() { + e.offset = 0 + e.id = 0 + e.hash = 0 +} + +func hashKey(key []byte) uint64 { + h := xxhash.Sum64(key) + if h == 0 { + h = 1 + } + return h +} + +func pow2(v uint64) uint64 { + for i := uint64(2); i < 1<<62; i *= 2 { + if i >= v { + return i + } + } + panic("unreachable") +} + +// TranslateFileReader implements a reader that continuously streams data from a store. +type TranslateFileReader struct { + ctx context.Context + store *TranslateFile + file *os.File + offset int64 + notify <-chan struct{} + + once sync.Once + closing chan struct{} +} + +// NewTranslateFileReader returns a new instance of TranslateFileReader. +func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { + return &TranslateFileReader{ + ctx: ctx, + store: store, + offset: offset, + notify: store.WriteNotify(), + closing: make(chan struct{}), + } +} + +// Open initializes the reader. +func (r *TranslateFileReader) Open() (err error) { + if r.file, err = os.Open(r.store.Path); err != nil { + return err + } + return nil +} + +// Close closes the underlying file reader. +func (r *TranslateFileReader) Close() error { + r.once.Do(func() { close(r.closing) }) + + if r.file != nil { + return r.file.Close() + } + return nil +} + +// Read reads the next section of the available data to p. This should always +// read from the start of an entry and read n bytes to the end of another entry. +func (r *TranslateFileReader) Read(p []byte) (n int, err error) { + for { + // Obtain notification channel before we check for new data. + notify := r.store.WriteNotify() + + // Exit if we can read one or more valid entries or we receive an error. + if n, err = r.read(p); n > 0 || err != nil { + return n, err + } + + // Wait for new data or close. + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + case <-r.closing: + return 0, ErrTranslateStoreReaderClosed + case <-r.store.Closing(): + return 0, ErrTranslateStoreClosed + case <-notify: + continue + } + } +} + +// read writes the bytes for zero or more valid entries to p. +func (r *TranslateFileReader) read(p []byte) (n int, err error) { + sz := r.store.Size() + + // Exit if there is no new data. + if sz < r.offset { + return 0, fmt.Errorf("pilosa: translate store reader past file size: sz=%d off=%d", sz, r.offset) + } else if sz == r.offset { + return 0, nil + } + + // Shorten buffer to maximum read size. + if max := sz - r.offset; int64(len(p)) > max { + p = p[:max] + } + + // Read data from file at offset. + // Limit the number of bytes read to only whole entries. + n, err = r.file.ReadAt(p, r.offset) + n = ValidLogEntriesLen(p[:n]) + r.offset += int64(n) + return n, err +} + +// Copied & modified from encoding/binary. +func UvarintSize(x uint64) (i int) { + for x >= 0x80 { + x >>= 7 + i++ + } + return i + 1 +} + +func hexdump(b []byte) { os.Stderr.Write([]byte(hex.Dump(b))) } diff --git a/translate_test.go b/translate_test.go new file mode 100644 index 000000000..c73853d69 --- /dev/null +++ b/translate_test.go @@ -0,0 +1,565 @@ +package pilosa_test + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pilosa/pilosa" +) + +func TestTranslateFile_TranslateColumn(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateColumn_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateColumnsToUint64("IDX0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_TranslateRow(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateRow_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_Reader(t *testing.T) { + t.Run("NoOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + rc, err := s.Reader(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // Read first entry. Should read 'entry length' (13) plus uvarint(size) (1) = 14b. + var entry pilosa.LogEntry + if n, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if n != 14 { + t.Fatalf("unexpected n: %d", n) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{1}, + Keys: [][]byte{[]byte("foo")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Read second entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + + // Write new entry. + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"xyz"}); err != nil { + t.Fatal(err) + } + + // Read new entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{2}, + Keys: [][]byte{[]byte("xyz")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Close reader and ensure it returns EOF. + if err := rc.Close(); err != nil { + t.Fatal(err) + } else if _, err := entry.ReadFrom(brc); err != pilosa.ErrTranslateStoreReaderClosed { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("WithOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Start offset after the first entry. + rc, err := s.Reader(context.Background(), 14) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // This should be the second entry. + var entry pilosa.LogEntry + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { + // Create a primary store that accepts writes. + primary := MustOpenTranslateFile() + defer primary.MustClose() + + // Create a replica that accepts writes from primary. + replica := NewTranslateFile() + replica.PrimaryTranslateStore = primary + if err := replica.Open(); err != nil { + t.Fatal(err) + } + defer replica.MustClose() + + // Write to the primary. + if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until writes appear. + if err := retryFor(2*time.Second, func() error { + // Verify that replica have received writes. + if value, err := replica.TranslateColumnToString("IDX0", 1); err != nil { + return err + } else if value != "foo" { + return fmt.Errorf("unexpected column 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil { + return err + } else if value != "bar" { + return fmt.Errorf("unexpected row 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected row 2 value: %s", value) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect primary store & write more values. + if err := primary.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected column 2 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect replica store & write more values. + if err := replica.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foobar"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 3); err != nil { + return err + } else if value != "foobar" { + return fmt.Errorf("unexpected column 3 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/batchSize) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} + +type TranslateFile struct { + *pilosa.TranslateFile +} + +func NewTranslateFile() *TranslateFile { + f, err := ioutil.TempFile("", "") + if err != nil { + panic(err) + } + f.Close() + + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s.Path = f.Name() + return s +} + +func MustOpenTranslateFile() *TranslateFile { + s := NewTranslateFile() + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +func (s *TranslateFile) Close() error { + defer os.Remove(s.Path) + return s.TranslateFile.Close() +} + +func (s *TranslateFile) MustClose() { + if err := s.Close(); err != nil { + panic(err) + } +} + +// Reopen closes the store and opens a new instance of it for the same path. +func (s *TranslateFile) Reopen() error { + prev := s.TranslateFile + if err := s.TranslateFile.Close(); err != nil { + return err + } + + s.TranslateFile = pilosa.NewTranslateFile() + s.Path = prev.Path + s.PrimaryTranslateStore = prev.PrimaryTranslateStore + if err := s.Open(); err != nil { + return err + } + return nil +} + +// retryFor executes fn every 100ms until d time passes or until fn return nil. +func retryFor(d time.Duration, fn func() error) (err error) { + timer, ticker := time.NewTimer(d), time.NewTicker(100*time.Millisecond) + defer timer.Stop() + defer ticker.Stop() + + for { + if err = fn(); err == nil { + return nil + } + + select { + case <-timer.C: + return err + case <-ticker.C: + } + } +} From 9a74763156e7a21eb3931f1c7dd3e71cd2a1c8d6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 12:40:59 -0500 Subject: [PATCH 09/10] move pilosa/test/client.go helpers into pilosa/client_test.go --- http/client_test.go | 28 +++++++++++++++++++++------- test/client.go | 35 ----------------------------------- 2 files changed, 21 insertions(+), 42 deletions(-) delete mode 100644 test/client.go diff --git a/http/client_test.go b/http/client_test.go index 851f1140e..185a3c378 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -148,10 +148,10 @@ func TestClient_MultiNode(t *testing.T) { hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() // Connect to each node to compare results. - client := make([]*test.Client, 3) - client[0] = test.MustNewClient(s[0].Host(), defaultClient) - client[1] = test.MustNewClient(s[1].Host(), defaultClient) - client[2] = test.MustNewClient(s[2].Host(), defaultClient) + client := make([]*Client, 3) + client[0] = MustNewClient(s[0].Host(), defaultClient) + client[1] = MustNewClient(s[1].Host(), defaultClient) + client[2] = MustNewClient(s[2].Host(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ @@ -231,7 +231,7 @@ func TestClient_Import(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -276,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, @@ -345,7 +345,7 @@ func TestClient_FragmentBlocks(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Retrieve blocks. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) if err != nil { t.Fatal(err) @@ -362,3 +362,17 @@ func TestClient_FragmentBlocks(t *testing.T) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } + +// Client represents a test wrapper for pilosa.Client. +type Client struct { + *http.InternalClient +} + +// MustNewClient returns a new instance of Client. Panic on error. +func MustNewClient(host string, h *gohttp.Client) *Client { + c, err := http.NewInternalClient(host, h) + if err != nil { + panic(err) + } + return &Client{InternalClient: c} +} diff --git a/test/client.go b/test/client.go deleted file mode 100644 index 34d5e17d0..000000000 --- a/test/client.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - gohttp "net/http" - - "github.com/pilosa/pilosa/http" -) - -// Client represents a test wrapper for pilosa.Client. -type Client struct { - *http.InternalClient -} - -// MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string, h *gohttp.Client) *Client { - c, err := http.NewInternalClient(host, h) - if err != nil { - panic(err) - } - return &Client{InternalClient: c} -} From 60dee04ed130c06e482471a11c06016e22072818 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Jun 2018 12:52:58 -0500 Subject: [PATCH 10/10] move pilosa/test/attr.go into pilosa/attr_test.go --- attr_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++--- test/attr.go | 90 ---------------------------------------------------- 2 files changed, 74 insertions(+), 95 deletions(-) delete mode 100644 test/attr.go diff --git a/attr_test.go b/attr_test.go index 8253c8e1d..0c848e2ff 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,15 +15,20 @@ package pilosa_test import ( + "io/ioutil" + "os" "reflect" + "runtime" + "sync" "testing" - "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) { // Ensure database returns a non-nil empty map if unset. func TestAttrStore_Attrs_Empty(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() if m, err := s.Attrs(100); err != nil { @@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) { // Ensure database can unset attributes if explicitly set to nil. func TestAttrStore_Attrs_Unset(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) { // Ensure attribute block checksums can be returned. func TestAttrStore_Blocks(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) { t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2]) } } + +// AttrStore represents a test wrapper for pilosa.AttrStore. +type AttrStore struct { + pilosa.AttrStore +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore(string) pilosa.AttrStore { + f, err := ioutil.TempFile("", "pilosa-attr-") + if err != nil { + panic(err) + } + f.Close() + os.Remove(f.Name()) + + return &AttrStore{boltdb.NewAttrStore(f.Name())} +} + +func BenchmarkAttrStore_Duplicate(b *testing.B) { + s := MustOpenAttrStore() + defer s.Close() + + // Set attributes. + const n = 5 + for i := 0; i < n; i++ { + if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + // Update attributes with an existing subset. + cpuN := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + for i := 0; i < cpuN; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < b.N/cpuN; j++ { + if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { + b.Fatal(err) + } + } + }() + } + wg.Wait() +} + +// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. +func MustOpenAttrStore() pilosa.AttrStore { + s := NewAttrStore("") + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +// Close closes the database and removes the underlying data. +func (s *AttrStore) Close() error { + defer os.RemoveAll(s.Path()) + return s.AttrStore.Close() +} diff --git a/test/attr.go b/test/attr.go deleted file mode 100644 index 16e8ff334..000000000 --- a/test/attr.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io/ioutil" - "os" - "runtime" - "sync" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/boltdb" -) - -// AttrStore represents a test wrapper for pilosa.AttrStore. -type AttrStore struct { - pilosa.AttrStore -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(string) pilosa.AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") - if err != nil { - panic(err) - } - f.Close() - os.Remove(f.Name()) - - return &AttrStore{boltdb.NewAttrStore(f.Name())} -} - -func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - const n = 5 - for i := 0; i < n; i++ { - if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - - // Update attributes with an existing subset. - cpuN := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - for i := 0; i < cpuN; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < b.N/cpuN; j++ { - if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { - b.Fatal(err) - } - } - }() - } - wg.Wait() -} - -// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() pilosa.AttrStore { - s := NewAttrStore("") - if err := s.Open(); err != nil { - panic(err) - } - return s -} - -// Close closes the database and removes the underlying data. -func (s *AttrStore) Close() error { - defer os.RemoveAll(s.Path()) - return s.AttrStore.Close() -}