scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into query-generator

Browse source
This commit is contained in:
tgruben 2018-06-14 10:55:42 -05:00 • committed by GitHub
commit bffa009fa7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 1092 additions and 1519 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
api.go
View file

@ -22,7 +22,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
@ -45,7 +44,6 @@ type API struct {
BroadcastHandler BroadcastHandler
StatusHandler StatusHandler
Cluster *Cluster
RemoteClient *http.Client
Logger Logger
}
@ -291,7 +289,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return ErrClusterDoesNotOwnSlice
}
@ -327,7 +325,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64)
return nil, errors.Wrap(err, "validating api method")
}
return api.Cluster.SliceNodes(indexName, slice), nil
return api.Cluster.sliceNodes(indexName, slice), nil
}
// MarshalFragment returns an object which can write the specified fragment's data
@ -681,7 +679,7 @@ func (api *API) LongQueryTime() time.Duration {
func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) {
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return nil, nil, ErrClusterDoesNotOwnSlice
}
@ -709,15 +707,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
return nil, nil, errors.Wrap(err, "validating api method")
}
oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator)
newNode = api.Cluster.NodeByID(id)
oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator)
newNode = api.Cluster.nodeByID(id)
if newNode == nil {
return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node")
}
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == api.LocalID() {
return oldNode, newNode, api.Cluster.SetCoordinator(newNode)
return oldNode, newNode, api.Cluster.setCoordinator(newNode)
}
// Send the set-coordinator message to new node.
@ -739,13 +737,13 @@ func (api *API) RemoveNode(id string) (*Node, error) {
return nil, errors.Wrap(err, "validating api method")
}
removeNode := api.Cluster.nodeByID(id)
removeNode := api.Cluster.unprotectedNodeByID(id)
if removeNode == nil {
return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove")
}
// Start the resize process (similar to NodeJoin)
err := api.Cluster.NodeLeave(removeNode)
err := api.Cluster.nodeLeave(removeNode)
if err != nil {
return removeNode, errors.Wrap(err, "calling node leave")
}
@ -758,7 +756,7 @@ func (api *API) ResizeAbort() error {
return errors.Wrap(err, "validating api method")
}
err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted)
err := api.Cluster.completeCurrentJob(resizeJobStateAborted)
return errors.Wrap(err, "complete current job")
}

26
attr.go
View file

@ -24,10 +24,10 @@ import (
// Attribute data type enum.
const (
AttrTypeString = 1
AttrTypeInt = 2
AttrTypeBool = 3
AttrTypeFloat = 4
attrTypeString = 1
attrTypeInt = 2
attrTypeBool = 3
attrTypeFloat = 4
)
// AttrStore represents an interface for handling row/column attributes.
@ -165,19 +165,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
pb := &internal.Attr{Key: key}
switch value := value.(type) {
case string:
pb.Type = AttrTypeString
pb.Type = attrTypeString
pb.StringValue = value
case float64:
pb.Type = AttrTypeFloat
pb.Type = attrTypeFloat
pb.FloatValue = value
case uint64:
pb.Type = AttrTypeInt
pb.Type = attrTypeInt
pb.IntValue = int64(value)
case int64:
pb.Type = AttrTypeInt
pb.Type = attrTypeInt
pb.IntValue = value
case bool:
pb.Type = AttrTypeBool
pb.Type = attrTypeBool
pb.BoolValue = value
}
return pb
@ -186,13 +186,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
// decodeAttr converts from an Attr internal representation to a key/value pair.
func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
switch attr.Type {
case AttrTypeString:
case attrTypeString:
return attr.Key, attr.StringValue
case AttrTypeInt:
case attrTypeInt:
return attr.Key, attr.IntValue
case AttrTypeBool:
case attrTypeBool:
return attr.Key, attr.BoolValue
case AttrTypeFloat:
case attrTypeFloat:
return attr.Key, attr.FloatValue
default:
return attr.Key, nil

90
broadcast.go
View file

@ -120,21 +120,21 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error {
// Broadcast message types.
const (
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
MessageTypeSetCoordinator
MessageTypeUpdateCoordinator
MessageTypeNodeState
MessageTypeRecalculateCaches
MessageTypeNodeEvent
messageTypeCreateSlice = iota
messageTypeCreateIndex
messageTypeDeleteIndex
messageTypeCreateField
messageTypeDeleteField
messageTypeCreateView
messageTypeDeleteView
messageTypeClusterStatus
messageTypeResizeInstruction
messageTypeResizeInstructionComplete
messageTypeSetCoordinator
messageTypeUpdateCoordinator
messageTypeNodeState
messageTypeRecalculateCaches
messageTypeNodeEvent
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -142,35 +142,35 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
var typ uint8
switch obj := m.(type) {
case *internal.CreateSliceMessage:
typ = MessageTypeCreateSlice
typ = messageTypeCreateSlice
case *internal.CreateIndexMessage:
typ = MessageTypeCreateIndex
typ = messageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = MessageTypeDeleteIndex
typ = messageTypeDeleteIndex
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
typ = messageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
typ = messageTypeDeleteField
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
typ = messageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
typ = messageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
typ = messageTypeClusterStatus
case *internal.ResizeInstruction:
typ = MessageTypeResizeInstruction
typ = messageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = MessageTypeResizeInstructionComplete
typ = messageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = MessageTypeSetCoordinator
typ = messageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = MessageTypeUpdateCoordinator
typ = messageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = MessageTypeNodeState
typ = messageTypeNodeState
case *internal.RecalculateCaches:
typ = MessageTypeRecalculateCaches
typ = messageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = MessageTypeNodeEvent
typ = messageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -187,35 +187,35 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
var m proto.Message
switch typ {
case MessageTypeCreateSlice:
case messageTypeCreateSlice:
m = &internal.CreateSliceMessage{}
case MessageTypeCreateIndex:
case messageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
case MessageTypeDeleteIndex:
case messageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
case MessageTypeCreateField:
case messageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
case messageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeCreateView:
case messageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
case messageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeClusterStatus:
case messageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:
case messageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
case MessageTypeResizeInstructionComplete:
case messageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
case MessageTypeSetCoordinator:
case messageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
case MessageTypeUpdateCoordinator:
case messageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
case MessageTypeNodeState:
case messageTypeNodeState:
m = &internal.NodeStateMessage{}
case MessageTypeRecalculateCaches:
case messageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
case MessageTypeNodeEvent:
case messageTypeNodeEvent:
m = &internal.NodeEventMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)

6
cache.go
View file

@ -27,8 +27,8 @@ import (
)
const (
// ThresholdFactor is used to calculate the threshold for new items entering the cache
ThresholdFactor = 1.1
// thresholdFactor is used to calculate the threshold for new items entering the cache
thresholdFactor = 1.1
)
// Cache represents a cache of counts.
@ -158,7 +158,7 @@ type RankCache struct {
func NewRankCache(maxEntries uint32) *RankCache {
return &RankCache{
maxEntries: maxEntries,
thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
entries: make(map[uint64]uint64),
stats: NopStatsClient,
}

426
cluster.go
View file

@ -21,7 +21,6 @@ import (
"hash/fnv"
"io/ioutil"
"math/rand"
"net/http"
"os"
"path/filepath"
"sort"
@ -49,14 +48,14 @@ const (
NodeStateLoading = "LOADING"
NodeStateReady = "READY"
// ResizeJob states.
ResizeJobStateRunning = "RUNNING"
// resizeJob states.
resizeJobStateRunning = "RUNNING"
// Final states.
ResizeJobStateDone = "DONE"
ResizeJobStateAborted = "ABORTED"
resizeJobStateDone = "DONE"
resizeJobStateAborted = "ABORTED"
ResizeJobActionAdd = "ADD"
ResizeJobActionRemove = "REMOVE"
resizeJobActionAdd = "ADD"
resizeJobActionRemove = "REMOVE"
)
// Node represents a node in the cluster.
@ -255,8 +254,8 @@ type Cluster struct {
joined bool
mu sync.RWMutex
jobs map[int64]*ResizeJob
currentJob *ResizeJob
jobs map[int64]*resizeJob
currentJob *resizeJob
// Close management
wg sync.WaitGroup
@ -264,9 +263,6 @@ type Cluster struct {
Logger Logger
//
RemoteClient *http.Client
InternalClient InternalClient
}
@ -279,7 +275,7 @@ func NewCluster() *Cluster {
EventReceiver: NopEventReceiver,
joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
jobs: make(map[int64]*ResizeJob),
jobs: make(map[int64]*resizeJob),
closing: make(chan struct{}),
joining: make(chan struct{}),
@ -289,27 +285,27 @@ func NewCluster() *Cluster {
}
}
// Coordinator returns the coordinator node.
func (c *Cluster) CoordinatorNode() *Node {
return c.nodeByID(c.Coordinator)
// coordinatorNode returns the coordinator node.
func (c *Cluster) coordinatorNode() *Node {
return c.unprotectedNodeByID(c.Coordinator)
}
// IsCoordinator is true if this node is the coordinator.
func (c *Cluster) IsCoordinator() bool {
// isCoordinator is true if this node is the coordinator.
func (c *Cluster) isCoordinator() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.isCoordinator()
return c.unprotectedIsCoordinator()
}
func (c *Cluster) isCoordinator() bool {
func (c *Cluster) unprotectedIsCoordinator() bool {
return c.Coordinator == c.Node.ID
}
// SetCoordinator tells the current node to become the
// setCoordinator tells the current node to become the
// Coordinator. In response to this, the current node
// will consider itself coordinator and update the other
// nodes with its version of Cluster.Status.
func (c *Cluster) SetCoordinator(n *Node) error {
func (c *Cluster) setCoordinator(n *Node) error {
c.mu.Lock()
// Verify that the new Coordinator value matches
// this node.
@ -319,7 +315,7 @@ func (c *Cluster) SetCoordinator(n *Node) error {
}
// Update IsCoordinator on all nodes (locally).
_ = c.updateCoordinator(n)
_ = c.unprotectedUpdateCoordinator(n)
c.mu.Unlock()
// Send the update coordinator message to all nodes.
err := c.Broadcaster.SendSync(
@ -334,17 +330,17 @@ func (c *Cluster) SetCoordinator(n *Node) error {
return c.Broadcaster.SendSync(c.Status())
}
// UpdateCoordinator updates this nodes Coordinator value as well as
// updateCoordinator updates this nodes Coordinator value as well as
// changing the corresponding node's IsCoordinator value
// to true, and sets all other nodes to false. Returns true if the value
// changed.
func (c *Cluster) UpdateCoordinator(n *Node) bool {
func (c *Cluster) updateCoordinator(n *Node) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.updateCoordinator(n)
return c.unprotectedUpdateCoordinator(n)
}
func (c *Cluster) updateCoordinator(n *Node) bool {
func (c *Cluster) unprotectedUpdateCoordinator(n *Node) bool {
var changed bool
if c.Coordinator != n.ID {
c.Coordinator = n.ID
@ -360,9 +356,9 @@ func (c *Cluster) updateCoordinator(n *Node) bool {
return changed
}
// AddNode adds a node to the Cluster and updates and saves the
// addNode adds a node to the Cluster and updates and saves the
// new topology.
func (c *Cluster) AddNode(node *Node) error {
func (c *Cluster) addNode(node *Node) error {
c.Logger.Printf("add node %s to cluster on %s", node, c.Node)
// If the node being added is the coordinator, set it for this node.
@ -387,9 +383,9 @@ func (c *Cluster) AddNode(node *Node) error {
return c.saveTopology()
}
// RemoveNode removes a node from the Cluster and updates and saves the
// removeNode removes a node from the Cluster and updates and saves the
// new topology.
func (c *Cluster) RemoveNode(node *Node) error {
func (c *Cluster) removeNode(node *Node) error {
// remove from cluster
if !c.removeNodeBasicSorted(node) {
return nil
@ -407,8 +403,8 @@ func (c *Cluster) RemoveNode(node *Node) error {
return c.saveTopology()
}
// NodeIDs returns the list of IDs in the cluster.
func (c *Cluster) NodeIDs() []string {
// nodeIDs returns the list of IDs in the cluster.
func (c *Cluster) nodeIDs() []string {
return Nodes(c.Nodes).IDs()
}
@ -472,9 +468,9 @@ func (c *Cluster) setState(state string) {
}
}
func (c *Cluster) SetNodeState(state string) error {
if c.IsCoordinator() {
return c.ReceiveNodeState(c.Node.ID, state)
func (c *Cluster) setNodeState(state string) error {
if c.isCoordinator() {
return c.receiveNodeState(c.Node.ID, state)
}
// Send node state to coordinator.
@ -484,18 +480,18 @@ func (c *Cluster) SetNodeState(state string) error {
}
c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator)
if err := c.sendTo(c.CoordinatorNode(), ns); err != nil {
if err := c.sendTo(c.coordinatorNode(), ns); err != nil {
return fmt.Errorf("sending node state error: err=%s", err)
}
return nil
}
// ReceiveNodeState sets node state in Topology in order for the
// receiveNodeState sets node state in Topology in order for the
// Coordinator to keep track of, during startup, which nodes have
// finished opening their Holder.
func (c *Cluster) ReceiveNodeState(nodeID string, state string) error {
if !c.IsCoordinator() {
func (c *Cluster) receiveNodeState(nodeID string, state string) error {
if !c.isCoordinator() {
return nil
}
@ -515,11 +511,6 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error {
return nil
}
// localNode is not being used.
//func (c *Cluster) localNode() *Node {
// return c.NodeByURI(c.URI)
//}
// Status returns the internal ClusterStatus representation.
func (c *Cluster) Status() *internal.ClusterStatus {
return &internal.ClusterStatus{
@ -529,14 +520,14 @@ func (c *Cluster) Status() *internal.ClusterStatus {
}
}
func (c *Cluster) NodeByID(id string) *Node {
func (c *Cluster) nodeByID(id string) *Node {
c.mu.RLock()
defer c.mu.RUnlock()
return c.nodeByID(id)
return c.unprotectedNodeByID(id)
}
// nodeByID returns a node reference by ID.
func (c *Cluster) nodeByID(id string) *Node {
// unprotectedNodeByID returns a node reference by ID.
func (c *Cluster) unprotectedNodeByID(id string) *Node {
for _, n := range c.Nodes {
if n.ID == id {
return n
@ -558,7 +549,7 @@ func (c *Cluster) nodePositionByID(nodeID string) int {
// addNodeBasicSorted adds a node to the cluster, sorted by id.
// Returns a pointer to the node and true if the node was added.
func (c *Cluster) addNodeBasicSorted(node *Node) bool {
n := c.nodeByID(node.ID)
n := c.unprotectedNodeByID(node.ID)
if n != nil {
return false
}
@ -645,7 +636,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost {
t := make(fragsByHost)
for i := uint64(0); i <= maxSlice; i++ {
nodes := c.SliceNodes(idx, i)
nodes := c.sliceNodes(idx, i)
for _, n := range nodes {
// for each field/view combination:
for field, views := range fieldViews {
@ -673,10 +664,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error)
if lenTo-lenFrom > 1 {
return "", "", errors.New("adding more than one node at a time is not supported")
}
action = ResizeJobActionAdd
action = resizeJobActionAdd
// Determine the node ID that is being added.
for _, n := range other.Nodes {
if c.nodeByID(n.ID) == nil {
if c.unprotectedNodeByID(n.ID) == nil {
nodeID = n.ID
break
}
@ -686,10 +677,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error)
if lenFrom-lenTo > 1 {
return "", "", errors.New("removing more than one node at a time is not supported")
}
action = ResizeJobActionRemove
action = resizeJobActionRemove
// Determine the node ID that is being removed.
for _, n := range c.Nodes {
if other.nodeByID(n.ID) == nil {
if other.unprotectedNodeByID(n.ID) == nil {
nodeID = n.ID
break
}
@ -721,7 +712,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
// If a node is being removed, however, then it will most likely
// require that a replica fragment be the source data.
srcCluster := c
if action == ResizeJobActionAdd && c.ReplicaN > 1 {
if action == resizeJobActionAdd && c.ReplicaN > 1 {
srcCluster = NewCluster()
srcCluster.Nodes = Nodes(c.Nodes).Clone()
srcCluster.Hasher = c.Hasher
@ -740,7 +731,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
srcNodesByFrag := make(map[frag]string)
for nodeID, frags := range srcFrags {
// If a node is being removed, don't consider it as a source.
if action == ResizeJobActionRemove && nodeID == diffNodeID {
if action == resizeJobActionRemove && nodeID == diffNodeID {
continue
}
for _, frag := range frags {
@ -772,7 +763,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
}
src := &internal.ResizeSource{
Node: EncodeNode(c.nodeByID(srcNodeID)),
Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)),
Index: idx.Name(),
Field: frag.field,
View: frag.view,
@ -786,8 +777,8 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
return m, nil
}
// Partition returns the partition that a slice belongs to.
func (c *Cluster) Partition(index string, slice uint64) int {
// partition returns the partition that a slice belongs to.
func (c *Cluster) partition(index string, slice uint64) int {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], slice)
@ -798,18 +789,18 @@ func (c *Cluster) Partition(index string, slice uint64) int {
return int(h.Sum64() % uint64(c.PartitionN))
}
// SliceNodes returns a list of nodes that own a fragment.
func (c *Cluster) SliceNodes(index string, slice uint64) []*Node {
return c.PartitionNodes(c.Partition(index, slice))
// sliceNodes returns a list of nodes that own a fragment.
func (c *Cluster) sliceNodes(index string, slice uint64) []*Node {
return c.partitionNodes(c.partition(index, slice))
}
// OwnsSlice returns true if a host owns a fragment.
func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool {
return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID)
// ownsSlice returns true if a host owns a fragment.
func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool {
return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID)
}
// PartitionNodes returns a list of nodes that own a partition.
func (c *Cluster) PartitionNodes(partitionID int) []*Node {
// partitionNodes returns a list of nodes that own a partition.
func (c *Cluster) partitionNodes(partitionID int) []*Node {
// Default replica count to between one and the number of nodes.
// The replica count can be zero if there are no nodes.
replicaN := c.ReplicaN
@ -831,27 +822,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node {
return nodes
}
// OwnsSlices finds the set of slices owned by the node per Index
func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 {
// containsSlices is like OwnsSlices, but it includes replicas.
func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 {
var slices []uint64
for i := uint64(0); i <= maxSlice; i++ {
p := c.Partition(index, i)
// Determine primary owner node.
nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes))
if c.Nodes[nodeIndex].URI == uri {
slices = append(slices, i)
}
}
return slices
}
// ContainsSlices is like OwnsSlices, but it includes replicas.
func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 {
var slices []uint64
for i := uint64(0); i <= maxSlice; i++ {
p := c.Partition(index, i)
p := c.partition(index, i)
// Determine the nodes for partition.
nodes := c.PartitionNodes(p)
nodes := c.partitionNodes(p)
for _, n := range nodes {
if n.ID == node.ID {
slices = append(slices, i)
@ -884,7 +861,7 @@ func (h *jmphasher) Hash(key uint64, n int) int {
return int(b)
}
func (c *Cluster) Open() error {
func (c *Cluster) open() error {
// Cluster always comes up in state STARTING until cluster membership is determined.
c.state = ClusterStateStarting
@ -896,7 +873,7 @@ func (c *Cluster) Open() error {
c.ID = c.Topology.ClusterID
// Only the coordinator needs to consider the .topology file.
if c.IsCoordinator() {
if c.isCoordinator() {
err := c.considerTopology()
if err != nil {
return fmt.Errorf("considerTopology: %v", err)
@ -904,7 +881,7 @@ func (c *Cluster) Open() error {
}
// Add the local node to the cluster.
err := c.AddNode(c.Node)
err := c.addNode(c.Node)
if err != nil {
return errors.Wrap(err, "adding local node")
}
@ -920,7 +897,7 @@ func (c *Cluster) Open() error {
}
// If not coordinator then wait for ClusterStatus from coordinator.
if !c.IsCoordinator() {
if !c.isCoordinator() {
// In the case where a node has been restarted and memberlist has
// not had enough time to determine the node went down/up, then
// the coorninator needs to be alerted that this node is back up
@ -945,7 +922,7 @@ func (c *Cluster) Open() error {
return nil
}
func (c *Cluster) Close() error {
func (c *Cluster) close() error {
// Notify goroutines of closing and wait for completion.
close(c.closing)
c.wg.Wait()
@ -962,14 +939,14 @@ func (c *Cluster) markAsJoined() {
}
func (c *Cluster) needTopologyAgreement() bool {
return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs())
return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs())
}
func (c *Cluster) haveTopologyAgreement() bool {
if c.Static {
return true
}
return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs())
return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs())
}
func (c *Cluster) allNodesReady() bool {
@ -999,10 +976,10 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
// channel, which is not consumed until the code below.
var eg errgroup.Group
eg.Go(func() error {
return j.Run()
return j.run()
})
// Wait for the ResizeJob to finish or be aborted.
// Wait for the resizeJob to finish or be aborted.
c.Logger.Printf("wait for jobResult")
jobResult := <-j.result
@ -1013,18 +990,18 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
c.Logger.Printf("received jobResult: %s", jobResult)
switch jobResult {
case ResizeJobStateDone:
if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil {
case resizeJobStateDone:
if err := c.completeCurrentJob(resizeJobStateDone); err != nil {
return errors.Wrap(err, "completing finished job")
}
// Add/remove uri to/from the cluster.
if j.action == ResizeJobActionRemove {
return c.RemoveNode(nodeAction.node)
} else if j.action == ResizeJobActionAdd {
return c.AddNode(nodeAction.node)
if j.action == resizeJobActionRemove {
return c.removeNode(nodeAction.node)
} else if j.action == resizeJobActionAdd {
return c.addNode(nodeAction.node)
}
case ResizeJobStateAborted:
if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil {
case resizeJobStateAborted:
if err := c.completeCurrentJob(resizeJobStateAborted); err != nil {
return errors.Wrap(err, "completing aborted job")
}
}
@ -1045,64 +1022,63 @@ func (c *Cluster) sendTo(node *Node, msg proto.Message) error {
return nil
}
// ListenForJoins handles cluster-resize events.
func (c *Cluster) ListenForJoins() {
c.wg.Add(1)
go func() { defer c.wg.Done(); c.listenForJoins() }()
}
// listenForJoins handles cluster-resize events.
func (c *Cluster) listenForJoins() {
// When a cluster starts, the state is STARTING.
// We first want to wait for at least one node to join.
// Then we want to clear out the joiningLeavingNodes queue (buffered channel).
// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
// We use a bool `setNormal` to indicate when at least one node has joined.
c.wg.Add(1)
go func() {
defer c.wg.Done()
var setNormal bool
// When a cluster starts, the state is STARTING.
// We first want to wait for at least one node to join.
// Then we want to clear out the joiningLeavingNodes queue (buffered channel).
// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
// We use a bool `setNormal` to indicate when at least one node has joined.
var setNormal bool
for {
for {
// Handle all pending joins before changing state back to NORMAL.
select {
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
// Handle all pending joins before changing state back to NORMAL.
select {
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
continue
default:
}
// Only change state to NORMAL if we have successfully added at least one host.
if setNormal {
// Put the cluster back to state NORMAL and broadcast.
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
}
// Wait for a joining host or a close.
select {
case <-c.closing:
return
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
continue
}
setNormal = true
continue
default:
}
// Only change state to NORMAL if we have successfully added at least one host.
if setNormal {
// Put the cluster back to state NORMAL and broadcast.
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
}
// Wait for a joining host or a close.
select {
case <-c.closing:
return
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
continue
}
}
}()
}
// generateResizeJob creates a new ResizeJob based on the new node being
// added/removed. It also saves a reference to the ResizeJob in the `jobs` map
// generateResizeJob creates a new resizeJob based on the new node being
// added/removed. It also saves a reference to the resizeJob in the `jobs` map
// for future lookup by JobID.
func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) {
c.Logger.Printf("generateResizeJob: %v", nodeAction)
c.mu.Lock()
defer c.mu.Unlock()
@ -1111,7 +1087,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
if err != nil {
return nil, errors.Wrap(err, "generating job")
}
c.Logger.Printf("generated ResizeJob: %d", j.ID)
c.Logger.Printf("generated resizeJob: %d", j.ID)
// Save job in jobs map for future reference.
c.jobs[j.ID] = j
@ -1125,12 +1101,12 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
return j, nil
}
// generateResizeJobByAction returns a ResizeJob with instructions based on
// generateResizeJobByAction returns a resizeJob with instructions based on
// the difference between Cluster and a new Cluster with/without uri.
// Broadcaster is associated to the ResizeJob here for use in broadcasting
// Broadcaster is associated to the resizeJob here for use in broadcasting
// the resize instructions to other nodes in the cluster.
func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) {
j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action)
func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) {
j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action)
j.Broadcaster = c.Broadcaster
// toCluster is a clone of Cluster with the new node added/removed for comparison.
@ -1139,9 +1115,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
toCluster.Hasher = c.Hasher
toCluster.PartitionN = c.PartitionN
toCluster.ReplicaN = c.ReplicaN
if nodeAction.action == ResizeJobActionRemove {
if nodeAction.action == resizeJobActionRemove {
toCluster.removeNodeBasicSorted(nodeAction.node)
} else if nodeAction.action == ResizeJobActionAdd {
} else if nodeAction.action == resizeJobActionAdd {
toCluster.addNodeBasicSorted(nodeAction.node)
}
@ -1172,8 +1148,8 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
}
instr := &internal.ResizeInstruction{
JobID: j.ID,
Node: EncodeNode(toCluster.nodeByID(id)),
Coordinator: EncodeNode(c.CoordinatorNode()),
Node: EncodeNode(toCluster.unprotectedNodeByID(id)),
Coordinator: EncodeNode(c.coordinatorNode()),
Sources: sources,
Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node.
ClusterStatus: c.Status(),
@ -1184,28 +1160,28 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
return j, nil
}
// CompleteCurrentJob sets the state of the current ResizeJob
// completeCurrentJob sets the state of the current resizeJob
// then removes the pointer to currentJob.
func (c *Cluster) CompleteCurrentJob(state string) error {
func (c *Cluster) completeCurrentJob(state string) error {
c.mu.Lock()
defer c.mu.Unlock()
if !c.isCoordinator() {
if !c.unprotectedIsCoordinator() {
return ErrNodeNotCoordinator
}
if c.currentJob == nil {
return ErrResizeNotRunning
}
c.currentJob.SetState(state)
c.currentJob.setState(state)
c.currentJob = nil
return nil
}
// FollowResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// followResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) error {
c.Logger.Printf("follow resize instruction on %s", c.Node.ID)
// Make sure the cluster status on this node agrees with the Coordinator
// before attempting a resize.
if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil {
if err := c.mergeClusterStatus(instr.ClusterStatus); err != nil {
return errors.Wrap(err, "merging cluster status")
}
@ -1297,13 +1273,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
return nil
}
func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
func (c *Cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
j := c.Job(complete.JobID)
j := c.job(complete.JobID)
// Abort the job if an error exists in the complete object.
if complete.Error != "" {
j.result <- ResizeJobStateAborted
j.result <- resizeJobStateAborted
return errors.New(complete.Error)
}
@ -1311,29 +1287,27 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc
defer j.mu.Unlock()
if j.isComplete() {
return fmt.Errorf("ResizeJob %d is no longer running", j.ID)
return fmt.Errorf("resize job %d is no longer running", j.ID)
}
// Mark host complete.
j.IDs[complete.Node.ID] = true
if !j.nodesArePending() {
j.result <- ResizeJobStateDone
j.result <- resizeJobStateDone
}
return nil
}
// Job returns a ResizeJob by id.
func (c *Cluster) Job(id int64) *ResizeJob {
// job returns a resizeJob by id.
func (c *Cluster) job(id int64) *resizeJob {
c.mu.RLock()
defer c.mu.RUnlock()
return c.job(id)
return c.jobs[id]
}
func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] }
type ResizeJob struct {
type resizeJob struct {
ID int64
IDs map[string]bool
Instructions []*internal.ResizeInstruction
@ -1348,15 +1322,15 @@ type ResizeJob struct {
Logger Logger
}
// NewResizeJob returns a new instance of ResizeJob.
func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
// newResizeJob returns a new instance of resizeJob.
func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob {
// Build a map of uris to track their resize status.
// The value for a node will be set to true after that node
// has indicated that it has completed all resize instructions.
ids := make(map[string]bool)
if action == ResizeJobActionRemove {
if action == resizeJobActionRemove {
for _, n := range existingNodes {
// Exclude the removed node from the map.
if n.ID == node.ID {
@ -1364,7 +1338,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
}
ids[n.ID] = false
}
} else if action == ResizeJobActionAdd {
} else if action == resizeJobActionAdd {
for _, n := range existingNodes {
ids[n.ID] = false
}
@ -1372,7 +1346,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
ids[node.ID] = false
}
return &ResizeJob{
return &resizeJob{
ID: rand.Int63(),
IDs: ids,
action: action,
@ -1381,50 +1355,40 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
}
}
func (j *ResizeJob) State() string {
j.mu.RLock()
defer j.mu.RUnlock()
return j.state
}
func (j *ResizeJob) SetState(state string) {
func (j *resizeJob) setState(state string) {
j.mu.Lock()
j.setState(state)
if j.state == "" || j.state == resizeJobStateRunning {
j.state = state
}
j.mu.Unlock()
}
func (j *ResizeJob) setState(state string) {
if j.state == "" || j.state == ResizeJobStateRunning {
j.state = state
}
}
// Run distributes ResizeInstructions.
func (j *ResizeJob) Run() error {
j.Logger.Printf("run ResizeJob")
// run distributes ResizeInstructions.
func (j *resizeJob) run() error {
j.Logger.Printf("run resizeJob")
// Set job state to RUNNING.
j.SetState(ResizeJobStateRunning)
j.setState(resizeJobStateRunning)
// Job can be considered done in the case where it doesn't require any action.
if !j.nodesArePending() {
j.Logger.Printf("ResizeJob contains no pending tasks; mark as done")
j.result <- ResizeJobStateDone
j.Logger.Printf("resizeJob contains no pending tasks; mark as done")
j.result <- resizeJobStateDone
return nil
}
j.Logger.Printf("distribute tasks for ResizeJob")
j.Logger.Printf("distribute tasks for resizeJob")
err := j.distributeResizeInstructions()
if err != nil {
j.result <- ResizeJobStateAborted
j.result <- resizeJobStateAborted
return errors.Wrap(err, "distributing instructions")
}
return nil
}
// isComplete return true if the job is any one of several completion states.
func (j *ResizeJob) isComplete() bool {
func (j *resizeJob) isComplete() bool {
switch j.state {
case ResizeJobStateDone, ResizeJobStateAborted:
case resizeJobStateDone, resizeJobStateAborted:
return true
default:
return false
@ -1432,7 +1396,7 @@ func (j *ResizeJob) isComplete() bool {
}
// nodesArePending returns true if any node is still working on the resize.
func (j *ResizeJob) nodesArePending() bool {
func (j *resizeJob) nodesArePending() bool {
for _, complete := range j.IDs {
if !complete {
return true
@ -1441,9 +1405,9 @@ func (j *ResizeJob) nodesArePending() bool {
return false
}
func (j *ResizeJob) distributeResizeInstructions() error {
func (j *resizeJob) distributeResizeInstructions() error {
j.Logger.Printf("distributeResizeInstructions for job %d", j.ID)
// Loop through the ResizeInstructions in ResizeJob and send to each host.
// Loop through the ResizeInstructions in resizeJob and send to each host.
for _, instr := range j.Instructions {
// Because the node may not be in the cluster yet, create
// a dummy node object to use in the SendTo() method.
@ -1659,7 +1623,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error {
case NodeJoin:
c.Logger.Printf("received NodeJoin event: %v", e)
// Ignore the event if this is not the coordinator.
if !c.IsCoordinator() {
if !c.isCoordinator() {
return nil
}
return c.nodeJoin(e.Node)
@ -1681,7 +1645,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
return errors.New(err)
}
if err := c.AddNode(node); err != nil {
if err := c.addNode(node); err != nil {
return errors.Wrap(err, "adding node for agreement")
}
@ -1711,13 +1675,13 @@ func (c *Cluster) nodeJoin(node *Node) error {
// If the cluster already contains the node, just send it the cluster status.
// This is useful in the case where a node is restarted or temporarily leaves
// the cluster.
if node := c.nodeByID(node.ID); node != nil {
if node := c.unprotectedNodeByID(node.ID); node != nil {
return c.sendTo(node, c.Status())
}
// If the holder does not yet contain data, go ahead and add the node.
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.AddNode(node); err != nil {
if err := c.addNode(node); err != nil {
return errors.Wrap(err, "adding node")
}
return c.setStateAndBroadcast(ClusterStateNormal)
@ -1730,16 +1694,16 @@ func (c *Cluster) nodeJoin(node *Node) error {
if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
return errors.Wrap(err, "broadcasting state")
}
c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd}
c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd}
return nil
}
// NodeLeave initiates the removal of a node from the cluster.
func (c *Cluster) NodeLeave(node *Node) error {
// nodeLeave initiates the removal of a node from the cluster.
func (c *Cluster) nodeLeave(node *Node) error {
// Refuse the request if this is not the coordinator.
if !c.IsCoordinator() {
return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID)
if !c.isCoordinator() {
return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID)
}
if c.State() != ClusterStateNormal {
@ -1747,7 +1711,7 @@ func (c *Cluster) NodeLeave(node *Node) error {
}
// Ensure that node is in the cluster.
if c.nodeByID(node.ID) == nil {
if c.unprotectedNodeByID(node.ID) == nil {
return fmt.Errorf("Node is not a member of the cluster: %s", node.ID)
}
@ -1757,18 +1721,12 @@ func (c *Cluster) NodeLeave(node *Node) error {
}
// See if resize job can be generated
_, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove})
if err != nil {
if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil {
return errors.Wrap(err, "generating job")
}
return c.nodeLeave(node)
}
func (c *Cluster) nodeLeave(node *Node) error {
// Get the actual node in the local cluster.
n := c.nodeByID(node.ID)
n := c.unprotectedNodeByID(node.ID)
// Don't do anything else if the cluster doesn't contain the node.
if n == nil {
@ -1777,7 +1735,7 @@ func (c *Cluster) nodeLeave(node *Node) error {
// If the holder does not yet contain data, go ahead and remove the node.
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.RemoveNode(n); err != nil {
if err := c.removeNode(n); err != nil {
return errors.Wrap(err, "removing node")
}
return c.setStateAndBroadcast(ClusterStateNormal)
@ -1790,17 +1748,17 @@ func (c *Cluster) nodeLeave(node *Node) error {
if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
return errors.Wrap(err, "broadcasting state")
}
c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove}
c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove}
return nil
}
func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
c.Logger.Printf("merge cluster status: %v", cs)
// Ignore status updates from self (coordinator).
if c.isCoordinator() {
if c.unprotectedIsCoordinator() {
return nil
}
@ -1811,7 +1769,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
// Add all nodes from the coordinator.
for _, node := range officialNodes {
if err := c.AddNode(node); err != nil {
if err := c.addNode(node); err != nil {
return errors.Wrap(err, "adding node")
}
}
@ -1832,7 +1790,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
}
for _, nodeID := range nodeIDsToRemove {
if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil {
if err := c.removeNode(c.unprotectedNodeByID(nodeID)); err != nil {
return errors.Wrap(err, "removing node")
}
}

470
cluster_internal_test.go
View file

@ -15,11 +15,15 @@
package pilosa
import (
"bytes"
"io/ioutil"
"math/rand"
"reflect"
"strings"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/internal"
)
@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) {
{
existingNodes: []*Node{node0, node1},
node: node2,
action: ResizeJobActionAdd,
action: resizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: ResizeJobActionRemove,
action: resizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := NewResizeJob(test.existingNodes, test.node, test.action)
actual := newResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) {
}
}
}
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0], _ = quick.Value(reflect.TypeOf(""), rand)
values[1] = reflect.ValueOf(uint64(rand.Uint32()))
values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// Ensure the hasher can hash correctly.
func TestHasher(t *testing.T) {
for _, tt := range []struct {
key uint64
bucket []int
}{
// Generated from the reference C++ code
{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.containsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
// NEXT: move this test to internal and unexport IsCoordinator
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.isCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.isCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.addNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.nodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.Holder.Field("i", "f")
node0View := node0Field.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.Holder.Field("i", "f")
node1View := node1Field.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.updateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.updateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

494
cluster_test.go
View file

@ -1,494 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0], _ = quick.Value(reflect.TypeOf(""), rand)
values[1] = reflect.ValueOf(uint64(rand.Uint32()))
values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// Ensure the hasher can hash correctly.
func TestHasher(t *testing.T) {
for _, tt := range []struct {
key uint64
bucket []int
}{
// Generated from the reference C++ code
{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := NewTestCluster(5)
slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.IsCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.IsCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.AddNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.NodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.Holder.Field("i", "f")
node0View := node0Field.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.Holder.Field("i", "f")
node1View := node1Field.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.UpdateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.UpdateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

26
executor.go
View file

@ -25,13 +25,13 @@ import (
"github.com/pkg/errors"
)
// DefaultField is the field used if one is not specified.
// defaultField is the field used if one is not specified.
const (
DefaultField = "general"
defaultField = "general"
// MinThreshold is the lowest count to use in a Top-N operation when
// defaultMinThreshold is the lowest count to use in a Top-N operation when
// looking for additional id/count pairs.
MinThreshold = 1
defaultMinThreshold = 1
columnLabel = "col"
rowLabel = "row"
@ -588,7 +588,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
// Set default field.
if field == "" {
field = DefaultField
field = defaultField
}
f := e.Holder.Fragment(index, field, ViewStandard, slice)
@ -597,7 +597,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
}
if minThreshold <= 0 {
minThreshold = MinThreshold
minThreshold = defaultMinThreshold
}
if tanimotoThreshold > 100 {
@ -646,7 +646,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
// Fetch field & row label based on argument.
field, _ := c.Args["field"].(string)
if field == "" {
field = DefaultField
field = defaultField
}
f := e.Holder.Field(index, field)
if f == nil {
@ -700,7 +700,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Parse field, use default if unset.
field, _ := c.Args["field"].(string)
if field == "" {
field = DefaultField
field = defaultField
}
// Retrieve column label.
@ -752,7 +752,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, field, view, slice)
if f == nil {
continue
@ -1002,7 +1002,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(index, slice) {
for _, node := range e.Cluster.sliceNodes(index, slice) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.ClearBit(view, rowID, colID, nil)
@ -1078,7 +1078,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(index, slice) {
for _, node := range e.Cluster.sliceNodes(index, slice) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.SetBit(view, rowID, colID, timestamp)
@ -1414,7 +1414,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m
loop:
for _, slice := range slices {
for _, node := range e.Cluster.SliceNodes(index, slice) {
for _, node := range e.Cluster.sliceNodes(index, slice) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], slice)
continue loop
@ -1444,7 +1444,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
if !opt.Remote {
nodes = Nodes(e.Cluster.Nodes).Clone()
} else {
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.

66
executor_test.go
View file

@ -38,7 +38,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
@ -87,7 +87,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
@ -113,7 +113,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, 4)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
@ -127,7 +127,7 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "general", 10, 1)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil {
t.Fatalf("Empty Difference query should give error, but got %v", res)
}
@ -145,7 +145,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
@ -158,7 +158,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil {
t.Fatalf("Empty Intersect query should give error, but got %v", res)
}
@ -175,7 +175,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
@ -189,7 +189,7 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "general", 10, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
@ -208,7 +208,7 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
@ -224,7 +224,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
@ -240,7 +240,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if n := hldr.Row("i", "f", 11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
@ -284,7 +284,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
// Set bsiGroup values.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil {
@ -322,21 +322,21 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnBSIGroupValue", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType {
t.Fatalf("unexpected error: %s", err)
}
@ -359,7 +359,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two attrs on f/10.
// Also set attrs on other bitmaps and fields to test isolation.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
@ -385,7 +385,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
func TestExecutor_Execute_TopN(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
@ -437,7 +437,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr.SetBit("i", "f", 1, SliceWidth)
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -471,7 +471,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr.SetBit("i", "f", 4, 3*SliceWidth+1)
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -506,7 +506,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache()
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -530,7 +530,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -553,7 +553,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -567,7 +567,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
func TestExecutor_Execute_MinMax(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
@ -662,7 +662,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
@ -733,7 +733,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Create index.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -775,7 +775,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
func TestExecutor_Execute_Range(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
@ -955,7 +955,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Ensure a remote query can return a row.
func TestExecutor_Execute_Remote_Row(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
@ -1003,7 +1003,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
@ -1038,7 +1038,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
// Create secondary server and update second cluster node.
@ -1090,7 +1090,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
// Create secondary server and update second cluster node.
@ -1144,7 +1144,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Ensure a remote query can return a top-n query.
func TestExecutor_Execute_Remote_TopN(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
@ -1213,7 +1213,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.MaxWritesPerRequest = 3
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites {
t.Fatalf("unexpected error: %s", err)
@ -1229,7 +1229,7 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
targetAttrs := map[string]interface{}{
"foo": "bar",
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// SetColumnAttrs call should exclude the field attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil)

24
field.go
View file

@ -33,10 +33,10 @@ import (
const (
DefaultFieldType = FieldTypeSet
DefaultCacheType = CacheTypeRanked
defaultCacheType = CacheTypeRanked
// Default ranked field cache
DefaultCacheSize = 50000
defaultCacheSize = 50000
)
// Field types.
@ -82,7 +82,7 @@ func OptFieldFieldOptions(o FieldOptions) FieldOption {
// NewField returns a new instance of field.
func NewField(path, index, name string, opts ...FieldOption) (*Field, error) {
err := ValidateName(name)
err := validateName(name)
if err != nil {
return nil, err
}
@ -101,8 +101,8 @@ func NewField(path, index, name string, opts ...FieldOption) (*Field, error) {
options: FieldOptions{
Type: DefaultFieldType,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
CacheType: defaultCacheType,
CacheSize: defaultCacheSize,
},
Logger: NopLogger,
@ -645,7 +645,7 @@ func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) {
// SetBit sets a bit on a view within the field.
func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
if !IsValidView(name) {
if !isValidView(name) {
return false, ErrInvalidView
}
@ -668,7 +668,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
}
// If a timestamp is specified then set bits across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, errors.Wrapf(err, "creating view %s", subname)
@ -687,7 +687,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
// ClearBit clears a bit within the field.
func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
if !IsValidView(name) {
if !isValidView(name) {
return false, ErrInvalidView
}
@ -710,7 +710,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
}
// If a timestamp is specified then clear bits across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, errors.Wrapf(err, "creating view %s", subname)
@ -899,7 +899,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
if timestamp == nil {
standard = []string{ViewStandard}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
standard = viewsByTime(ViewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, ViewStandard)
@ -1233,8 +1233,8 @@ const (
CacheTypeNone = "none"
)
// IsValidCacheType returns true if v is a valid cache type.
func IsValidCacheType(v string) bool {
// isValidCacheType returns true if v is a valid cache type.
func isValidCacheType(v string) bool {
switch v {
case CacheTypeLRU, CacheTypeRanked, CacheTypeNone:
return true

40
fragment.go
View file

@ -25,7 +25,6 @@ import (
"hash"
"io"
"io/ioutil"
"net/http"
"os"
"sort"
"sync"
@ -48,22 +47,20 @@ const (
// SliceWidth is the number of column IDs in a slice.
SliceWidth = 1048576
// SnapshotExt is the file extension used for an in-process snapshot.
SnapshotExt = ".snapshotting"
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
// CopyExt is the file extension used for the temp file used while copying.
CopyExt = ".copying"
// copyExt is the file extension used for the temp file used while copying.
copyExt = ".copying"
// CacheExt is the file extension for persisted cache ids.
CacheExt = ".cache"
// cacheExt is the file extension for persisted cache ids.
cacheExt = ".cache"
// HashBlockSize is the number of rows in a merkle hash block.
HashBlockSize = 100
)
const (
// DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
DefaultFragmentMaxOpN = 2000
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
defaultFragmentMaxOpN = 2000
)
// Fragment represents the intersection of a field and slice in an index.
@ -120,18 +117,18 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment {
field: field,
view: view,
slice: slice,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
CacheType: defaultCacheType,
CacheSize: defaultCacheSize,
Logger: NopLogger,
MaxOpN: DefaultFragmentMaxOpN,
MaxOpN: defaultFragmentMaxOpN,
stats: NopStatsClient,
}
}
// cachePath returns the path to the fragment's cache data.
func (f *Fragment) cachePath() string { return f.path + CacheExt }
func (f *Fragment) cachePath() string { return f.path + cacheExt }
// Open opens the underlying storage.
func (f *Fragment) Open() error {
@ -1432,7 +1429,7 @@ func (f *Fragment) snapshot() error {
defer track(start, completeMessage, f.stats, f.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
snapshotPath := f.path + snapshotExt
file, err := os.Create(snapshotPath)
if err != nil {
return fmt.Errorf("create snapshot file: %s", err)
@ -1636,7 +1633,7 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) {
func (f *Fragment) readStorageFromArchive(r io.Reader) error {
// Create a temporary file to copy into.
path := f.path + CopyExt
path := f.path + copyExt
file, err := os.Create(path)
if err != nil {
return errors.Wrap(err, "creating directory")
@ -1719,9 +1716,8 @@ func (h *blockHasher) WriteValue(v uint64) {
type FragmentSyncer struct {
Fragment *Fragment
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Node *Node
Cluster *Cluster
Closing <-chan struct{}
}
@ -1740,7 +1736,7 @@ func (s *FragmentSyncer) isClosing() bool {
// then merges any blocks which have differences.
func (s *FragmentSyncer) syncFragment() error {
// Determine replica set.
nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice)
nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice)
if len(nodes) == 1 {
return nil
}
@ -1821,7 +1817,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Read pairs from each remote block.
var uris []*URI
var pairSets []pairSet
for _, node := range s.Cluster.SliceNodes(f.index, f.slice) {
for _, node := range s.Cluster.sliceNodes(f.index, f.slice) {
if s.Node.ID == node.ID {
continue
}

2
fragment_internal_test.go
View file

@ -1245,7 +1245,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string)
file.Close()
if cacheType == "" {
cacheType = DefaultCacheType
cacheType = defaultCacheType
}
f := NewFragment(file.Name(), index, field, view, slice)

21
gossip/gossip.go
View file

@ -18,6 +18,7 @@ import (
"fmt"
"io/ioutil"
"log"
"net"
"strconv"
"strings"
"sync"
@ -213,7 +214,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
conf.AdvertiseAddr = hostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
@ -580,3 +581,21 @@ type Config struct {
Nodes int `toml:"nodes"`
ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
}
// hostToIP converts host to an IP4 address based on net.LookupIP().
func hostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}

6
handler.go
View file

@ -2,7 +2,7 @@ package pilosa
import (
"encoding/json"
"net/http"
"net"
)
// QueryRequest represent a request to process a query.
@ -61,13 +61,13 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
}
type Handler interface {
http.Handler
Serve(ln net.Listener, closing <-chan struct{})
GetAPI() *API
}
type NopHandler struct{}
func (n *NopHandler) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {}
func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {}
func (n *NopHandler) GetAPI() *API {
return nil

25
holder.go
View file

@ -18,7 +18,6 @@ import (
"context"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
@ -34,8 +33,8 @@ import (
)
const (
// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
DefaultCacheFlushInterval = 1 * time.Minute
// defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
defaultCacheFlushInterval = 1 * time.Minute
// FileLimit is the maximum open file limit (ulimit -n) to automatically set.
FileLimit = 262144 // (512^2)
@ -84,7 +83,7 @@ func NewHolder() *Holder {
NewAttrStore: NewNopAttrStore,
CacheFlushInterval: DefaultCacheFlushInterval,
CacheFlushInterval: defaultCacheFlushInterval,
Logger: NopLogger,
}
@ -563,9 +562,8 @@ func (h *Holder) logStartup() error {
type HolderSyncer struct {
Holder *Holder
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Node *Node
Cluster *Cluster
// Stats
Stats StatsClient
@ -619,7 +617,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) {
if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) {
continue
}
@ -755,11 +753,10 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err
// Sync fragments together.
fs := FragmentSyncer{
Fragment: frag,
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
RemoteClient: s.RemoteClient,
Fragment: frag,
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
}
if err := fs.syncFragment(); err != nil {
return errors.Wrap(err, "syncing fragment")
@ -799,7 +796,7 @@ func (c *HolderCleaner) CleanHolder() error {
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, field := range index.Fields() {

16
holder_test.go
View file

@ -362,7 +362,6 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
client := http.GetHTTPClient(nil)
httpClient := http.NewInternalClientFromURI(uri, client)
cluster.InternalClient = httpClient
cluster.RemoteClient = client
// Create a local holder.
hldr0 := test.MustOpenHolder()
@ -383,7 +382,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create fields on nodes.
@ -419,11 +418,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Set up syncer.
syncer := pilosa.HolderSyncer{
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: http.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
Stats: pilosa.NopStatsClient,
}
if err := syncer.SyncHolder(); err != nil {
@ -456,7 +454,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Ensure holder can clean up orphaned fragments.
func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster := test.NewCluster(2)
cluster := pilosa.NewTestCluster(2)
// Create a local holder.
hldr0 := test.MustOpenHolder()
@ -465,7 +463,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
// Create fields on nodes.
for _, hldr := range []*test.Holder{hldr0} {

11
http/client_test.go
View file

@ -87,10 +87,17 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())`
owns := [][]uint64{
{1, 3, 4, 8, 10, 13, 17, 19},
{2, 5, 7, 11, 12, 14, 18},
{0, 6, 9, 15, 16, 20},
}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
for _, ownNum := range owns[i] {
if ownNum == num {
ownsNum = true
break

12
http/handler.go
View file

@ -117,6 +117,18 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
return handler, nil
}
func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) {
server := &http.Server{Handler: h}
go func() {
<-closing
server.Close()
}()
err := server.Serve(ln)
if err != nil && err.Error() != "http: Server closed" {
h.Logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")

4
index.go
View file

@ -53,7 +53,7 @@ type Index struct {
// NewIndex returns a new instance of Index.
func NewIndex(path, name string) (*Index, error) {
err := ValidateName(name)
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
@ -295,7 +295,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, e
func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
if name == "" {
return nil, errors.New("field name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}

68
pilosa.go
View file

@ -16,9 +16,7 @@ package pilosa
import (
"errors"
"net"
"regexp"
"strings"
"github.com/pilosa/pilosa/internal"
)
@ -108,26 +106,16 @@ func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// ValidateName ensures that the name is a valid format.
func ValidateName(name string) error {
// validateName ensures that the name is a valid format.
func validateName(name string) error {
if !nameRegexp.Match([]byte(name)) {
return ErrName
}
return nil
}
// StringInSlice checks for substring a in the slice.
func StringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
// StringSlicesAreEqual determines if two string slices are equal.
func StringSlicesAreEqual(a, b []string) bool {
// stringSlicesAreEqual determines if two string slices are equal.
func stringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
@ -150,54 +138,6 @@ func StringSlicesAreEqual(a, b []string) bool {
return true
}
// SliceDiff returns the difference between two uint64 slices.
func SliceDiff(a, b []uint64) []uint64 {
m := make(map[uint64]uint64)
for _, y := range b {
m[y]++
}
var ret []uint64
for _, x := range a {
if m[x] > 0 {
m[x]--
continue
}
ret = append(ret, x)
}
return ret
}
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {
if strings.Contains(b, a) {
return true
}
}
return false
}
// HostToIP converts host to an IP4 address based on net.LookupIP().
func HostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}
// AddressWithDefaults converts addr into a valid address,
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {

43
pilosa_internal_test.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"testing"
)
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
if validateName(name) != nil {
t.Fatalf("Should be valid index name: %s", name)
}
}
}
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, name := range names {
if validateName(name) == nil {
t.Fatalf("Should be invalid index name: %s", name)
}
}
}

48
pilosa_test.go
View file

@ -22,54 +22,6 @@ import (
_ "github.com/pilosa/pilosa/test"
)
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
if pilosa.ValidateName(name) != nil {
t.Fatalf("Should be valid index name: %s", name)
}
}
}
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, name := range names {
if pilosa.ValidateName(name) == nil {
t.Fatalf("Should be invalid index name: %s", name)
}
}
}
func TestStringInSlice(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "localhost:10101"
if !pilosa.StringInSlice(substr, list) {
t.Fatalf("Expected substring %s in %v", substr, list)
}
substr = "10101"
if pilosa.StringInSlice(substr, list) {
t.Fatalf("Expected substring %s not in %v", substr, list)
}
}
func TestContainsSubstring(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "10101"
if !pilosa.ContainsSubstring(substr, list) {
t.Fatalf("Expected substring %s contained in %v", substr, list)
}
substr = "4000"
if pilosa.ContainsSubstring(substr, list) {
t.Fatalf("Expected substring %s in not contained in %v", substr, list)
}
}
func TestAddressWithDefaults(t *testing.T) {
tests := []struct {
addr string

60
server.go
View file

@ -19,7 +19,6 @@ import (
"fmt"
"log"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
@ -63,7 +62,6 @@ type Server struct {
Broadcaster Broadcaster
BroadcastReceiver BroadcastReceiver
Gossiper Gossiper
remoteClient *http.Client
systemInfo SystemInfo
gcNotifier GCNotifier
NewAttrStore func(string) AttrStore
@ -162,15 +160,6 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption {
}
}
// TODO: Remove RemoteClient
func OptServerRemoteClient(c *http.Client) ServerOption {
return func(s *Server) error {
s.remoteClient = c
s.Cluster.RemoteClient = c
return nil
}
}
func OptServerInternalClient(c InternalClient) ServerOption {
return func(s *Server) error {
s.executor = NewExecutor(OptExecutorInternalQueryClient(c))
@ -313,18 +302,8 @@ func (s *Server) Open() error {
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
server := &http.Server{Handler: s.handler}
go func() {
<-s.closing
server.Close()
}()
err := server.Serve(s.ln)
if err != nil && err.Error() != "http: Server closed" {
s.logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// Serve handler.
go s.handler.Serve(s.ln, s.closing)
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
@ -332,7 +311,7 @@ func (s *Server) Open() error {
}
// Open Cluster management.
if err := s.Cluster.Open(); err != nil {
if err := s.Cluster.open(); err != nil {
return fmt.Errorf("opening Cluster: %v", err)
}
@ -340,7 +319,7 @@ func (s *Server) Open() error {
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.Cluster.SetNodeState(NodeStateReady); err != nil {
if err := s.Cluster.setNodeState(NodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
}
@ -349,7 +328,7 @@ func (s *Server) Open() error {
// the cluster without waiting for data to load on the coordinator. Before
// this starts, the joins are queued up in the Cluster.joiningLeavingNodes
// buffered channel.
s.Cluster.ListenForJoins()
s.Cluster.listenForJoins()
// Start background monitoring.
s.wg.Add(3)
@ -370,7 +349,7 @@ func (s *Server) Close() error {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.Close()
s.Cluster.close()
}
if s.Holder != nil {
s.Holder.Close()
@ -424,7 +403,6 @@ func (s *Server) monitorAntiEntropy() {
syncer.Node = s.Cluster.Node
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.remoteClient
syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer")
// Sync holders.
@ -493,26 +471,26 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
return err
}
case *internal.ClusterStatus:
err := s.Cluster.MergeClusterStatus(obj)
err := s.Cluster.mergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
err := s.Cluster.FollowResizeInstruction(obj)
err := s.Cluster.followResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
err := s.Cluster.MarkResizeInstructionComplete(obj)
err := s.Cluster.markResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.Cluster.SetCoordinator(DecodeNode(obj.New))
s.Cluster.setCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.Cluster.UpdateCoordinator(DecodeNode(obj.New))
s.Cluster.updateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State)
err := s.Cluster.receiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
@ -650,7 +628,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Logger = s.logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.nodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("NodeID", s.NodeID)
@ -659,7 +637,7 @@ func (s *Server) monitorDiagnostics() {
// Flush the diagnostics metrics at startup, then on each tick interval
flush := func() {
openFiles, err := CountOpenFiles()
openFiles, err := countOpenFiles()
if err == nil {
s.diagnostics.Set("OpenFiles", openFiles)
}
@ -716,7 +694,7 @@ func (s *Server) monitorRuntime() {
// Record the number of go routines.
s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0)
openFiles, err := CountOpenFiles()
openFiles, err := countOpenFiles()
// Open File handles.
if err == nil {
s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0)
@ -732,8 +710,8 @@ func (s *Server) monitorRuntime() {
}
}
// CountOpenFiles on operating systems that support lsof.
func CountOpenFiles() (int, error) {
// countOpenFiles on operating systems that support lsof.
func countOpenFiles() (int, error) {
switch runtime.GOOS {
case "darwin", "linux", "unix", "freebsd":
// -b option avoid kernel blocks
@ -747,9 +725,9 @@ func CountOpenFiles() (int, error) {
return len(lines), nil
case "windows":
// TODO: count open file handles on windows
return 0, errors.New("CountOpenFiles() on Windows is not supported")
return 0, errors.New("countOpenFiles() on Windows is not supported")
default:
return 0, errors.New("CountOpenFiles() on this OS is not supported")
return 0, errors.New("countOpenFiles() on this OS is not supported")
}
}

2
server/server.go
View file

@ -216,7 +216,6 @@ func (m *Command) SetupServer() error {
}
c := http.GetHTTPClient(TLSConfig)
api.RemoteClient = c
m.Server, err = pilosa.NewServer(
pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
@ -235,7 +234,6 @@ func (m *Command) SetupServer() error {
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerListener(ln),
pilosa.OptServerURI(uri),
pilosa.OptServerRemoteClient(c),
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
)

16
server/server_test.go
View file

@ -21,7 +21,6 @@ import (
"io/ioutil"
"math/rand"
"reflect"
"runtime"
"sort"
"strings"
"testing"
@ -263,21 +262,6 @@ func tempMkdir(t *testing.T) string {
return dir
}
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported CountOpenFiles test on Windows.")
}
count, err := pilosa.CountOpenFiles()
if err != nil {
t.Errorf("CountOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("CountOpenFiles returned invalid value 0.")
}
}
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)

35
server_internal_test.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"runtime"
"testing"
)
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported countOpenFiles test on Windows.")
}
count, err := countOpenFiles()
if err != nil {
t.Errorf("countOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("countOpenFiles returned invalid value 0.")
}
}

14
server_test.go
View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (

6
stats.go
View file

@ -110,7 +110,7 @@ func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient {
return &ExpvarStatsClient{
m: m,
tags: UnionStringSlice(c.tags, tags),
tags: unionStringSlice(c.tags, tags),
}
}
@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error {
return nil
}
// UnionStringSlice returns a sorted set of tags which combine a & b.
func UnionStringSlice(a, b []string) []string {
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)

8
stats_test.go
View file

@ -95,7 +95,7 @@ func TestStatsCount_TopN(t *testing.T) {
// Execute query.
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "TopN" {
@ -124,7 +124,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "Bitmap" {
@ -154,7 +154,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
field := e.Holder.Field("d", "f")
if field == nil {
t.Fatal("field not found")
@ -184,7 +184,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx := e.Holder.Index("d")
if idx == nil {
t.Fatal("idex not found")

38
statsd/statsd.go
View file

@ -15,6 +15,7 @@
package statsd
import (
"sort"
"time"
"github.com/DataDog/datadog-go/statsd"
@ -72,7 +73,7 @@ func (c *StatsClient) Tags() []string {
func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient {
return &StatsClient{
client: c.client,
tags: pilosa.UnionStringSlice(c.tags, tags),
tags: unionStringSlice(c.tags, tags),
logger: c.logger,
}
}
@ -124,3 +125,38 @@ func (c *StatsClient) Timing(name string, value time.Duration, rate float64) {
func (c *StatsClient) SetLogger(logger pilosa.Logger) {
c.logger = logger
}
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)
// Find size of largest slice.
n := len(a)
if len(b) > n {
n = len(b)
}
// Exit if both sets are empty.
if n == 0 {
return nil
}
// Iterate over both in order and merge.
other := make([]string, 0, n)
for len(a) > 0 || len(b) > 0 {
if len(a) == 0 {
other, b = append(other, b[0]), b[1:]
} else if len(b) == 0 {
other, a = append(other, a[0]), a[1:]
} else if a[0] < b[0] {
other, a = append(other, a[0]), a[1:]
} else if b[0] < a[0] {
other, b = append(other, b[0]), b[1:]
} else {
other, a, b = append(other, a[0]), a[1:], b[1:]
}
}
return other
}

378
test/cluster.go
View file

@ -15,17 +15,10 @@
package test
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
@ -37,14 +30,14 @@ func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Hasher = newModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
ID: fmt.Sprintf("node%d", i),
URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)),
URI: newURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
}
@ -55,372 +48,19 @@ func NewCluster(n int) *pilosa.Cluster {
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// modHasher represents a simple, mod-based hashing.
type modHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
// newModHasher returns a new instance of ModHasher with n buckets.
func newModHasher() *modHasher { return &modHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
func (*modHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }
// NewURI is a test URI creator that intentionally swallows errors.
func NewURI(scheme, host string, port uint16) pilosa.URI {
// newURI is a test URI creator that intentionally swallows errors.
func newURI(scheme, host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
func NewURIFromHostPort(host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
// TestCluster represents a cluster of test nodes, each of which
// has a pilosa.Cluster.
type TestCluster struct {
Clusters []*pilosa.Cluster
common *commonClusterSettings
mu sync.RWMutex
resizing bool
resizeDone chan struct{}
}
type commonClusterSettings struct {
Nodes []*pilosa.Node
}
func (t *TestCluster) CreateIndex(name string) error {
for _, c := range t.Clusters {
if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) CreateField(index, field string, opt pilosa.FieldOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateField(field, opt); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Field(index, field)
if f == nil {
return fmt.Errorf("index/field does not exist: %s/%s", index, field)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) clusterByID(id string) *pilosa.Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
return c
}
}
return nil
}
// AddNode adds a node to the cluster and (potentially) starts a resize job.
func (t *TestCluster) AddNode(saveTopology bool) error {
id := len(t.Clusters)
c, err := t.addCluster(id, saveTopology)
if err != nil {
return err
}
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &pilosa.NodeEvent{
Event: pilosa.NodeJoin,
Node: c.Node,
}
if err := coord.ReceiveEvent(ev); err != nil {
return err
}
// Wait for the AddNode job to finish.
if c.State() != pilosa.ClusterStateNormal {
t.resizeDone = make(chan struct{})
t.mu.Lock()
t.resizing = true
t.mu.Unlock()
<-t.resizeDone
}
}
return nil
}
// WriteTopology writes the given topology to disk.
func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
}
return nil
}
func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &pilosa.Node{
ID: id,
URI: uri,
}
// add URI to common
//t.common.NodeIDs = append(t.common.NodeIDs, id)
//sort.Sort(t.common.NodeIDs)
// add node to common
t.common.Nodes = append(t.common.Nodes, node)
// create node-specific temp directory
path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
if err != nil {
return nil, err
}
// holder
h := pilosa.NewHolder()
h.Path = path
// cluster
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
c.Holder = h
c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes)
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
c.Broadcaster = t
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
}
}
// Add this node to the TestCluster.
t.Clusters = append(t.Clusters, c)
return c, nil
}
// NewTestCluster returns a new instance of test.Cluster.
func NewTestCluster(n int) *TestCluster {
tc := &TestCluster{
common: &commonClusterSettings{},
}
// add clusters
for i := 0; i < n; i++ {
_, err := tc.addCluster(i, true)
if err != nil {
panic(err)
}
}
return tc
}
// SetState sets the state of the cluster on each node.
func (t *TestCluster) SetState(state string) {
for _, c := range t.Clusters {
c.SetState(state)
}
}
// Open opens all clusters in the test cluster.
func (t *TestCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(pilosa.NodeStateReady); err != nil {
return err
}
}
// Start the listener on the coordinator.
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
return nil
}
// Close closes all clusters in the test cluster.
func (t *TestCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// TestCluster implements Broadcaster interface.
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *TestCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == pilosa.ClusterStateNormal && t.resizing {
close(t.resizeDone)
}
t.mu.RUnlock()
}
return nil
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *TestCluster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
}
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
}
// Stop processing on any error.
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI)
instrNode := pilosa.DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
for _, src := range instr.Sources {
srcNode := pilosa.DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Field(src.Index, src.Field)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
return err
}
}
buf := bytes.NewBuffer(nil)
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
}
}
return nil
}(); err != nil {
complete.Error = err.Error()
}
node := pilosa.DecodeNode(instr.Coordinator)
if err := t.SendTo(node, complete); err != nil {
return err
}
return nil
}

28
time.go
View file

@ -79,8 +79,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) {
return q, nil
}
// ViewByTimeUnit returns the view name for time with a given quantum unit.
func ViewByTimeUnit(name string, t time.Time, unit rune) string {
// viewByTimeUnit returns the view name for time with a given quantum unit.
func viewByTimeUnit(name string, t time.Time, unit rune) string {
switch unit {
case 'Y':
return fmt.Sprintf("%s_%s", name, t.Format("2006"))
@ -95,11 +95,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string {
}
}
// ViewsByTime returns a list of views for a given timestamp.
func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
// viewsByTime returns a list of views for a given timestamp.
func viewsByTime(name string, t time.Time, q TimeQuantum) []string {
a := make([]string, 0, len(q))
for _, unit := range q {
view := ViewByTimeUnit(name, t, unit)
view := viewByTimeUnit(name, t, unit)
if view == "" {
continue
}
@ -108,8 +108,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
return a
}
// ViewsByTimeRange returns a list of views to traverse to query a time range.
func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
// viewsByTimeRange returns a list of views to traverse to query a time range.
func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
t := start
// Save flags for performance.
@ -127,7 +127,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextDayGTE(t, end) {
break
} else if t.Hour() != 0 {
results = append(results, ViewByTimeUnit(name, t, 'H'))
results = append(results, viewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
continue
}
@ -138,7 +138,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextMonthGTE(t, end) {
break
} else if t.Day() != 1 {
results = append(results, ViewByTimeUnit(name, t, 'D'))
results = append(results, viewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
continue
}
@ -148,7 +148,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextYearGTE(t, end) {
break
} else if t.Month() != 1 {
results = append(results, ViewByTimeUnit(name, t, 'M'))
results = append(results, viewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
continue
}
@ -164,16 +164,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
// Walk back down from largest units to smallest units.
for t.Before(end) {
if hasYear && nextYearGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'Y'))
results = append(results, viewByTimeUnit(name, t, 'Y'))
t = t.AddDate(1, 0, 0)
} else if hasMonth && nextMonthGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'M'))
results = append(results, viewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
} else if hasDay && nextDayGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'D'))
results = append(results, viewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
} else if hasHour {
results = append(results, ViewByTimeUnit(name, t, 'H'))
results = append(results, viewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
} else {
break

58
time_test.go → time_internal_test.go
View file

@ -12,28 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package pilosa
import (
"reflect"
"testing"
"time"
"github.com/pilosa/pilosa"
)
// Ensure string can be parsed into time quantum.
func TestParseTimeQuantum(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil {
if q, err := ParseTimeQuantum("YMDH"); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if q != pilosa.TimeQuantum("YMDH") {
} else if q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %#v", q)
}
})
t.Run("ErrInvalidTimeQuantum", func(t *testing.T) {
if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum {
if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
t.Fatalf("unexpected error: %s", err)
}
})
@ -44,22 +42,22 @@ func TestViewByTimeUnit(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("Y", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("M", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" {
if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("D", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("H", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
t.Fatalf("unexpected name: %s", s)
}
})
@ -70,14 +68,14 @@ func TestViewsByTime(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH"))
a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) {
t.Fatalf("unexpected names: %+v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D"))
a := viewsByTime("F", ts, mustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000102"}) {
t.Fatalf("unexpected names: %+v", a)
}
@ -87,82 +85,82 @@ func TestViewsByTime(t *testing.T) {
// Ensure sets of fields can be returned for a given time range.
func TestViewsByTimeRange(t *testing.T) {
t.Run("Y", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YM", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM"))
a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM"))
if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD"))
a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD"))
if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH"))
a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("M", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M"))
if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD"))
a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD"))
if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH"))
a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH"))
if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("DH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH"))
if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("H", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H"))
if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
}
// DefaultTimeLayout is the time layout used by the tests.
const DefaultTimeLayout = "2006-01-02 15:04"
// defaultTimeLayout is the time layout used by the tests.
const defaultTimeLayout = "2006-01-02 15:04"
// MustParseTime parses value using DefaultTimeLayout. Panic on error.
func MustParseTime(value string) time.Time {
v, err := time.Parse(DefaultTimeLayout, value)
// mustParseTime parses value using DefaultTimeLayout. Panic on error.
func mustParseTime(value string) time.Time {
v, err := time.Parse(defaultTimeLayout, value)
if err != nil {
panic(err)
}
return v
}
// MustParseTimeQuantum parses v into a time quantum. Panic on error.
func MustParseTimeQuantum(v string) pilosa.TimeQuantum {
q, err := pilosa.ParseTimeQuantum(v)
// mustParseTimeQuantum parses v into a time quantum. Panic on error.
func mustParseTimeQuantum(v string) TimeQuantum {
q, err := ParseTimeQuantum(v)
if err != nil {
panic(err)
}

16
utils_test.go → utils_internal_test.go
View file

@ -121,7 +121,7 @@ func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64,
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / SliceWidth
nodes := c0.SliceNodes(index, slice)
nodes := c0.sliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
@ -236,7 +236,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error)
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
c.addNode(n)
}
}
@ -273,13 +273,13 @@ func (t *ClusterCluster) SetState(state string) {
// Open opens all clusters in the test cluster.
func (t *ClusterCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
if err := c.open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(NodeStateReady); err != nil {
if err := c.setNodeState(NodeStateReady); err != nil {
return err
}
}
@ -288,7 +288,7 @@ func (t *ClusterCluster) Open() error {
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
t.Clusters[0].listenForJoins()
return nil
}
@ -296,7 +296,7 @@ func (t *ClusterCluster) Open() error {
// Close closes all clusters in the test cluster.
func (t *ClusterCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
err := c.close()
if err != nil {
return err
}
@ -310,7 +310,7 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
c.mergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == ClusterStateNormal && t.resizing {
@ -337,7 +337,7 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
go coord.markResizeInstructionComplete(obj)
}
return nil
}

6
view.go
View file

@ -34,8 +34,8 @@ const (
viewBSIGroupPrefix = "bsig_"
)
// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
// isValidView returns true if name is valid.
func isValidView(name string) bool {
return name == ViewStandard
}
@ -73,7 +73,7 @@ func NewView(path, index, field, name string, cacheSize uint32) *View {
name: name,
cacheSize: cacheSize,
cacheType: DefaultCacheType,
cacheType: defaultCacheType,
fragments: make(map[uint64]*Fragment),
broadcaster: NopBroadcaster,

2
view_internal_test.go
View file

@ -26,7 +26,7 @@ func mustOpenView(index, field, name string) *View {
panic(err)
}
v := NewView(path, index, field, name, DefaultCacheSize)
v := NewView(path, index, field, name, defaultCacheSize)
if err := v.open(); err != nil {
panic(err)
}