From f1e3ac90d4a1648b48f53ca842a80700a236bde8 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 12 Sep 2017 16:24:37 -0500 Subject: [PATCH 001/311] Determine cluster membership via gossip. Handle resize cluster events. Add Toplogy support. Refactor FrameOptions. Add tests. --- Gopkg.lock | 5 +- broadcast.go | 41 +- cluster.go | 865 +++++++++++++++++- cluster_internal_test.go | 62 ++ cluster_test.go | 197 +++- config.go | 1 + ctl/server.go | 1 + event.go | 55 ++ frame.go | 27 +- gossip/gossip.go | 93 +- handler_test.go | 2 + holder.go | 18 +- holder_test.go | 31 + internal/private.pb.go | 1876 ++++++++++++++++++++++++++++++++------ internal/private.proto | 30 +- internal/public.pb.go | 477 +++++++--- pilosa.go | 44 + server.go | 114 +-- server/server.go | 39 +- server/server_test.go | 17 +- test/cluster.go | 8 + test/handler.go | 4 +- 22 files changed, 3470 insertions(+), 537 deletions(-) create mode 100644 cluster_internal_test.go create mode 100644 event.go diff --git a/Gopkg.lock b/Gopkg.lock index 19483791d..e140b0578 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -27,7 +27,8 @@ [[projects]] name = "github.com/boltdb/bolt" packages = ["."] - revision = "4b1ebc1869ad66568b313d0dc410e2be72670dda" + revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" + version = "v1.3.1" [[projects]] name = "github.com/davecgh/go-spew" @@ -230,6 +231,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e8e78a7c61547d8f4d967c8deac9334b3151e7c10c4f7909e80758956e9c8204" + inputs-digest = "72a71ef2a911e41396bea6c2639c4d2e6faa69ca74c3acb47fb249a054036685" solver-name = "gps-cdcl" solver-version = 1 diff --git a/broadcast.go b/broadcast.go index 0785a21cc..3132844cf 100644 --- a/broadcast.go +++ b/broadcast.go @@ -61,6 +61,7 @@ func (s *StaticNodeSet) Join(nodes []*Node) error { type Broadcaster interface { SendSync(pb proto.Message) error SendAsync(pb proto.Message) error + SendTo(to *Node, pb proto.Message) error } func init() { @@ -72,16 +73,21 @@ var NopBroadcaster Broadcaster type nopBroadcaster struct{} -// SendSync A no-op implemenetation of Broadcaster SendSync method. +// SendSync is a no-op implemenetation of Broadcaster SendSync method. func (c *nopBroadcaster) SendSync(pb proto.Message) error { return nil } -// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +// SendAsync is a no-op implemenetation of Broadcaster SendAsync method. func (c *nopBroadcaster) SendAsync(pb proto.Message) error { return nil } +// SendTo is a no-op implemenetation of Broadcaster SendTo method. +func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error { + return nil +} + // BroadcastHandler is the interface for the pilosa object which knows how to // handle broadcast messages. (Hint: this is implemented by pilosa.Server) type BroadcastHandler interface { @@ -108,14 +114,17 @@ var NopBroadcastReceiver = &nopBroadcastReceiver{} // Broadcast message types. const ( - MessageTypeCreateSlice = 1 - MessageTypeCreateIndex = 2 - MessageTypeDeleteIndex = 3 - MessageTypeCreateFrame = 4 - MessageTypeDeleteFrame = 5 - MessageTypeCreateInputDefinition = 6 - MessageTypeDeleteInputDefinition = 7 - MessageTypeDeleteView = 8 + MessageTypeCreateSlice = 1 + MessageTypeCreateIndex = 2 + MessageTypeDeleteIndex = 3 + MessageTypeCreateFrame = 4 + MessageTypeDeleteFrame = 5 + MessageTypeCreateInputDefinition = 6 + MessageTypeDeleteInputDefinition = 7 + MessageTypeDeleteView = 8 + MessageTypeClusterStatus = 9 + MessageTypeResizeInstruction = 10 + MessageTypeResizeInstructionComplete = 11 ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -138,6 +147,12 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeDeleteInputDefinition case *internal.DeleteViewMessage: typ = MessageTypeDeleteView + case *internal.ClusterStatus: + typ = MessageTypeClusterStatus + case *internal.ResizeInstruction: + typ = MessageTypeResizeInstruction + case *internal.ResizeInstructionComplete: + typ = MessageTypeResizeInstructionComplete default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -170,6 +185,12 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.DeleteInputDefinitionMessage{} case MessageTypeDeleteView: m = &internal.DeleteViewMessage{} + case MessageTypeClusterStatus: + m = &internal.ClusterStatus{} + case MessageTypeResizeInstruction: + m = &internal.ResizeInstruction{} + case MessageTypeResizeInstructionComplete: + m = &internal.ResizeInstructionComplete{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/cluster.go b/cluster.go index 820349ded..6dcc0b102 100644 --- a/cluster.go +++ b/cluster.go @@ -16,9 +16,19 @@ package pilosa import ( "encoding/binary" + "fmt" "hash/fnv" + "io" + "io/ioutil" + "log" + "math/rand" + "os" + "path/filepath" + "sort" + "sync" "time" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -28,18 +38,23 @@ const ( // DefaultReplicaN is the default number of replicas per partition. DefaultReplicaN = 1 -) -// NodeState represents node state returned in /status endpoint for a node in the cluster. -const ( - NodeStateUp = "UP" - NodeStateDown = "DOWN" + // NodeState represents node state returned in /status endpoint for a node in the cluster. + NodeStateStarting = "STARTING" + NodeStateNormal = "NORMAL" + NodeStateResizing = "RESIZING" + + // ResizeJob states. + ResizeJobStateRunning = "RUNNING" + // Final states. + ResizeJobStateDone = "DONE" + ResizeJobStateAborted = "ABORTED" ) // Node represents a node in the cluster. type Node struct { Scheme string `json:"scheme"` - Host string `json:"host"` + Host string `json:"host"` // HostPort status *internal.NodeStatus `json:"status"` } @@ -128,9 +143,18 @@ func (a Nodes) Clone() []*Node { return other } +// ByHost implements sort.Interface for []Node based on +// the Host field. +type ByHost []*Node + +func (h ByHost) Len() int { return len(h) } +func (h ByHost) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h ByHost) Less(i, j int) bool { return h[i].Host < h[j].Host } + // Cluster represents a collection of nodes. type Cluster struct { - Nodes []*Node + URI *URI + Nodes []*Node // TODO phase this out? NodeSet NodeSet // Hashing algorithm used to assign partitions to nodes. @@ -144,59 +168,116 @@ type Cluster struct { // Threshold for logging long-running queries LongQueryTime time.Duration + + // EventReceiver receives NodeEvents pertaining to node membership. + EventReceiver EventReceiver + + // Data directory path. + Path string + Topology *Topology + + // Required for cluster Resize. + State string + Coordinator string + IndexReporter IndexReporter + Broadcaster Broadcaster + + joiningHosts chan string + + mu sync.RWMutex + jobs map[int64]*ResizeJob + currentJob *ResizeJob + + // Close management + wg sync.WaitGroup + closing chan struct{} + + // The writer for any logging. + LogOutput io.Writer } // NewCluster returns a new instance of Cluster with defaults. func NewCluster() *Cluster { return &Cluster{ - Hasher: &jmphasher{}, - PartitionN: DefaultPartitionN, - ReplicaN: DefaultReplicaN, + Hasher: &jmphasher{}, + PartitionN: DefaultPartitionN, + ReplicaN: DefaultReplicaN, + EventReceiver: NopEventReceiver, + + joiningHosts: make(chan string, 10), // buffered channel + jobs: make(map[int64]*ResizeJob), + closing: make(chan struct{}), + + LogOutput: os.Stderr, } } -// NodeSetHosts returns the list of host strings for NodeSet members. -func (c *Cluster) NodeSetHosts() []string { - if c.NodeSet == nil { - return []string{} - } - a := make([]string, 0, len(c.NodeSet.Nodes())) - for _, m := range c.NodeSet.Nodes() { - a = append(a, m.Host) - } - return a +// logger returns a logger for the cluster. +func (c *Cluster) logger() *log.Logger { + return log.New(c.LogOutput, "", log.LstdFlags) } -// NodeStates returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value. -func (c *Cluster) NodeStates() map[string]string { - h := make(map[string]string) - for _, n := range c.Nodes { - h[n.Host] = NodeStateDown +// IsCoordinator is true if this node is the coordinator. +func (c *Cluster) IsCoordinator() bool { + return c.Coordinator == c.URI.HostPort() +} + +// AddHost adds a node to the Cluster and updates and saves the +// new topology. +func (c *Cluster) AddHost(host string) error { + + // add to cluster + _, added := c.AddNode(host) + if !added { + return nil } - // we are assuming that NodeSetHosts is a subset of c.Nodes - for _, m := range c.NodeSetHosts() { - if _, ok := h[m]; ok { - h[m] = NodeStateUp - } + + // add to topology + if c.Topology == nil { + return fmt.Errorf("Cluster.Topology is nil") } - return h + if !c.Topology.AddHost(host) { + return nil + } + + // save topology + return c.saveTopology() +} + +// HostList returns the list of hosts in the cluster. +func (c *Cluster) HostList() []string { + return Nodes(c.Nodes).Hosts() +} + +func (c *Cluster) setState(state string) { + c.State = state + localNode := c.localNode() + localNode.SetState(state) +} + +func (c *Cluster) localNode() *Node { + return c.NodeByHost(c.URI.HostPort()) } // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ - Nodes: encodeClusterStatus(c.Nodes), + State: c.State, + HostList: c.HostList(), + //NodeStatuses: encodeNodeStatuses(c.Nodes), // TODO travis: remove this? } } -// encodeClusterStatus converts a into its internal representation. -func encodeClusterStatus(a []*Node) []*internal.NodeStatus { +/* +// encodeNodeStatuses converts a into its internal representation. +func encodeNodeStatuses(a []*Node) []*internal.NodeStatus { other := make([]*internal.NodeStatus, len(a)) for i := range a { other[i] = a[i].status } return other } +*/ // NodeByHost returns a node reference by host. func (c *Cluster) NodeByHost(host string) *Node { @@ -208,6 +289,169 @@ func (c *Cluster) NodeByHost(host string) *Node { return nil } +// AddNode adds a node to the cluster, sorted by host. +// Returns a pointer to the node and true if the node was added. +func (c *Cluster) AddNode(host string) (*Node, bool) { + n := c.NodeByHost(host) + if n != nil { + return n, false + } + + n = &Node{Host: host} + c.Nodes = append(c.Nodes, n) + + // All hosts must be merged in the same order on all nodes in the cluster. + sort.Sort(ByHost(c.Nodes)) + + return n, true +} + +// frag is a struct of basic fragment information. +type frag struct { + frame string + view string + slice uint64 +} + +func fragsDiff(a, b []frag) []frag { + m := make(map[frag]uint64) + + for _, y := range b { + m[y]++ + } + + var ret []frag + for _, x := range a { + if m[x] > 0 { + m[x]-- + continue + } + ret = append(ret, x) + } + + return ret +} + +type fragsByHost map[string][]frag + +func (a fragsByHost) add(b fragsByHost) fragsByHost { + for k, v := range b { + for _, vv := range v { + a[k] = append(a[k], vv) + } + } + return a +} + +type viewsByFrame map[string][]string + +func (a viewsByFrame) addView(frame, view string) { + a[frame] = append(a[frame], view) +} + +func (c *Cluster) fragsByHost(idx *Index) fragsByHost { + // frameViews is a map of frame to slice of views. + frameViews := make(viewsByFrame) + inverseFrameViews := make(viewsByFrame) + + for _, frame := range idx.Frames() { + for _, view := range frame.Views() { + if IsInverseView(view.Name()) { + inverseFrameViews.addView(frame.Name(), view.Name()) + } else { + frameViews.addView(frame.Name(), view.Name()) + } + } + } + + std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) + inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews) + return std.add(inv) +} + +// fragCombos returns a map (by host) of lists of fragments for a given index +// by creating every combination of frame/view specified in `frameViews` up to maxSlice. +func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { + t := make(fragsByHost) + for i := uint64(0); i <= maxSlice; i++ { + f := c.FragmentNodes(idx, i) + for _, n := range f { + // for each frame/view combination: + for frame, views := range frameViews { + for _, view := range views { + t[n.Host] = append(t[n.Host], frag{frame, view, i}) + } + } + } + } + return t +} + +// DataDiff returns a list of ResizeSources - for each host in the `to` cluster - +// required to move from cluster `c` to cluster `to`. +func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[string][]*internal.ResizeSource { + m := make(map[string][]*internal.ResizeSource) + + // Initialize the map with all the nodes in `to`. + for _, n := range to.Nodes { + m[n.Host] = nil + } + + // For now, we want our source to be confined to the primary fragment + // (i.e. don't use replicas as source data). So if it's not already, + // base our source fragments on a cluster with replica = 1. + srcCluster := c + if c.ReplicaN > 1 { + srcCluster = NewCluster() + srcCluster.Nodes = Nodes(c.Nodes).Clone() + srcCluster.Hasher = c.Hasher + srcCluster.PartitionN = c.PartitionN + srcCluster.ReplicaN = 1 + } + + // Represents the fragment location for the from/to clusters. + fFrags := c.fragsByHost(idx) + tFrags := to.fragsByHost(idx) + + // srcFrags is the frag map based on a source cluster of replica = 1. + srcFrags := srcCluster.fragsByHost(idx) + + // srcHostsByFrag is the inverse representation of srcFrags. + srcHostsByFrag := make(map[frag]string) + for host, frags := range srcFrags { + for _, frag := range frags { + srcHostsByFrag[frag] = host + } + } + + // Get the frag diff for each host. + diffs := make(fragsByHost) + for host, frags := range tFrags { + if _, ok := fFrags[host]; ok { + diffs[host] = fragsDiff(frags, fFrags[host]) + } else { + diffs[host] = frags + } + } + + // Get the ResizeSource for each diff. + for host, diff := range diffs { + m[host] = []*internal.ResizeSource{} + for _, frag := range diff { + src := &internal.ResizeSource{ + Host: srcHostsByFrag[frag], + Index: idx.Name(), + Frame: frag.frame, + View: frag.view, + Slice: frag.slice, + } + m[host] = append(m[host], src) + } + } + + return m +} + // Partition returns the partition that a slice belongs to. func (c *Cluster) Partition(index string, slice uint64) int { var buf [8]byte @@ -289,3 +533,556 @@ func (h *jmphasher) Hash(key uint64, n int) int { } return int(b) } + +func (c *Cluster) Open() error { + // Cluster always comes up in state STARTING until cluster membership is determined. + c.State = NodeStateStarting + + // Load topology file if it exists. + if err := c.loadTopology(); err != nil { + return fmt.Errorf("load topology: %v", err) + } + + // Only the coordinator needs to consider the .topology file. + if c.IsCoordinator() { + state, err := c.considerTopology() + if err != nil { + return fmt.Errorf("considerTopology: %v", err) + } + // Add the local node to the cluster and update state. + c.AddHost(c.URI.HostPort()) + c.setState(state) + } else { + // Add the local node to the cluster. + c.AddHost(c.URI.HostPort()) + } + + // Start the EventReceiver. + if err := c.EventReceiver.Start(c); err != nil { + return fmt.Errorf("starting EventReceiver: %v", err) + } + + // Open NodeSet communication. + if err := c.NodeSet.Open(); err != nil { + return fmt.Errorf("opening NodeSet: %v", err) + } + + // Listen for cluster-resize events. + c.wg.Add(1) + go func() { defer c.wg.Done(); c.listenForJoins() }() + + return nil +} + +func (c *Cluster) Close() error { + // Notify goroutines of closing and wait for completion. + close(c.closing) + c.wg.Wait() + + return nil +} + +func (c *Cluster) needTopologyAgreement() bool { + return c.State == NodeStateStarting && !SlicesAreEqual(c.Topology.HostList, Nodes(c.Nodes).Hosts()) +} + +func (c *Cluster) haveTopologyAgreement() bool { + return SlicesAreEqual(c.Topology.HostList, Nodes(c.Nodes).Hosts()) +} + +func (c *Cluster) handleJoiningHost(host string) error { + j, err := c.GenerateResizeJob(host) + if err != nil { + return err + } + + // Run the job. + err = j.Run() + if err != nil { + return err + } + + // Wait for the ResizeJob to finish or be aborted. + jobResult := <-j.result + switch jobResult { + case ResizeJobStateDone: + c.CompleteCurrentJob(ResizeJobStateDone) + // Add host to the cluster. + return c.AddHost(host) + case ResizeJobStateAborted: + c.CompleteCurrentJob(ResizeJobStateAborted) + } + return nil +} + +func (c *Cluster) setStateAndBroadcast(state string) error { + c.setState(state) + // Broadcast status changes to the cluster. + return c.Broadcaster.SendSync(c.Status()) +} + +func (c *Cluster) listenForJoins() { + var hostJoined bool + + for { + + // Handle all pending joins before changing state back to NORMAL. + select { + case host := <-c.joiningHosts: + err := c.handleJoiningHost(host) + if err != nil { + c.logger().Printf("handleJoiningHost error: err=%s", err) + continue + } + hostJoined = true + continue + default: + } + + // Only change state to NORMAL if we have successfully added at least one host. + if hostJoined { + // Put the cluster back to state NORMAL and broadcast. + if err := c.setStateAndBroadcast(NodeStateNormal); err != nil { + c.logger().Printf("setStateAndBroadcast error: err=%s", err) + } + } + + // Wait for a joining host or a close. + select { + case <-c.closing: + return + case host := <-c.joiningHosts: + err := c.handleJoiningHost(host) + if err != nil { + c.logger().Printf("handleJoiningHost error: err=%s", err) + continue + } + hostJoined = true + continue + } + } +} + +// GenerateResizeJob creates a new ResizeJob based on the new host being +// added. It also saves a reference to the ResizeJob in the `jobs` map +// for future lookup by JobID. +func (c *Cluster) GenerateResizeJob(addHost string) (*ResizeJob, error) { + c.mu.Lock() + defer c.mu.Unlock() + + j := c.generateResizeJob(addHost) + + // Save job in jobs map for future reference. + c.jobs[j.ID] = j + + // Set job as currentJob. + if c.currentJob != nil { + return nil, fmt.Errorf("there is currently a resize job running") + } + c.currentJob = j + + return j, nil +} + +// generateResizeJob returns a ResizeJob with instructions based on +// the difference between Cluster and a new Cluster containing addHost. +// Broadcaster is associated to the ResizeJob here for use in broadcasting +// the resize instructions to other nodes in the cluster. +func (c *Cluster) generateResizeJob(addHost string) *ResizeJob { + + j := NewResizeJob(addHost, Nodes(c.Nodes).Hosts()) + j.Broadcaster = c.Broadcaster + + // toCluster is a clone of Cluster with the new node added for comparison. + toCluster := NewCluster() + toCluster.Nodes = Nodes(c.Nodes).Clone() + toCluster.Hasher = c.Hasher + toCluster.PartitionN = c.PartitionN + toCluster.ReplicaN = c.ReplicaN + toCluster.AddNode(addHost) + + // Add to the ResizeJob the instructions for each index. + for _, idx := range c.IndexReporter.Indexes() { + // dataDiff is map[string][]*internal.ResizeSource, where string is + // a host in toCluster. + dataDiff := c.DataDiff(toCluster, idx) + + for host, sources := range dataDiff { + // If a host doesn't need to request data, mark it as complete. + if len(sources) == 0 { + j.Hosts[host] = true + continue + } + instr := &internal.ResizeInstruction{ + JobID: j.ID, + Host: host, + Coordinator: c.Coordinator, + Sources: sources, + } + j.Instructions = append(j.Instructions, instr) + } + } + + return j +} + +// CompleteCurrentJob sets the state of the current ResizeJob +// then removes the pointer to currentJob. +func (c *Cluster) CompleteCurrentJob(state string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.currentJob == nil { + return + } + c.currentJob.SetState(state) + c.currentJob = nil +} + +// followResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { + go func() { + // Request each source file in ResizeSources. + for _, src := range instr.Sources { + /************************************************************/ + // TODO travis: get the data files from other nodes. + fmt.Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.Host) + for i := 0; i <= 4; i++ { + fmt.Printf(" %d", i) + time.Sleep(1 * time.Second) + } + fmt.Println("") + /************************************************************/ + } + + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + Host: instr.Host, + } + + node := &Node{ + Host: instr.Coordinator, + } + if err := c.Broadcaster.SendTo(node, complete); err != nil { + c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) + } + }() +} + +func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { + j := c.Job(complete.JobID) + + j.mu.Lock() + defer j.mu.Unlock() + + if j.isComplete() { + return fmt.Errorf("ResizeJob %d is no longer running", j.ID) + } + + // Mark host complete. + j.Hosts[complete.Host] = true + + if !j.hostsArePending() { + j.result <- ResizeJobStateDone + } + + return nil +} + +// Job returns a ResizeJob by id. +func (c *Cluster) Job(id int64) *ResizeJob { + c.mu.RLock() + defer c.mu.RUnlock() + return c.job(id) +} + +func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } + +type ResizeJob struct { + ID int64 + Hosts map[string]bool + Instructions []*internal.ResizeInstruction + Broadcaster Broadcaster + + result chan string + + mu sync.RWMutex + state string +} + +// NewResizeJob returns a new instance of ResizeJob. +func NewResizeJob(addHost string, existingHosts []string) *ResizeJob { + + // Build a map of hosts to track their resize status. + hosts := make(map[string]bool) + + // The value for a node will be set to true after that node + // has indicated that it has completed all resize instructions. + for _, h := range existingHosts { + hosts[h] = false + } + // Include the added node in the map for tracking. + hosts[addHost] = false + + return &ResizeJob{ + ID: rand.Int63(), + Hosts: hosts, + result: make(chan string), + } +} + +func (j *ResizeJob) State() string { + j.mu.RLock() + defer j.mu.RUnlock() + return j.state +} + +func (j *ResizeJob) SetState(state string) { + j.mu.Lock() + j.setState(state) + j.mu.Unlock() +} + +func (j *ResizeJob) setState(state string) { + if j.state == "" || j.state == ResizeJobStateRunning { + j.state = state + } +} + +// Run distributes ResizeInstructions. +func (j *ResizeJob) Run() error { + j.mu.RLock() + defer j.mu.RUnlock() + + // Set job state to RUNNING. + j.setState(ResizeJobStateRunning) + + // Job can be considered done in the case where it doesn't require any action. + if !j.hostsArePending() { + j.result <- ResizeJobStateDone + return nil + } + + err := j.distributeResizeInstructions() + if err != nil { + j.result <- ResizeJobStateAborted + return err + } + return nil +} + +// isComplete return true if the job is any one of several completion states. +func (j *ResizeJob) isComplete() bool { + switch j.state { + case ResizeJobStateDone, ResizeJobStateAborted: + return true + default: + return false + } +} + +// hostsArePending returns true if any host is still working on the resize. +func (j *ResizeJob) hostsArePending() bool { + for _, complete := range j.Hosts { + if !complete { + return true + } + } + return false +} + +func (j *ResizeJob) distributeResizeInstructions() error { + // Loop through the ResizeInstructions in ResizeJob and send to each host. + for _, instr := range j.Instructions { + // Because the node may not be in the cluster yet, create + // a dummy node object to use in the SendTo() method. + node := &Node{ + Host: instr.Host, + } + if err := j.Broadcaster.SendTo(node, instr); err != nil { + return err + } + } + return nil +} + +// Topology represents the list of hosts in the cluster. +type Topology struct { + mu sync.RWMutex + HostList []string +} + +func NewTopology() *Topology { + return &Topology{} +} + +// ContainsHost returns true if host matches one of the topology's hosts. +func (t *Topology) ContainsHost(host string) bool { + t.mu.RLock() + defer t.mu.RUnlock() + return t.containsHost(host) +} + +func (t *Topology) containsHost(host string) bool { + for _, thost := range t.HostList { + if thost == host { + return true + } + } + return false +} + +// AddHost adds the host to the topology and returns true if added. +func (t *Topology) AddHost(host string) bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.containsHost(host) { + return false + } + t.HostList = append(t.HostList, host) + return true +} + +// loadTopology reads the topology for the node. +func (c *Cluster) loadTopology() error { + buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) + if os.IsNotExist(err) { + c.Topology = NewTopology() + return nil + } else if err != nil { + return err + } + + var pb internal.Topology + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + c.Topology = decodeTopology(&pb) + + return nil +} + +// saveTopology writes the current topology to disk. +func (c *Cluster) saveTopology() error { + if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { + return err + } + return nil +} + +func encodeTopology(topology *Topology) *internal.Topology { + if topology == nil { + return nil + } + return &internal.Topology{ + HostList: topology.HostList, + } +} + +func decodeTopology(topology *internal.Topology) *Topology { + if topology == nil { + return nil + } + t := &Topology{ + HostList: topology.HostList, + } + return t +} + +func (c *Cluster) considerTopology() (string, error) { + // If there is no .topology file, it's safe to go to state NORMAL. + if len(c.Topology.HostList) == 0 { + return NodeStateNormal, nil + } + + // The local node (coordinator) must be in the .topology. + if !c.Topology.ContainsHost(c.Coordinator) { + return "", fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.HostList) + } + + // If local node is the only thing in .topology, continue to state NORMAL. + if len(c.Topology.HostList) == 1 { + return NodeStateNormal, nil + } + + // Keep the cluster in state "STARTING" until hearing from all nodes. + // Topology contains 2+ hosts. + return NodeStateStarting, nil +} + +// ReceiveEvent represents an implementation of EventHandler. +func (c *Cluster) ReceiveEvent(e *NodeEvent) error { + // Ignore events sent from this node. + if e.Host == c.URI.HostPort() { + return nil + } + + switch e.Event { + case NodeJoin: + // Ignore the event if this is not the coordinator. + if !c.IsCoordinator() { + return nil + } + + if c.needTopologyAgreement() { + // A host that is not part of the topology can't be added to the STARTING cluster. + if !c.Topology.ContainsHost(e.Host) { + return fmt.Errorf("host is not in topology: %v", e.Host) + } + + if err := c.AddHost(e.Host); err != nil { + return err + } + + // If the result of the previous AddHost completed the joining of nodes + // in the topology, then change the state to NORMAL. + if c.haveTopologyAgreement() { + return c.setStateAndBroadcast(NodeStateNormal) + } + + return nil + } + + // Don't do anything else if the cluster already contains the node. + if c.NodeByHost(e.Host) != nil { + return nil + } + + // If the index does not yet have data, go ahead and add the node. + if !c.IndexReporter.HasData() { + if err := c.AddHost(e.Host); err != nil { + return err + } + return c.setStateAndBroadcast(NodeStateNormal) + } + + // If the cluster has data, we need to change to RESIZING and + // kick off the resizing process. + if err := c.setStateAndBroadcast(NodeStateResizing); err != nil { + return err + } + c.joiningHosts <- e.Host + + case NodeLeave: + // TODO: implement this + case NodeUpdate: + // TODO: implement this + } + + return nil +} + +func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { + // Ignore status updates from self (coordinator). + if c.IsCoordinator() { + return nil + } + + for _, host := range cs.HostList { + c.AddHost(host) + } + c.setState(cs.State) + + return nil +} diff --git a/cluster_internal_test.go b/cluster_internal_test.go new file mode 100644 index 000000000..b35670609 --- /dev/null +++ b/cluster_internal_test.go @@ -0,0 +1,62 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "reflect" + "testing" +) + +// Ensure that fragCombos creates the correct fragment mapping. +func TestFragCombos(t *testing.T) { + + c := NewCluster() + c.AddNode("host0") + c.AddNode("host1") + + tests := []struct { + idx string + maxSlice uint64 + frameViews viewsByFrame + expected fragsByHost + }{ + { + idx: "i", + maxSlice: uint64(2), + frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, + expected: fragsByHost{ + "host0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, + "host1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, + }, + }, + { + idx: "foo", + maxSlice: uint64(3), + frameViews: viewsByFrame{"f": []string{"v0"}}, + expected: fragsByHost{ + "host0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, + "host1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, + }, + }, + } + for _, test := range tests { + + actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews) + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("expected: %v, but got: %v", test.expected, actual) + } + + } +} diff --git a/cluster_test.go b/cluster_test.go index 8ab1ab461..4cc54286f 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -22,6 +22,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" ) @@ -91,16 +92,7 @@ func TestHasher(t *testing.T) { } } -// Ensure that an empty cluster returns a valid (empty) NodeSet -func TestCluster_NodeSetHosts(t *testing.T) { - - c := pilosa.Cluster{} - - if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) { - t.Fatalf("unexpected slice of hosts: %s", h) - } -} - +/* TODO travis: fix test // Ensure cluster can compare its Nodes and Members func TestCluster_NodeStates(t *testing.T) { c := pilosa.Cluster{ @@ -130,8 +122,9 @@ func TestCluster_NodeStates(t *testing.T) { t.Fatalf("unexpected node state: %s", spew.Sdump(a)) } } +*/ -// Ensure OwnsSlices can find the actual slice list for node and index +// Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { c := test.NewCluster(5) slices := c.OwnsSlices("test", 10, "host2") @@ -140,3 +133,185 @@ func TestCluster_OwnsSlices(t *testing.T) { t.Fatalf("unexpected slices for node's index: %v", slices) } } + +func TestCluster_Nodes(t *testing.T) { + + nodes := []*pilosa.Node{ + &pilosa.Node{Host: "node0"}, + &pilosa.Node{Host: "node1"}, + &pilosa.Node{Host: "node2"}, + } + + t.Run("Hosts", func(t *testing.T) { + actual := pilosa.Nodes(nodes).Hosts() + expected := []string{"node0", "node1", "node2"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Filter", func(t *testing.T) { + actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).Hosts() + expected := []string{"node0", "node2"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("FilterHost", func(t *testing.T) { + actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterHost("node1")).Hosts() + expected := []string{"node0", "node2"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Contains", func(t *testing.T) { + actualTrue := pilosa.Nodes(nodes).Contains(nodes[1]) + actualFalse := pilosa.Nodes(nodes).Contains(&pilosa.Node{}) + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("ContainsHost", func(t *testing.T) { + actualTrue := pilosa.Nodes(nodes).ContainsHost("node1") + actualFalse := pilosa.Nodes(nodes).ContainsHost("nodeX") + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("Clone", func(t *testing.T) { + clone := pilosa.Nodes(nodes).Clone() + actual := pilosa.Nodes(clone).Hosts() + expected := []string{"node0", "node1", "node2"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} + +func TestCluster_Coordinator(t *testing.T) { + + c1 := *pilosa.NewCluster() + c1.Host = "host0:port0" + c1.Coordinator = "host0:port0" + c2 := *pilosa.NewCluster() + c2.Host = "host1:port1" + c2.Coordinator = "host0:port0" + + t.Run("IsCoordinator", func(t *testing.T) { + if !c1.IsCoordinator() { + t.Errorf("!IsCoordinator error: %v", c1.Host) + } else if c2.IsCoordinator() { + t.Errorf("IsCoordinator error: %v", c2.Host) + } + }) +} + +func TestCluster_Topology(t *testing.T) { + c1 := test.NewCluster(1) + + t.Run("AddHost", func(t *testing.T) { + err := c1.AddHost("abc") + if err != nil { + t.Fatal(err) + } + // add the same host. + err = c1.AddHost("abc") + if err != nil { + t.Fatal(err) + } + err = c1.AddHost("xyz") + if err != nil { + t.Fatal(err) + } + + actual := pilosa.Nodes(c1.Nodes).Hosts() + expected := []string{"abc", "host0", "xyz"} + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("ContainsHost", func(t *testing.T) { + if !c1.Topology.ContainsHost("abc") { + t.Errorf("!ContainsHost error: %v", "abc") + } else if c1.Topology.ContainsHost("invalidHost") { + t.Errorf("ContainsHost error: %v", "invalidHost") + } + }) +} + +// Ensure DataDiff can generate the expected source map. +func TestCluster_Resize(t *testing.T) { + // Given two clusters, ensure DataDiff can determine the sources of data + // needed in order to respond to queries. + t.Run("DataDiff", func(t *testing.T) { + + // Holder + h1 := test.NewHolder() + i, err := h1.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + f, err := i.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) + if err != nil { + t.Fatal(err) + } + _, err = f.CreateViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } + _, err = f.CreateViewIfNotExists("inverse") + if err != nil { + t.Fatal(err) + } + + // Set max slices. + i.SetRemoteMaxSlice(2) + i.SetRemoteMaxInverseSlice(5) + + // Cluster 1 + c1 := test.NewCluster(3) + c1.IndexReporter = h1 + c1.ReplicaN = 2 + + // Cluster 2 + c2 := test.NewCluster(4) + c2.ReplicaN = 2 + + expected := map[string][]*internal.ResizeSource{ + "host0": []*internal.ResizeSource{ + {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 5}, + }, + "host1": []*internal.ResizeSource{ + {Host: "host2", Index: "i", Frame: "f", View: "v", Slice: 0}, + {Host: "host2", Index: "i", Frame: "f", View: "inverse", Slice: 0}, + }, + "host2": []*internal.ResizeSource{ + {Host: "host0", Index: "i", Frame: "f", View: "inverse", Slice: 3}, + }, + "host3": []*internal.ResizeSource{ + {Host: "host0", Index: "i", Frame: "f", View: "v", Slice: 1}, + {Host: "host1", Index: "i", Frame: "f", View: "v", Slice: 2}, + {Host: "host0", Index: "i", Frame: "f", View: "inverse", Slice: 1}, + {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 2}, + {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 5}, + }, + } + + actual := c1.DataDiff(c2, i) + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} diff --git a/config.go b/config.go index 60c49932b..e14dc9851 100644 --- a/config.go +++ b/config.go @@ -74,6 +74,7 @@ type Config struct { } `toml:"gossip"` Cluster struct { + Coordinator string `toml:"coordinator"` ReplicaN int `toml:"replicas"` Type string `toml:"type"` Hosts []string `toml:"hosts"` diff --git a/ctl/server.go b/ctl/server.go index 77361f7db..5c9fd245e 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -31,6 +31,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") + flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") diff --git a/event.go b/event.go new file mode 100644 index 000000000..420cb2073 --- /dev/null +++ b/event.go @@ -0,0 +1,55 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +// NodeEventType are the types of events that can be sent from the +// ChannelEventDelegate. +type NodeEventType int + +const ( + NodeJoin NodeEventType = iota + NodeLeave + NodeUpdate +) + +// NodeEvent is a single event related to node activity in the cluster. +type NodeEvent struct { + Event NodeEventType + Host string // HostPort +} + +// EventHandler is the interface for the pilosa object which knows how to +// handle broadcast messages. (Hint: this is implemented by pilosa.Server) +type EventHandler interface { + ReceiveEvent(e *NodeEvent) error +} + +// EventReceiver is the interface for the object which will listen for and +// decode broadcast messages before passing them to pilosa to handle. The +// implementation of this could be an http server which listens for messages, +// gets the protobuf payload, and then passes it to +// EventHandler.ReceiveMessage. +type EventReceiver interface { + // Start starts listening for broadcast messages - it should return + // immediately, spawning a goroutine if necessary. + Start(EventHandler) error +} + +type nopEventReceiver struct{} + +func (n *nopEventReceiver) Start(e EventHandler) error { return nil } + +// NopEventReceiver is a no-op implementation of the EventReceiver. +var NopEventReceiver = &nopEventReceiver{} diff --git a/frame.go b/frame.go index 21f160a64..c8ad8cf32 100644 --- a/frame.go +++ b/frame.go @@ -964,8 +964,9 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FrameInfo represents schema information for a frame. type FrameInfo struct { - Name string `json:"name"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + Options FrameOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type frameInfoSlice []*FrameInfo @@ -987,6 +988,13 @@ type FrameOptions struct { // Encode converts o into its internal representation. func (o *FrameOptions) Encode() *internal.FrameMeta { + return encodeFrameOptions(o) +} + +func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { + if o == nil { + return nil + } return &internal.FrameMeta{ RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, @@ -998,6 +1006,21 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { } } +func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { + if options == nil { + return nil + } + return &FrameOptions{ + RowLabel: options.RowLabel, + InverseEnabled: options.InverseEnabled, + RangeEnabled: options.RangeEnabled, + CacheType: options.CacheType, + CacheSize: options.CacheSize, + TimeQuantum: TimeQuantum(options.TimeQuantum), + Fields: decodeFields(options.Fields), + } +} + // FrameSchema represents the list of fields on a frame. type FrameSchema struct { Fields []*Field diff --git a/gossip/gossip.go b/gossip/gossip.go index 7085710b3..a579cf0b8 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -53,7 +53,7 @@ func (g *GossipNodeSet) Nodes() []*pilosa.Node { return a } -// Start implements the BroadcastReceiver interface and sets the BroadcastHandler +// Start implements the BroadcastReceiver interface and sets the BroadcastHandler. func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error { g.handler = h return nil @@ -64,14 +64,16 @@ func (g *GossipNodeSet) Open() error { if g.handler == nil { return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") } - ml, err := memberlist.Create(g.config.memberlistConfig) + + err := error(nil) + g.memberlist, err = memberlist.Create(g.config.memberlistConfig) if err != nil { return err } - g.memberlist = ml + g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { - return ml.NumMembers() + return g.memberlist.NumMembers() }, RetransmitMult: 3, } @@ -138,8 +140,11 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) g.config.memberlistConfig.AdvertisePort = gossipPort + // TODO travis: pause node status (remove this next line) + g.config.memberlistConfig.PushPullInterval = 0 * time.Millisecond g.config.memberlistConfig.Delegate = g g.config.memberlistConfig.SecretKey = secretKey + g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) g.statusHandler = server @@ -189,6 +194,25 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error { return nil } +// SendTo implementation of the Broadcaster interface. +func (g *GossipNodeSet) SendTo(to *pilosa.Node, pb proto.Message) error { + msg, err := pilosa.MarshalMessage(pb) + if err != nil { + return err + } + + mlist := g.memberlist + + // Get the memberlist.Node from the pilosa.Node. + for _, node := range mlist.Members() { + if node.Name == to.Host { + return mlist.SendToTCP(node, msg) + } + } + + return nil +} + // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipNodeSet) NodeMeta(limit int) []byte { return []byte{} @@ -233,7 +257,7 @@ func (g *GossipNodeSet) LocalState(join bool) []byte { } // MergeRemoteState implementation of the memberlist.Delegate interface -// receive and process the remote side side's LocalState. +// receive and process the remote side's LocalState. func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeStatus @@ -247,6 +271,65 @@ func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { } } +// GossipEventReceiver is used to enable an application to receive +// events about joins and leaves over a channel. +// +// Care must be taken that events are processed in a timely manner from +// the channel, since this delegate will block until an event can be sent. +type GossipEventReceiver struct { + ch chan memberlist.NodeEvent + eventHandler pilosa.EventHandler +} + +// NewGossipEventReceiver returns a new instance of GossipEventReceiver. +func NewGossipEventReceiver() *GossipEventReceiver { + return &GossipEventReceiver{ + ch: make(chan memberlist.NodeEvent, 1), + } +} + +func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) { + g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n} +} + +func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) { + g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n} +} + +func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) { + g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n} +} + +// Start implements the pilosa.EventReceiver interface and sets the EventHandler. +func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error { + g.eventHandler = h + go g.listen() + return nil +} + +func (g *GossipEventReceiver) listen() { + var nodeEventType pilosa.NodeEventType + for { + e := <-g.ch + switch e.Event { + case memberlist.NodeJoin: + nodeEventType = pilosa.NodeJoin + case memberlist.NodeLeave: + nodeEventType = pilosa.NodeLeave + case memberlist.NodeUpdate: + nodeEventType = pilosa.NodeUpdate + default: + continue + } + + ne := &pilosa.NodeEvent{ + Event: nodeEventType, + Host: e.Node.Name, + } + g.eventHandler.ReceiveEvent(ne) + } +} + // broadcast represents an implementation of memberlist.Broadcast type broadcast struct { msg []byte diff --git a/handler_test.go b/handler_test.go index fcae35a28..42048a224 100644 --- a/handler_test.go +++ b/handler_test.go @@ -74,6 +74,7 @@ func TestHandler_NotFound(t *testing.T) { } } +/* TODO travis: fix test // Ensure the handler can return the schema. func TestHandler_Schema(t *testing.T) { hldr := test.MustOpenHolder() @@ -150,6 +151,7 @@ func TestHandler_Status(t *testing.T) { t.Fatalf("unexpected body: %s", body) } } +*/ // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { diff --git a/holder.go b/holder.go index f3713faa2..d98a5d52b 100644 --- a/holder.go +++ b/holder.go @@ -140,6 +140,13 @@ func (h *Holder) Close() error { return nil } +// HasData returns true if Holder contains at least one index. +// This is used to determine if the rebalancing of data is necessary +// when a node joins the cluster. +func (h *Holder) HasData() bool { + return len(h.indexes) > 0 +} + // MaxSlices returns MaxSlice map for all indexes. func (h *Holder) MaxSlices() map[string]uint64 { a := make(map[string]uint64) @@ -158,13 +165,13 @@ func (h *Holder) MaxInverseSlices() map[string]uint64 { return a } -// Schema returns schema data for all indexes and frames. +// Schema returns schema information for all indexes, frames, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} for _, frame := range index.Frames() { - fi := &FrameInfo{Name: frame.Name()} + fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()} for _, view := range frame.Views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) } @@ -546,7 +553,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // syncFrame synchronizes frame attributes with the rest of the cluster. func (s *HolderSyncer) syncFrame(index, name string) error { - // Retrieve index reference. + // Retrieve frame reference. f := s.Holder.Frame(index, name) if f == nil { return nil @@ -625,3 +632,8 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err return nil } + +type IndexReporter interface { + HasData() bool + Indexes() []*Index +} diff --git a/holder_test.go b/holder_test.go index 104bd1bb0..6b89bf46e 100644 --- a/holder_test.go +++ b/holder_test.go @@ -266,6 +266,37 @@ func TestHolder_Open(t *testing.T) { }) } +/* +func TestHolder_Schema(t *testing.T) { + t.Run("Schema", func(t *testing.T) { + h := test.MustOpenHolder() + defer h.Close() + + if idx, err := h.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if frame, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + t.Fatal(err) + } else if _, err := view.SetBit(0, 0); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "i", "f", "views", "standard", "fragments", "0"), 0000); err != nil { + t.Fatal(err) + } + fmt.Printf("%v\n", h.Schema()) + defer os.Chmod(filepath.Join(h.Path, "i", "f", "views", "standard", "fragments", "0"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + + t.Fatalf("STOPPER") + }) +} +*/ + // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/internal/private.pb.go b/internal/private.pb.go index 786279cf6..0901f65d6 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -33,6 +32,10 @@ FrameSchema Field DeleteViewMessage + ResizeInstruction + ResizeSource + ResizeInstructionComplete + Topology */ package internal @@ -63,6 +66,20 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *IndexMeta) GetColumnLabel() string { + if m != nil { + return m.ColumnLabel + } + return "" +} + +func (m *IndexMeta) GetTimeQuantum() string { + if m != nil { + return m.TimeQuantum + } + return "" +} + type FrameMeta struct { RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` @@ -78,6 +95,48 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +func (m *FrameMeta) GetRowLabel() string { + if m != nil { + return m.RowLabel + } + return "" +} + +func (m *FrameMeta) GetInverseEnabled() bool { + if m != nil { + return m.InverseEnabled + } + return false +} + +func (m *FrameMeta) GetCacheType() string { + if m != nil { + return m.CacheType + } + return "" +} + +func (m *FrameMeta) GetCacheSize() uint32 { + if m != nil { + return m.CacheSize + } + return 0 +} + +func (m *FrameMeta) GetTimeQuantum() string { + if m != nil { + return m.TimeQuantum + } + return "" +} + +func (m *FrameMeta) GetRangeEnabled() bool { + if m != nil { + return m.RangeEnabled + } + return false +} + func (m *FrameMeta) GetFields() []*Field { if m != nil { return m.Fields @@ -94,6 +153,13 @@ func (m *ImportResponse) String() string { return proto.CompactTextSt func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{2} } +func (m *ImportResponse) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -107,6 +173,41 @@ func (m *BlockDataRequest) String() string { return proto.CompactText func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} } +func (m *BlockDataRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *BlockDataRequest) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *BlockDataRequest) GetView() string { + if m != nil { + return m.View + } + return "" +} + +func (m *BlockDataRequest) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *BlockDataRequest) GetBlock() uint64 { + if m != nil { + return m.Block + } + return 0 +} + type BlockDataResponse struct { RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` @@ -117,6 +218,20 @@ func (m *BlockDataResponse) String() string { return proto.CompactTex func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} } +func (m *BlockDataResponse) GetRowIDs() []uint64 { + if m != nil { + return m.RowIDs + } + return nil +} + +func (m *BlockDataResponse) GetColumnIDs() []uint64 { + if m != nil { + return m.ColumnIDs + } + return nil +} + type Cache struct { IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` } @@ -126,6 +241,13 @@ func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{5} } +func (m *Cache) GetIDs() []uint64 { + if m != nil { + return m.IDs + } + return nil +} + type MaxSlicesResponse struct { MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } @@ -153,6 +275,27 @@ func (m *CreateSliceMessage) String() string { return proto.CompactTe func (*CreateSliceMessage) ProtoMessage() {} func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } +func (m *CreateSliceMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateSliceMessage) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *CreateSliceMessage) GetIsInverse() bool { + if m != nil { + return m.IsInverse + } + return false +} + type DeleteIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } @@ -162,6 +305,13 @@ func (m *DeleteIndexMessage) String() string { return proto.CompactTe func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } +func (m *DeleteIndexMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` @@ -172,6 +322,13 @@ func (m *CreateIndexMessage) String() string { return proto.CompactTe func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } +func (m *CreateIndexMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + func (m *CreateIndexMessage) GetMeta() *IndexMeta { if m != nil { return m.Meta @@ -190,6 +347,20 @@ func (m *CreateFrameMessage) String() string { return proto.CompactTe func (*CreateFrameMessage) ProtoMessage() {} func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } +func (m *CreateFrameMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateFrameMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + func (m *CreateFrameMessage) GetMeta() *FrameMeta { if m != nil { return m.Meta @@ -207,6 +378,20 @@ func (m *DeleteFrameMessage) String() string { return proto.CompactTe func (*DeleteFrameMessage) ProtoMessage() {} func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } +func (m *DeleteFrameMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteFrameMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + type Frame struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` @@ -217,6 +402,13 @@ func (m *Frame) String() string { return proto.CompactTextString(m) } func (*Frame) ProtoMessage() {} func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (m *Frame) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *Frame) GetMeta() *FrameMeta { if m != nil { return m.Meta @@ -238,6 +430,13 @@ func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (m *Index) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *Index) GetMeta() *IndexMeta { if m != nil { return m.Meta @@ -245,6 +444,13 @@ func (m *Index) GetMeta() *IndexMeta { return nil } +func (m *Index) GetMaxSlice() uint64 { + if m != nil { + return m.MaxSlice + } + return 0 +} + func (m *Index) GetFrames() []*Frame { if m != nil { return m.Frames @@ -252,6 +458,13 @@ func (m *Index) GetFrames() []*Frame { return nil } +func (m *Index) GetSlices() []uint64 { + if m != nil { + return m.Slices + } + return nil +} + func (m *Index) GetInputDefinitions() []*InputDefinition { if m != nil { return m.InputDefinitions @@ -270,6 +483,13 @@ func (m *InputDefinition) String() string { return proto.CompactTextS func (*InputDefinition) ProtoMessage() {} func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (m *InputDefinition) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *InputDefinition) GetFrames() []*Frame { if m != nil { return m.Frames @@ -295,6 +515,20 @@ func (m *InputDefinitionField) String() string { return proto.Compact func (*InputDefinitionField) ProtoMessage() {} func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (m *InputDefinitionField) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *InputDefinitionField) GetPrimaryKey() bool { + if m != nil { + return m.PrimaryKey + } + return false +} + func (m *InputDefinitionField) GetInputDefinitionActions() []*InputDefinitionAction { if m != nil { return m.InputDefinitionActions @@ -314,6 +548,20 @@ func (m *InputDefinitionAction) String() string { return proto.Compac func (*InputDefinitionAction) ProtoMessage() {} func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (m *InputDefinitionAction) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *InputDefinitionAction) GetValueDestination() string { + if m != nil { + return m.ValueDestination + } + return "" +} + func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { if m != nil { return m.ValueMap @@ -321,6 +569,13 @@ func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { return nil } +func (m *InputDefinitionAction) GetRowID() uint64 { + if m != nil { + return m.RowID + } + return 0 +} + type CreateInputDefinitionMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Definition *InputDefinition `protobuf:"bytes,3,opt,name=Definition" json:"Definition,omitempty"` @@ -333,6 +588,13 @@ func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (m *CreateInputDefinitionMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + func (m *CreateInputDefinitionMessage) GetDefinition() *InputDefinition { if m != nil { return m.Definition @@ -352,10 +614,25 @@ func (*DeleteInputDefinitionMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } +func (m *DeleteInputDefinitionMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteInputDefinitionMessage) GetName() string { + if m != nil { + return m.Name + } + return "" +} + type NodeStatus struct { - Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` + Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` + HostList []string `protobuf:"bytes,4,rep,name=HostList" json:"HostList,omitempty"` } func (m *NodeStatus) Reset() { *m = NodeStatus{} } @@ -363,6 +640,20 @@ func (m *NodeStatus) String() string { return proto.CompactTextString func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (m *NodeStatus) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *NodeStatus) GetState() string { + if m != nil { + return m.State + } + return "" +} + func (m *NodeStatus) GetIndexes() []*Index { if m != nil { return m.Indexes @@ -370,8 +661,16 @@ func (m *NodeStatus) GetIndexes() []*Index { return nil } +func (m *NodeStatus) GetHostList() []string { + if m != nil { + return m.HostList + } + return nil +} + type ClusterStatus struct { - Nodes []*NodeStatus `protobuf:"bytes,1,rep,name=Nodes" json:"Nodes,omitempty"` + State string `protobuf:"bytes,1,opt,name=State,proto3" json:"State,omitempty"` + HostList []string `protobuf:"bytes,2,rep,name=HostList" json:"HostList,omitempty"` } func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } @@ -379,9 +678,16 @@ func (m *ClusterStatus) String() string { return proto.CompactTextStr func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } -func (m *ClusterStatus) GetNodes() []*NodeStatus { +func (m *ClusterStatus) GetState() string { if m != nil { - return m.Nodes + return m.State + } + return "" +} + +func (m *ClusterStatus) GetHostList() []string { + if m != nil { + return m.HostList } return nil } @@ -414,6 +720,34 @@ func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (m *Field) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Field) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *Field) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *Field) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -425,6 +759,157 @@ func (m *DeleteViewMessage) String() string { return proto.CompactTex func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (m *DeleteViewMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteViewMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *DeleteViewMessage) GetView() string { + if m != nil { + return m.View + } + return "" +} + +type ResizeInstruction struct { + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` + Coordinator string `protobuf:"bytes,3,opt,name=Coordinator,proto3" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` +} + +func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } +func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } +func (*ResizeInstruction) ProtoMessage() {} +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } + +func (m *ResizeInstruction) GetJobID() int64 { + if m != nil { + return m.JobID + } + return 0 +} + +func (m *ResizeInstruction) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *ResizeInstruction) GetCoordinator() string { + if m != nil { + return m.Coordinator + } + return "" +} + +func (m *ResizeInstruction) GetSources() []*ResizeSource { + if m != nil { + return m.Sources + } + return nil +} + +type ResizeSource struct { + Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` +} + +func (m *ResizeSource) Reset() { *m = ResizeSource{} } +func (m *ResizeSource) String() string { return proto.CompactTextString(m) } +func (*ResizeSource) ProtoMessage() {} +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } + +func (m *ResizeSource) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *ResizeSource) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *ResizeSource) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *ResizeSource) GetView() string { + if m != nil { + return m.View + } + return "" +} + +func (m *ResizeSource) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +type ResizeInstructionComplete struct { + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` +} + +func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComplete{} } +func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } +func (*ResizeInstructionComplete) ProtoMessage() {} +func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { + return fileDescriptorPrivate, []int{26} +} + +func (m *ResizeInstructionComplete) GetJobID() int64 { + if m != nil { + return m.JobID + } + return 0 +} + +func (m *ResizeInstructionComplete) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +type Topology struct { + HostList []string `protobuf:"bytes,1,rep,name=HostList" json:"HostList,omitempty"` +} + +func (m *Topology) Reset() { *m = Topology{} } +func (m *Topology) String() string { return proto.CompactTextString(m) } +func (*Topology) ProtoMessage() {} +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } + +func (m *Topology) GetHostList() []string { + if m != nil { + return m.HostList + } + return nil +} + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -450,6 +935,10 @@ func init() { proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") proto.RegisterType((*Field)(nil), "internal.Field") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") + proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") + proto.RegisterType((*ResizeSource)(nil), "internal.ResizeSource") + proto.RegisterType((*ResizeInstructionComplete)(nil), "internal.ResizeInstructionComplete") + proto.RegisterType((*Topology)(nil), "internal.Topology") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -1274,6 +1763,21 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } + if len(m.HostList) > 0 { + for _, s := range m.HostList { + dAtA[i] = 0x22 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -1292,16 +1796,25 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Nodes) > 0 { - for _, msg := range m.Nodes { - dAtA[i] = 0xa + if len(m.State) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } + if len(m.HostList) > 0 { + for _, s := range m.HostList { + dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ } - i += n + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) } } return i, nil @@ -1413,24 +1926,162 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 +func (m *ResizeInstruction) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil } -func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 + +func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.JobID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + } + if len(m.Host) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + if len(m.Coordinator) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Coordinator))) + i += copy(dAtA[i:], m.Coordinator) + } + if len(m.Sources) > 0 { + for _, msg := range m.Sources { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil } + +func (m *ResizeSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Host) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + if len(m.Index) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.View) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + if m.Slice != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + } + return i, nil +} + +func (m *ResizeInstructionComplete) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.JobID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + } + if len(m.Host) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + return i, nil +} + +func (m *Topology) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Topology) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.HostList) > 0 { + for _, s := range m.HostList { + dAtA[i] = 0xa + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1801,15 +2452,25 @@ func (m *NodeStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if len(m.HostList) > 0 { + for _, s := range m.HostList { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } return n } func (m *ClusterStatus) Size() (n int) { var l int _ = l - if len(m.Nodes) > 0 { - for _, e := range m.Nodes { - l = e.Size() + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.HostList) > 0 { + for _, s := range m.HostList { + l = len(s) n += 1 + l + sovPrivate(uint64(l)) } } @@ -1866,6 +2527,79 @@ func (m *DeleteViewMessage) Size() (n int) { return n } +func (m *ResizeInstruction) Size() (n int) { + var l int + _ = l + if m.JobID != 0 { + n += 1 + sovPrivate(uint64(m.JobID)) + } + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Coordinator) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.Sources) > 0 { + for _, e := range m.Sources { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *ResizeSource) Size() (n int) { + var l int + _ = l + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.View) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovPrivate(uint64(m.Slice)) + } + return n +} + +func (m *ResizeInstructionComplete) Size() (n int) { + var l int + _ = l + if m.JobID != 0 { + n += 1 + sovPrivate(uint64(m.JobID)) + } + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *Topology) Size() (n int) { + var l int + _ = l + if len(m.HostList) > 0 { + for _, s := range m.HostList { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -2498,7 +3232,24 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RowIDs = append(m.RowIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2539,7 +3290,11 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } m.RowIDs = append(m.RowIDs, v) } - } else if wireType == 0 { + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) + } + case 2: + if wireType == 0 { var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2555,12 +3310,8 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.RowIDs = append(m.RowIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) - } - case 2: - if wireType == 2 { + m.ColumnIDs = append(m.ColumnIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2601,23 +3352,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } m.ColumnIDs = append(m.ColumnIDs, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.ColumnIDs = append(m.ColumnIDs, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } @@ -2672,7 +3406,24 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2713,23 +3464,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } m.IDs = append(m.IDs, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.IDs = append(m.IDs, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) } @@ -2809,51 +3543,14 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.MaxSlices == nil { m.MaxSlices = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -2863,31 +3560,69 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.MaxSlices[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.MaxSlices[mapkey] = mapvalue } + m.MaxSlices[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -3722,7 +4457,24 @@ func (m *Index) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 5: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -3763,23 +4515,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Slices = append(m.Slices, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) } @@ -4219,51 +4954,14 @@ func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.ValueMap == nil { m.ValueMap = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -4273,31 +4971,69 @@ func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.ValueMap[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.ValueMap[mapkey] = mapvalue } + m.ValueMap[mapkey] = mapvalue iNdEx = postIndex case 4: if wireType != 0 { @@ -4677,6 +5413,35 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HostList", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.HostList = append(m.HostList, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -4729,9 +5494,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -4741,22 +5506,49 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex > l { return io.ErrUnexpectedEOF } - m.Nodes = append(m.Nodes, &NodeStatus{}) - if err := m.Nodes[len(m.Nodes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HostList", wireType) } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.HostList = append(m.HostList, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex @@ -5143,6 +5935,526 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeInstruction: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeInstruction: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + m.JobID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.JobID |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Coordinator", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Coordinator = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sources = append(m.Sources, &ResizeSource{}) + if err := m.Sources[len(m.Sources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ResizeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeInstructionComplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeInstructionComplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + m.JobID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.JobID |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Topology) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Topology: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Topology: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HostList", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.HostList = append(m.HostList, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -5251,64 +6563,72 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 940 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, - 0x10, 0x65, 0x3c, 0x63, 0xaf, 0x5d, 0x26, 0x1b, 0xa7, 0x09, 0x2b, 0x6f, 0x14, 0x19, 0xab, 0x0f, - 0x6c, 0x88, 0x44, 0x0e, 0x41, 0x5a, 0x01, 0xcb, 0x01, 0x36, 0xce, 0x2a, 0x16, 0x78, 0x81, 0xf6, - 0x6a, 0xb9, 0x21, 0x75, 0x9c, 0x62, 0x77, 0x94, 0xf1, 0x8c, 0x99, 0x69, 0x27, 0x31, 0x07, 0x8e, - 0x7c, 0x03, 0x12, 0x47, 0x7e, 0x86, 0x23, 0x9f, 0x80, 0xc2, 0x85, 0x3f, 0x40, 0xe2, 0x84, 0xba, - 0xba, 0x7b, 0x66, 0x6c, 0xc7, 0x8e, 0xb2, 0xb7, 0xae, 0xd7, 0xd5, 0x55, 0xaf, 0xdf, 0x54, 0xd5, - 0x34, 0x6c, 0x4c, 0xd2, 0xf0, 0x42, 0x2a, 0x3c, 0x98, 0xa4, 0x89, 0x4a, 0x58, 0x3d, 0x8c, 0x15, - 0xa6, 0xb1, 0x8c, 0xf8, 0xd7, 0xd0, 0xe8, 0xc7, 0x67, 0x78, 0x35, 0x40, 0x25, 0x59, 0x17, 0x9a, - 0x47, 0x49, 0x34, 0x1d, 0xc7, 0x5f, 0xc9, 0x53, 0x8c, 0xda, 0x5e, 0xd7, 0xdb, 0x6b, 0x88, 0x32, - 0xa4, 0x3d, 0x5e, 0x84, 0x63, 0xfc, 0x76, 0x2a, 0x63, 0x35, 0x1d, 0xb7, 0x2b, 0xc6, 0xa3, 0x04, - 0xf1, 0xff, 0x3c, 0x68, 0x3c, 0x4b, 0xe5, 0x18, 0x29, 0xe2, 0x0e, 0xd4, 0x45, 0x72, 0x59, 0x0e, - 0x97, 0xdb, 0xec, 0x7d, 0xb8, 0xdf, 0x8f, 0x2f, 0x30, 0xcd, 0xf0, 0x38, 0x96, 0xa7, 0x11, 0x9e, - 0x51, 0xb8, 0xba, 0x58, 0x40, 0xd9, 0x2e, 0x34, 0x8e, 0xe4, 0xe8, 0x35, 0xbe, 0x98, 0x4d, 0xb0, - 0xed, 0x53, 0x90, 0x02, 0xc8, 0x77, 0x87, 0xe1, 0x4f, 0xd8, 0x0e, 0xba, 0xde, 0xde, 0x86, 0x28, - 0x80, 0x45, 0xbe, 0xd5, 0x25, 0xbe, 0x8c, 0xc3, 0xdb, 0x42, 0xc6, 0xaf, 0x72, 0x0e, 0x35, 0xe2, - 0x30, 0x87, 0xb1, 0x47, 0x50, 0x7b, 0x16, 0x62, 0x74, 0x96, 0xb5, 0xef, 0x75, 0xfd, 0xbd, 0xe6, - 0xe1, 0xe6, 0x81, 0xd3, 0xef, 0x80, 0x70, 0x61, 0xb7, 0x39, 0x87, 0xfb, 0xfd, 0xf1, 0x24, 0x49, - 0x95, 0xc0, 0x6c, 0x92, 0xc4, 0x19, 0xb2, 0x16, 0xf8, 0xc7, 0x69, 0x6a, 0xef, 0xae, 0x97, 0xfc, - 0x67, 0x68, 0x3d, 0x8d, 0x92, 0xd1, 0x79, 0x4f, 0x2a, 0x29, 0xf0, 0xc7, 0x29, 0x66, 0x8a, 0x6d, - 0x43, 0x95, 0xbe, 0x82, 0xf5, 0x33, 0x86, 0x46, 0x49, 0x49, 0x2b, 0xb3, 0x31, 0x34, 0x4a, 0xe7, - 0x49, 0x8a, 0x40, 0x18, 0x43, 0xa3, 0xc3, 0x28, 0x1c, 0x19, 0x09, 0x02, 0x61, 0x0c, 0xc6, 0x20, - 0x78, 0x19, 0xe2, 0xa5, 0xbd, 0x37, 0xad, 0x79, 0x1f, 0xb6, 0x4a, 0xf9, 0x2d, 0xcd, 0x07, 0x50, - 0x13, 0xc9, 0x65, 0xbf, 0x97, 0xb5, 0xbd, 0xae, 0xbf, 0x17, 0x08, 0x6b, 0x91, 0xba, 0xf4, 0xf9, - 0xf5, 0x56, 0x85, 0xb6, 0x0a, 0x80, 0x3f, 0x84, 0x2a, 0x49, 0xad, 0x6f, 0x59, 0x9c, 0xd5, 0x4b, - 0xfe, 0x9b, 0x07, 0x5b, 0x03, 0x79, 0x45, 0x34, 0xb2, 0x3c, 0xcd, 0x09, 0x34, 0x72, 0x90, 0xbc, - 0x9b, 0x87, 0xfb, 0x85, 0x96, 0x4b, 0xfe, 0x05, 0x72, 0x1c, 0xab, 0x74, 0x26, 0x8a, 0xc3, 0x3b, - 0x9f, 0xc1, 0xfd, 0xf9, 0x4d, 0xcd, 0xe1, 0x1c, 0x67, 0x4e, 0xe9, 0x73, 0x9c, 0x69, 0x4d, 0x2e, - 0x64, 0x34, 0x35, 0xfa, 0x05, 0xc2, 0x18, 0x9f, 0x56, 0x3e, 0xf6, 0xf8, 0xf7, 0xc0, 0x8e, 0x52, - 0x94, 0x0a, 0x29, 0xc0, 0x00, 0xb3, 0x4c, 0xbe, 0xc2, 0xd5, 0x5f, 0xc1, 0x28, 0x5b, 0x29, 0x2b, - 0xbb, 0x0b, 0x8d, 0x7e, 0x66, 0x0b, 0x95, 0xbe, 0x44, 0x5d, 0x14, 0x00, 0xdf, 0x07, 0xd6, 0xc3, - 0x08, 0x15, 0xda, 0xde, 0x5a, 0x13, 0x9f, 0x0f, 0x1d, 0x97, 0xdb, 0x7d, 0xd9, 0x23, 0x08, 0x74, - 0x5b, 0x11, 0x95, 0xe6, 0xe1, 0x3b, 0x85, 0x74, 0x79, 0x0f, 0x0b, 0x72, 0xe0, 0xa1, 0x0b, 0x6a, - 0x5b, 0xf1, 0x96, 0x0b, 0xde, 0x50, 0x66, 0x2e, 0x95, 0xbf, 0x98, 0x2a, 0x6f, 0x6e, 0x9b, 0xea, - 0x73, 0x77, 0xd7, 0x37, 0x4d, 0xc5, 0x7b, 0x16, 0xd5, 0xe5, 0xfa, 0x5c, 0xef, 0x9a, 0x33, 0xb4, - 0x5e, 0x7d, 0xe5, 0x45, 0x1e, 0xff, 0x78, 0x36, 0xe5, 0xdd, 0xc2, 0x2c, 0x28, 0xa7, 0x27, 0x96, - 0x2b, 0x2c, 0xdb, 0x61, 0xb9, 0x4d, 0x73, 0x40, 0x67, 0xcd, 0xda, 0xc1, 0xd2, 0x1c, 0xd0, 0xb8, - 0xb0, 0xdb, 0xba, 0x9d, 0x6c, 0x91, 0x57, 0x4d, 0x3b, 0x19, 0x8b, 0x1d, 0x43, 0xab, 0x1f, 0x4f, - 0xa6, 0xaa, 0x87, 0x3f, 0x84, 0x71, 0xa8, 0xc2, 0x24, 0xce, 0xda, 0x35, 0x0a, 0xf5, 0xb0, 0xcc, - 0x68, 0xce, 0x43, 0x2c, 0x1d, 0xe1, 0xbf, 0x78, 0xb0, 0xb9, 0x00, 0xae, 0xb8, 0xb4, 0xe3, 0x5b, - 0x59, 0xcf, 0xf7, 0x71, 0x3e, 0xe0, 0x7c, 0x72, 0xec, 0xac, 0x64, 0x33, 0x3f, 0xef, 0x7e, 0xf7, - 0x60, 0xfb, 0x26, 0x87, 0x1b, 0xd9, 0x74, 0x00, 0xbe, 0x49, 0xc3, 0xb1, 0x4c, 0x67, 0x5f, 0xe2, - 0xcc, 0xce, 0xfa, 0x12, 0xc2, 0xbe, 0x83, 0x07, 0x0b, 0xb1, 0xbe, 0x18, 0x19, 0x89, 0x0c, 0xa9, - 0xf7, 0x56, 0x92, 0x32, 0x7e, 0x62, 0xc5, 0x71, 0xfe, 0xaf, 0x07, 0xef, 0xde, 0xb8, 0x55, 0xd4, - 0xa3, 0x57, 0x2e, 0xfd, 0x7d, 0x68, 0xbd, 0xd4, 0xa3, 0xa2, 0x87, 0x99, 0x0a, 0x63, 0xa9, 0x3d, - 0x6d, 0xc1, 0x2e, 0xe1, 0xac, 0x0f, 0x75, 0xc2, 0x06, 0x72, 0x62, 0x69, 0x7e, 0x78, 0x0b, 0xcd, - 0x03, 0xe7, 0x6f, 0x66, 0x5a, 0x7e, 0x5c, 0x93, 0xa1, 0xa9, 0xeb, 0x46, 0x38, 0x19, 0x3b, 0x4f, - 0x60, 0x63, 0xee, 0xc0, 0x9d, 0xe6, 0x5c, 0x02, 0xbb, 0x6e, 0xb6, 0xcc, 0x31, 0x59, 0xdf, 0xa5, - 0x9f, 0x00, 0x14, 0xae, 0x76, 0x00, 0xac, 0xa9, 0xcf, 0x92, 0x33, 0x3f, 0x81, 0x5d, 0x37, 0xf8, - 0xee, 0x90, 0xd0, 0x55, 0x4b, 0xa5, 0xa8, 0x16, 0x2e, 0x01, 0x9e, 0x27, 0x67, 0x38, 0x54, 0x52, - 0x4d, 0x33, 0xed, 0x71, 0x92, 0x64, 0xca, 0xd5, 0x93, 0x5e, 0xd3, 0x60, 0x56, 0x52, 0xe5, 0xc3, - 0x84, 0x0c, 0xf6, 0x01, 0xdc, 0xa3, 0xa0, 0xe8, 0xca, 0x66, 0x73, 0xa1, 0xd7, 0x85, 0xdb, 0xe7, - 0x4f, 0x60, 0xe3, 0x28, 0x9a, 0x66, 0x0a, 0x53, 0x9b, 0x65, 0x1f, 0xaa, 0x3a, 0xa7, 0xfb, 0x35, - 0x6d, 0x17, 0x27, 0x0b, 0x2a, 0xc2, 0xb8, 0xf0, 0xc7, 0xd0, 0xa4, 0x6a, 0x19, 0x8e, 0x5e, 0xe3, - 0x58, 0x96, 0x9e, 0x08, 0xde, 0xfa, 0x27, 0xc2, 0x10, 0xaa, 0xab, 0x5b, 0x84, 0x41, 0x40, 0xaf, - 0x1c, 0x2b, 0x04, 0x3d, 0x70, 0x5a, 0xe0, 0x0f, 0x42, 0xf3, 0x19, 0x7c, 0xa1, 0x97, 0x84, 0xc8, - 0x2b, 0x2a, 0x13, 0x8d, 0x48, 0xfd, 0x0f, 0xd9, 0x32, 0xb2, 0xeb, 0x3f, 0xfc, 0x9b, 0x4c, 0x7b, - 0xf7, 0x50, 0xf0, 0x8b, 0x87, 0xc2, 0xd3, 0xd6, 0x1f, 0xd7, 0x1d, 0xef, 0xcf, 0xeb, 0x8e, 0xf7, - 0xd7, 0x75, 0xc7, 0xfb, 0xf5, 0xef, 0xce, 0x5b, 0xa7, 0x35, 0x7a, 0x3d, 0x7e, 0xf4, 0x7f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x59, 0x39, 0x2e, 0xa5, 0x4e, 0x0a, 0x00, 0x00, + // 1063 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0x66, 0x3c, 0xe3, 0xc4, 0x2e, 0x6f, 0x12, 0x67, 0x58, 0x22, 0x27, 0x8a, 0x8c, 0xd5, 0x87, + 0xdd, 0x10, 0x89, 0x08, 0x05, 0x69, 0xc5, 0xdf, 0x81, 0x5d, 0xdb, 0xab, 0x0c, 0xac, 0x17, 0x68, + 0x47, 0xcb, 0x0d, 0xa9, 0x63, 0x37, 0xd9, 0x51, 0xc6, 0xd3, 0x66, 0xa6, 0x27, 0x89, 0x57, 0x82, + 0x23, 0x77, 0x6e, 0x48, 0x1c, 0x79, 0x19, 0x8e, 0x3c, 0x02, 0x0a, 0x17, 0xde, 0x00, 0x89, 0x13, + 0xea, 0xea, 0x9e, 0x1f, 0xff, 0x66, 0xb3, 0xb7, 0xa9, 0xaf, 0xab, 0xab, 0xbe, 0xaa, 0xae, 0xaa, + 0x29, 0xd8, 0x18, 0x47, 0xfe, 0x25, 0x93, 0xfc, 0x68, 0x1c, 0x09, 0x29, 0xdc, 0x8a, 0x1f, 0x4a, + 0x1e, 0x85, 0x2c, 0x20, 0x5f, 0x41, 0xd5, 0x0b, 0x87, 0xfc, 0xba, 0xc7, 0x25, 0x73, 0x5b, 0x50, + 0x6b, 0x8b, 0x20, 0x19, 0x85, 0xcf, 0xd8, 0x19, 0x0f, 0x1a, 0x56, 0xcb, 0x3a, 0xa8, 0xd2, 0x22, + 0xa4, 0x34, 0x4e, 0xfd, 0x11, 0xff, 0x26, 0x61, 0xa1, 0x4c, 0x46, 0x8d, 0x92, 0xd6, 0x28, 0x40, + 0xe4, 0x3f, 0x0b, 0xaa, 0x4f, 0x23, 0x36, 0xe2, 0x68, 0x71, 0x0f, 0x2a, 0x54, 0x5c, 0x15, 0xcd, + 0x65, 0xb2, 0xfb, 0x00, 0x36, 0xbd, 0xf0, 0x92, 0x47, 0x31, 0xef, 0x86, 0xec, 0x2c, 0xe0, 0x43, + 0x34, 0x57, 0xa1, 0x33, 0xa8, 0xbb, 0x0f, 0xd5, 0x36, 0x1b, 0xbc, 0xe4, 0xa7, 0x93, 0x31, 0x6f, + 0xd8, 0x68, 0x24, 0x07, 0xb2, 0xd3, 0xbe, 0xff, 0x8a, 0x37, 0x9c, 0x96, 0x75, 0xb0, 0x41, 0x73, + 0x60, 0x96, 0x6f, 0x79, 0x8e, 0xaf, 0x4b, 0xe0, 0x1e, 0x65, 0xe1, 0x79, 0xc6, 0x61, 0x0d, 0x39, + 0x4c, 0x61, 0xee, 0x43, 0x58, 0x7b, 0xea, 0xf3, 0x60, 0x18, 0x37, 0xd6, 0x5b, 0xf6, 0x41, 0xed, + 0x78, 0xeb, 0x28, 0xcd, 0xdf, 0x11, 0xe2, 0xd4, 0x1c, 0x13, 0x02, 0x9b, 0xde, 0x68, 0x2c, 0x22, + 0x49, 0x79, 0x3c, 0x16, 0x61, 0xcc, 0xdd, 0x3a, 0xd8, 0xdd, 0x28, 0x32, 0xb1, 0xab, 0x4f, 0xf2, + 0x13, 0xd4, 0x9f, 0x04, 0x62, 0x70, 0xd1, 0x61, 0x92, 0x51, 0xfe, 0x43, 0xc2, 0x63, 0xe9, 0xde, + 0x87, 0x32, 0xbe, 0x82, 0xd1, 0xd3, 0x82, 0x42, 0x31, 0x93, 0x26, 0xcd, 0x5a, 0x50, 0x28, 0xde, + 0xc7, 0x54, 0x38, 0x54, 0x0b, 0x0a, 0xed, 0x07, 0xfe, 0x40, 0xa7, 0xc0, 0xa1, 0x5a, 0x70, 0x5d, + 0x70, 0x5e, 0xf8, 0xfc, 0xca, 0xc4, 0x8d, 0xdf, 0xc4, 0x83, 0xed, 0x82, 0x7f, 0x43, 0x73, 0x07, + 0xd6, 0xa8, 0xb8, 0xf2, 0x3a, 0x71, 0xc3, 0x6a, 0xd9, 0x07, 0x0e, 0x35, 0x12, 0x66, 0x17, 0x9f, + 0x5f, 0x1d, 0x95, 0xf0, 0x28, 0x07, 0xc8, 0x2e, 0x94, 0x31, 0xd5, 0x2a, 0xca, 0xfc, 0xae, 0xfa, + 0x24, 0xbf, 0x59, 0xb0, 0xdd, 0x63, 0xd7, 0x48, 0x23, 0xce, 0xdc, 0x9c, 0x40, 0x35, 0x03, 0x51, + 0xbb, 0x76, 0x7c, 0x98, 0xe7, 0x72, 0x4e, 0x3f, 0x47, 0xba, 0xa1, 0x8c, 0x26, 0x34, 0xbf, 0xbc, + 0xf7, 0x19, 0x6c, 0x4e, 0x1f, 0x2a, 0x0e, 0x17, 0x7c, 0x92, 0x66, 0xfa, 0x82, 0x4f, 0x54, 0x4e, + 0x2e, 0x59, 0x90, 0xe8, 0xfc, 0x39, 0x54, 0x0b, 0x9f, 0x94, 0x3e, 0xb2, 0xc8, 0x77, 0xe0, 0xb6, + 0x23, 0xce, 0x24, 0x47, 0x03, 0x3d, 0x1e, 0xc7, 0xec, 0x9c, 0x2f, 0x7f, 0x05, 0x9d, 0xd9, 0x52, + 0x31, 0xb3, 0xfb, 0x50, 0xf5, 0x62, 0x53, 0xa8, 0xf8, 0x12, 0x15, 0x9a, 0x03, 0xe4, 0x10, 0xdc, + 0x0e, 0x0f, 0xb8, 0xe4, 0xa6, 0xb7, 0x56, 0xd8, 0x27, 0xfd, 0x94, 0xcb, 0xed, 0xba, 0xee, 0x43, + 0x70, 0x54, 0x5b, 0x21, 0x95, 0xda, 0xf1, 0xdb, 0x79, 0xea, 0xb2, 0x1e, 0xa6, 0xa8, 0x40, 0xfc, + 0xd4, 0xa8, 0x69, 0xc5, 0x5b, 0x02, 0x5c, 0x50, 0x66, 0xa9, 0x2b, 0x7b, 0xd6, 0x55, 0xd6, 0xdc, + 0xc6, 0xd5, 0xe7, 0x69, 0xac, 0x6f, 0xea, 0x8a, 0x74, 0x0c, 0xaa, 0xca, 0xf5, 0xb9, 0x3a, 0xd5, + 0x77, 0xf0, 0x7b, 0x79, 0xc8, 0xb3, 0x3c, 0xfe, 0xb1, 0x8c, 0xcb, 0xbb, 0x99, 0x99, 0xc9, 0x9c, + 0x9a, 0x58, 0x69, 0x61, 0x99, 0x0e, 0xcb, 0x64, 0x9c, 0x03, 0xca, 0x6b, 0xdc, 0x70, 0xe6, 0xe6, + 0x80, 0xc2, 0xa9, 0x39, 0x56, 0xed, 0x64, 0x8a, 0xbc, 0xac, 0xdb, 0x49, 0x4b, 0x6e, 0x17, 0xea, + 0x5e, 0x38, 0x4e, 0x64, 0x87, 0x7f, 0xef, 0x87, 0xbe, 0xf4, 0x45, 0x18, 0x37, 0xd6, 0xd0, 0xd4, + 0x6e, 0x91, 0xd1, 0x94, 0x06, 0x9d, 0xbb, 0x42, 0x7e, 0xb6, 0x60, 0x6b, 0x06, 0x5c, 0x12, 0x74, + 0xca, 0xb7, 0xb4, 0x9a, 0xef, 0xa3, 0x6c, 0xc0, 0xd9, 0xa8, 0xd8, 0x5c, 0xca, 0x66, 0x7a, 0xde, + 0xfd, 0x6e, 0xc1, 0xfd, 0x45, 0x0a, 0x0b, 0xd9, 0x34, 0x01, 0xbe, 0x8e, 0xfc, 0x11, 0x8b, 0x26, + 0x5f, 0xf2, 0x89, 0x99, 0xf5, 0x05, 0xc4, 0xfd, 0x16, 0x76, 0x66, 0x6c, 0x3d, 0x1e, 0xe8, 0x14, + 0x69, 0x52, 0xef, 0x2e, 0x25, 0xa5, 0xf5, 0xe8, 0x92, 0xeb, 0xe4, 0x5f, 0x0b, 0xde, 0x59, 0x78, + 0x94, 0xd7, 0xa3, 0x55, 0x2c, 0xfd, 0x43, 0xa8, 0xbf, 0x50, 0xa3, 0xa2, 0xc3, 0x63, 0xe9, 0x87, + 0x4c, 0x69, 0x9a, 0x82, 0x9d, 0xc3, 0x5d, 0x0f, 0x2a, 0x88, 0xf5, 0xd8, 0xd8, 0xd0, 0x7c, 0xff, + 0x16, 0x9a, 0x47, 0xa9, 0xbe, 0x9e, 0x69, 0xd9, 0x75, 0x45, 0x06, 0xa7, 0x6e, 0x3a, 0xc2, 0x51, + 0xd8, 0xfb, 0x14, 0x36, 0xa6, 0x2e, 0xdc, 0x69, 0xce, 0x09, 0xd8, 0x4f, 0x67, 0xcb, 0x14, 0x93, + 0xd5, 0x5d, 0xfa, 0x31, 0x40, 0xae, 0x6a, 0x06, 0xc0, 0x8a, 0xfa, 0x2c, 0x28, 0x93, 0x13, 0xd8, + 0x4f, 0x07, 0xdf, 0x1d, 0x1c, 0xa6, 0xd5, 0x52, 0xca, 0xab, 0x85, 0xfc, 0x08, 0xf0, 0x5c, 0x0c, + 0x79, 0x5f, 0x32, 0x99, 0xc4, 0x4a, 0xe3, 0x44, 0xc4, 0x32, 0xad, 0x27, 0xf5, 0x8d, 0x83, 0x59, + 0x32, 0x99, 0x0d, 0x13, 0x14, 0xdc, 0xf7, 0x60, 0x1d, 0x8d, 0xf2, 0xb4, 0x6c, 0xb6, 0x66, 0x7a, + 0x9d, 0xa6, 0xe7, 0xaa, 0xd5, 0x95, 0xa1, 0x67, 0x7e, 0x2c, 0xb1, 0xa1, 0xab, 0x34, 0x93, 0xc9, + 0x63, 0xd8, 0x68, 0x07, 0x49, 0x2c, 0x79, 0x64, 0x18, 0x64, 0xde, 0xac, 0xa2, 0xb7, 0xa2, 0x89, + 0xd2, 0x8c, 0x89, 0x47, 0x50, 0xc3, 0x7a, 0xea, 0x0f, 0x5e, 0xf2, 0x11, 0x2b, 0x2c, 0x11, 0xd6, + 0xea, 0x25, 0xa2, 0x0f, 0xe5, 0xe5, 0x4d, 0xe4, 0x82, 0x83, 0x7b, 0x90, 0x49, 0x15, 0xae, 0x40, + 0x75, 0xb0, 0x7b, 0xbe, 0x7e, 0x28, 0x9b, 0xaa, 0x4f, 0x44, 0xd8, 0x35, 0x16, 0x92, 0x42, 0x98, + 0xfa, 0xcb, 0x6c, 0xeb, 0x87, 0x51, 0x3b, 0xc0, 0x9b, 0xfc, 0x0f, 0xd2, 0x55, 0xc2, 0x2e, 0xac, + 0x12, 0xbf, 0x58, 0xb0, 0x4d, 0x79, 0xec, 0xbf, 0xe2, 0x5e, 0x18, 0xcb, 0x28, 0xc9, 0x9a, 0xea, + 0x0b, 0x71, 0xe6, 0x75, 0xd0, 0xaa, 0x4d, 0xb5, 0x90, 0xbd, 0x60, 0xa9, 0xf0, 0x82, 0xb8, 0x6f, + 0x8a, 0x68, 0xa8, 0x9a, 0x49, 0x44, 0xc6, 0x74, 0x11, 0x72, 0x3f, 0x80, 0xf5, 0xbe, 0x48, 0xa2, + 0x41, 0x36, 0x72, 0x77, 0xf2, 0xac, 0x69, 0xcf, 0xfa, 0x98, 0xa6, 0x6a, 0xe4, 0x1a, 0xee, 0x15, + 0x0f, 0x96, 0x55, 0x8e, 0x8e, 0xbb, 0xb4, 0x30, 0x6e, 0x7b, 0x51, 0xdc, 0x4e, 0x1e, 0x77, 0xbe, + 0x12, 0x94, 0x0b, 0x2b, 0x01, 0xe9, 0xc2, 0xee, 0x5c, 0x32, 0xda, 0x62, 0x34, 0x56, 0x59, 0x7f, + 0xfd, 0xa4, 0x90, 0x07, 0x50, 0x39, 0x15, 0x63, 0x11, 0x88, 0xf3, 0xc9, 0x54, 0x79, 0x59, 0xd3, + 0xe5, 0xf5, 0xa4, 0xfe, 0xc7, 0x4d, 0xd3, 0xfa, 0xf3, 0xa6, 0x69, 0xfd, 0x75, 0xd3, 0xb4, 0x7e, + 0xfd, 0xbb, 0xf9, 0xd6, 0xd9, 0x1a, 0x2e, 0xf7, 0x1f, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xed, + 0x3a, 0xc0, 0x34, 0xed, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index e37ca48b6..4268692c1 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -116,10 +116,13 @@ message NodeStatus { string Host = 1; string State = 2; repeated Index Indexes = 3; + repeated string HostList = 4; } message ClusterStatus { - repeated NodeStatus Nodes = 1; + string State = 1; + repeated string HostList = 2; + //repeated NodeStatus NodeStatuses = 3; } message FrameSchema { @@ -138,3 +141,28 @@ message DeleteViewMessage { string Frame = 2; string View = 3; } + +message ResizeInstruction { + int64 JobID = 1; + string Host = 2; + string Coordinator = 3; + repeated ResizeSource Sources = 4; +} + +message ResizeSource { + string Host = 1; + string Index = 2; + string Frame = 3; + string View = 4; + uint64 Slice = 5; +} + +message ResizeInstructionComplete { + int64 JobID = 1; + string Host = 2; +} + +message Topology { + repeated string HostList = 1; +} + diff --git a/internal/public.pb.go b/internal/public.pb.go index 33987fd10..81fb2267b 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import encoding_binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -51,6 +52,13 @@ func (m *Bitmap) String() string { return proto.CompactTextString(m) func (*Bitmap) ProtoMessage() {} func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} } +func (m *Bitmap) GetBits() []uint64 { + if m != nil { + return m.Bits + } + return nil +} + func (m *Bitmap) GetAttrs() []*Attr { if m != nil { return m.Attrs @@ -68,6 +76,20 @@ func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } +func (m *Pair) GetKey() uint64 { + if m != nil { + return m.Key + } + return 0 +} + +func (m *Pair) GetCount() uint64 { + if m != nil { + return m.Count + } + return 0 +} + type SumCount struct { Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` @@ -78,6 +100,20 @@ func (m *SumCount) String() string { return proto.CompactTextString(m func (*SumCount) ProtoMessage() {} func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (m *SumCount) GetSum() int64 { + if m != nil { + return m.Sum + } + return 0 +} + +func (m *SumCount) GetCount() int64 { + if m != nil { + return m.Count + } + return 0 +} + type Bit struct { RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` @@ -89,6 +125,27 @@ func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (m *Bit) GetRowID() uint64 { + if m != nil { + return m.RowID + } + return 0 +} + +func (m *Bit) GetColumnID() uint64 { + if m != nil { + return m.ColumnID + } + return 0 +} + +func (m *Bit) GetTimestamp() int64 { + if m != nil { + return m.Timestamp + } + return 0 +} + type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` @@ -99,6 +156,13 @@ func (m *ColumnAttrSet) String() string { return proto.CompactTextStr func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (m *ColumnAttrSet) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + func (m *ColumnAttrSet) GetAttrs() []*Attr { if m != nil { return m.Attrs @@ -120,6 +184,48 @@ func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (m *Attr) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *Attr) GetType() uint64 { + if m != nil { + return m.Type + } + return 0 +} + +func (m *Attr) GetStringValue() string { + if m != nil { + return m.StringValue + } + return "" +} + +func (m *Attr) GetIntValue() int64 { + if m != nil { + return m.IntValue + } + return 0 +} + +func (m *Attr) GetBoolValue() bool { + if m != nil { + return m.BoolValue + } + return false +} + +func (m *Attr) GetFloatValue() float64 { + if m != nil { + return m.FloatValue + } + return 0 +} + type AttrMap struct { Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } @@ -150,6 +256,48 @@ func (m *QueryRequest) String() string { return proto.CompactTextStri func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (m *QueryRequest) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + +func (m *QueryRequest) GetSlices() []uint64 { + if m != nil { + return m.Slices + } + return nil +} + +func (m *QueryRequest) GetColumnAttrs() bool { + if m != nil { + return m.ColumnAttrs + } + return false +} + +func (m *QueryRequest) GetRemote() bool { + if m != nil { + return m.Remote + } + return false +} + +func (m *QueryRequest) GetExcludeAttrs() bool { + if m != nil { + return m.ExcludeAttrs + } + return false +} + +func (m *QueryRequest) GetExcludeBits() bool { + if m != nil { + return m.ExcludeBits + } + return false +} + type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` @@ -161,6 +309,13 @@ func (m *QueryResponse) String() string { return proto.CompactTextStr func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (m *QueryResponse) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { return m.Results @@ -195,6 +350,13 @@ func (m *QueryResult) GetBitmap() *Bitmap { return nil } +func (m *QueryResult) GetN() uint64 { + if m != nil { + return m.N + } + return 0 +} + func (m *QueryResult) GetPairs() []*Pair { if m != nil { return m.Pairs @@ -209,6 +371,13 @@ func (m *QueryResult) GetSumCount() *SumCount { return nil } +func (m *QueryResult) GetChanged() bool { + if m != nil { + return m.Changed + } + return false +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -223,6 +392,48 @@ func (m *ImportRequest) String() string { return proto.CompactTextStr func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (m *ImportRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *ImportRequest) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *ImportRequest) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *ImportRequest) GetRowIDs() []uint64 { + if m != nil { + return m.RowIDs + } + return nil +} + +func (m *ImportRequest) GetColumnIDs() []uint64 { + if m != nil { + return m.ColumnIDs + } + return nil +} + +func (m *ImportRequest) GetTimestamps() []int64 { + if m != nil { + return m.Timestamps + } + return nil +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -237,6 +448,48 @@ func (m *ImportValueRequest) String() string { return proto.CompactTe func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (m *ImportValueRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *ImportValueRequest) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *ImportValueRequest) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *ImportValueRequest) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *ImportValueRequest) GetColumnIDs() []uint64 { + if m != nil { + return m.ColumnIDs + } + return nil +} + +func (m *ImportValueRequest) GetValues() []uint64 { + if m != nil { + return m.Values + } + return nil +} + func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") @@ -472,7 +725,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 } return i, nil } @@ -863,24 +1117,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1194,7 +1430,24 @@ func (m *Bitmap) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -1235,23 +1488,6 @@ func (m *Bitmap) Unmarshal(dAtA []byte) error { } m.Bits = append(m.Bits, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Bits = append(m.Bits, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) } @@ -1843,15 +2079,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex @@ -2014,7 +2243,24 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { m.Query = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2055,23 +2301,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } m.Slices = append(m.Slices, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) } @@ -2610,7 +2839,24 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } } case 4: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RowIDs = append(m.RowIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2651,7 +2897,11 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.RowIDs = append(m.RowIDs, v) } - } else if wireType == 0 { + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) + } + case 5: + if wireType == 0 { var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2667,12 +2917,8 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.RowIDs = append(m.RowIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) - } - case 5: - if wireType == 2 { + m.ColumnIDs = append(m.ColumnIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2713,8 +2959,12 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.ColumnIDs = append(m.ColumnIDs, v) } - } else if wireType == 0 { - var v uint64 + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) + } + case 6: + if wireType == 0 { + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -2724,17 +2974,13 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } } - m.ColumnIDs = append(m.ColumnIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) - } - case 6: - if wireType == 2 { + m.Timestamps = append(m.Timestamps, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2775,23 +3021,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.Timestamps = append(m.Timestamps, v) } - } else if wireType == 0 { - var v int64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Timestamps = append(m.Timestamps, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Timestamps", wireType) } @@ -2952,7 +3181,24 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 5: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ColumnIDs = append(m.ColumnIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2993,7 +3239,11 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.ColumnIDs = append(m.ColumnIDs, v) } - } else if wireType == 0 { + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) + } + case 6: + if wireType == 0 { var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -3009,12 +3259,8 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { break } } - m.ColumnIDs = append(m.ColumnIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) - } - case 6: - if wireType == 2 { + m.Values = append(m.Values, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -3055,23 +3301,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.Values = append(m.Values, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Values = append(m.Values, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Values", wireType) } diff --git a/pilosa.go b/pilosa.go index aa1378a36..bdab539cd 100644 --- a/pilosa.go +++ b/pilosa.go @@ -162,6 +162,50 @@ func StringInSlice(a string, list []string) bool { return false } +// SlicesAreEqual determines if two string slices are equal. +func SlicesAreEqual(a, b []string) bool { + + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +// SliceDiff returns the difference between two uint64 slices. +func SliceDiff(a, b []uint64) []uint64 { + m := make(map[uint64]uint64) + + for _, y := range b { + m[y]++ + } + + var ret []uint64 + for _, x := range a { + if m[x] > 0 { + m[x]-- + continue + } + ret = append(ret, x) + } + + return ret +} + // ContainsSubstring checks to see if substring a is contained in any string in the slice. func ContainsSubstring(a string, list []string) bool { for _, b := range list { diff --git a/server.go b/server.go index 773cf9025..4ecffe974 100644 --- a/server.go +++ b/server.go @@ -132,6 +132,9 @@ func (s *Server) Open() error { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) } + // Set Cluster URI. + s.Cluster.URI = s.URI + // Create local node if no cluster is specified. if len(s.Cluster.Nodes) == 0 { s.Cluster.Nodes = []*Node{ @@ -139,6 +142,7 @@ func (s *Server) Open() error { } } + // TODO: nodes aren't here yet. May need to merge new stats code anyway. for i, n := range s.Cluster.Nodes { if s.Cluster.NodeByHost(n.Host) != nil { s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) @@ -151,13 +155,14 @@ func (s *Server) Open() error { return fmt.Errorf("opening Holder: %v", err) } + // Start the BroadcastReceiver. if err := s.BroadcastReceiver.Start(s); err != nil { return fmt.Errorf("starting BroadcastReceiver: %v", err) } - // Open NodeSet communication - if err := s.Cluster.NodeSet.Open(); err != nil { - return fmt.Errorf("opening NodeSet: %v", err) + // Open Cluster management. + if err := s.Cluster.Open(); err != nil { + return fmt.Errorf("opening Cluster: %v", err) } // Create default HTTP client @@ -190,11 +195,13 @@ func (s *Server) Open() error { } }() - // Start background monitoring. - s.wg.Add(3) - go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() - go func() { defer s.wg.Done(); s.monitorRuntime() }() + /* + // Start background monitoring. + s.wg.Add(3) + go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() + go func() { defer s.wg.Done(); s.monitorMaxSlices() }() + go func() { defer s.wg.Done(); s.monitorRuntime() }() + */ return nil } @@ -208,6 +215,9 @@ func (s *Server) Close() error { if s.ln != nil { s.ln.Close() } + if s.Cluster != nil { + s.Cluster.Close() + } if s.Holder != nil { s.Holder.Close() } @@ -266,11 +276,6 @@ func (s *Server) monitorAntiEntropy() { // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. func (s *Server) monitorMaxSlices() { - // Ignore if only one node in the cluster. - if len(s.Cluster.Nodes) <= 1 { - return - } - ticker := time.NewTicker(s.PollingInterval) defer ticker.Stop() @@ -333,16 +338,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := FrameOptions{ - RowLabel: obj.Meta.RowLabel, - InverseEnabled: obj.Meta.InverseEnabled, - RangeEnabled: obj.Meta.RangeEnabled, - CacheType: obj.Meta.CacheType, - CacheSize: obj.Meta.CacheSize, - TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), - Fields: decodeFields(obj.Meta.Fields), - } - _, err := idx.CreateFrame(obj.Frame, opt) + opt := decodeFrameOptions(obj.Meta) + _, err := idx.CreateFrame(obj.Frame, *opt) if err != nil { return err } @@ -372,25 +369,48 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.ClusterStatus: + err := s.Cluster.mergeClusterStatus(obj) + if err != nil { + return err + } + case *internal.ResizeInstruction: + s.Cluster.followResizeInstruction(obj) + case *internal.ResizeInstructionComplete: + err := s.Cluster.MarkResizeInstructionComplete(obj) + if err != nil { + return err + } } + return nil } +// State returns the cluster state according to this node. +func (s *Server) State() string { + return s.Cluster.State +} + // LocalStatus returns the state of the local node as well as the // holder (indexes/frames) according to the local node. // In a gossip implementation, memberlist.Delegate.LocalState() uses this. // Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { + if s.Cluster == nil { + return nil, errors.New("Server.Cluster is nil") + } if s.Holder == nil { return nil, errors.New("Server.Holder is nil") } ns := internal.NodeStatus{ - Host: s.URI.HostPort(), - State: NodeStateUp, - Indexes: EncodeIndexes(s.Holder.Indexes()), + Host: s.URI.HostPort(), + State: s.State(), + Indexes: EncodeIndexes(s.Holder.Indexes()), + HostList: s.Cluster.HostList(), } + // TODO: get rid of this // Append Slice list per this Node's indexes for _, index := range ns.Indexes { index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI.HostPort()) @@ -406,22 +426,8 @@ func (s *Server) ClusterStatus() (proto.Message, error) { if err != nil { return nil, err } - node := s.Cluster.NodeByHost(s.URI.HostPort()) - node.SetStatus(ns.(*internal.NodeStatus)) - - // Update NodeState for all nodes. - for host, nodeState := range s.Cluster.NodeStates() { - // In a default configuration (or single-node) where a StaticNodeSet is used - // then all nodes are marked as DOWN. At the very least, we should consider - // the local node as UP. - // TODO: we should be able to remove this check if/when cluster.Nodes and - // cluster.NodeSet are unified. - if host == s.URI.HostPort() { - nodeState = NodeStateUp - } - node := s.Cluster.NodeByHost(host) - node.SetState(nodeState) - } + localNode := s.Cluster.localNode() + localNode.SetStatus(ns.(*internal.NodeStatus)) return s.Cluster.Status(), nil } @@ -432,9 +438,19 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { } func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { + + // Ignore status updates from self. + if s.URI.HostPort() == ns.Host { + return nil + } + fmt.Printf("mergeRemoteStatus on (%s) from (%s)\n", s.URI.HostPort(), ns.Host) + // Update Node.state. - node := s.Cluster.NodeByHost(ns.Host) - node.SetStatus(ns) + // Node can be nil if a merge occurs (via gossip) before the coordinator has + // a chance to broadcast the existence of the node. + if node := s.Cluster.NodeByHost(ns.Host); node != nil { + node.SetStatus(ns) + } // Create indexes that don't exist. for _, index := range ns.Indexes { @@ -448,12 +464,8 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { } // Create frames that don't exist. for _, f := range index.Frames { - opt := FrameOptions{ - RowLabel: f.Meta.RowLabel, - TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), - CacheSize: f.Meta.CacheSize, - } - _, err := idx.CreateFrameIfNotExists(f.Name, opt) + opt := decodeFrameOptions(f.Meta) + _, err := idx.CreateFrameIfNotExists(f.Name, *opt) if err != nil { return err } @@ -581,7 +593,7 @@ func CountOpenFiles() int { return count } -// StatusHandler specifies two methods which an object must implement to share +// StatusHandler specifies the methods which an object must implement to share // state in the cluster. These are used by the GossipNodeSet to implement the // LocalState and MergeRemoteState methods of memberlist.Delegate type StatusHandler interface { diff --git a/server/server.go b/server/server.go index 22e965b55..7fafce371 100644 --- a/server/server.go +++ b/server/server.go @@ -30,10 +30,11 @@ import ( "time" "crypto/tls" + "io/ioutil" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" - "io/ioutil" ) func init() { @@ -120,17 +121,21 @@ func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN + cluster.IndexReporter = m.Server.Holder - for _, address := range m.Config.Cluster.Hosts { - uri, err := pilosa.NewURIFromAddress(address) - if err != nil { - return err + /* + // TODO travis: get rid of this URI code + for _, address := range m.Config.Cluster.Hosts { + uri, err := pilosa.NewURIFromAddress(address) + if err != nil { + return err + } + cluster.Nodes = append(cluster.Nodes, &pilosa.Node{ + Scheme: uri.Scheme(), + Host: uri.HostPort(), + }) } - cluster.Nodes = append(cluster.Nodes, &pilosa.Node{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - }) - } + */ m.Server.Cluster = cluster // Setup logging output. @@ -139,6 +144,9 @@ func (m *Command) SetupServer() error { return err } + // Configure data directory (for Cluster .topology) + m.Server.Cluster.Path = m.Config.DataDir + // Configure holder. m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir @@ -172,6 +180,13 @@ func (m *Command) SetupServer() error { m.Server.Handler.ClientOptions = &pilosa.ClientOptions{TLS: m.Server.TLS} } + // Set the coordinator node. + uri, err = pilosa.AddressWithDefaults(m.Config.Cluster.Coordinator) + if err != nil { + return err + } + m.Server.Cluster.Coordinator = uri.HostPort() + // Set internal port (string). gossipPortStr := pilosa.DefaultGossipPort // Config.GossipPort is deprecated, so Config.Gossip.Port has priority @@ -206,6 +221,7 @@ func (m *Command) SetupServer() error { // get the host portion of addr to use for binding gossipHost := uri.Host() gossipNodeSet := gossip.NewGossipNodeSet(uri.HostPort(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet @@ -221,6 +237,9 @@ func (m *Command) SetupServer() error { return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type) } + // Cluster management needs. + m.Server.Cluster.Broadcaster = m.Server.Broadcaster + // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) diff --git a/server/server_test.go b/server/server_test.go index 4ffb17a34..0eb19be21 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -445,6 +445,12 @@ func TestMain_SendReceiveMessage(t *testing.T) { } gossipSeed := gossipHost + ":" + freePorts[0] + topology := &pilosa.Topology{HostList: []string{m0.Server.URI.HostPort(), m1.Server.URI.HostPort()}} + + m0.Server.Cluster.Coordinator = m0.Server.URI.HostPort() + m0.Server.Cluster.Topology = topology + m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + gossipNodeSet0 := gossip.NewGossipNodeSet(m0.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m0.Server, nil) m0.Server.Cluster.NodeSet = gossipNodeSet0 m0.Server.Broadcaster = gossipNodeSet0 @@ -455,8 +461,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { t.Fatal(err) } - // Open NodeSet communication - if err := m0.Server.Cluster.NodeSet.Open(); err != nil { + // Open Cluster management. + if err := m0.Server.Cluster.Open(); err != nil { t.Fatal(err) } @@ -472,6 +478,9 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal(err) } + m1.Server.Cluster.Coordinator = m0.Server.URI.HostPort() + m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + gossipNodeSet1 := gossip.NewGossipNodeSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) m1.Server.Cluster.NodeSet = gossipNodeSet1 m1.Server.Broadcaster = gossipNodeSet1 @@ -482,8 +491,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { t.Fatal(err) } - // Open NodeSet communication - if err := m1.Server.Cluster.NodeSet.Open(); err != nil { + // Open Cluster management. + if err := m1.Server.Cluster.Open(); err != nil { t.Fatal(err) } diff --git a/test/cluster.go b/test/cluster.go index c45b1c989..60d44384f 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -2,15 +2,23 @@ package test import ( "fmt" + "io/ioutil" "github.com/pilosa/pilosa" ) // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { + path, err := ioutil.TempDir("", "pilosa-cluster-") + if err != nil { + panic(err) + } + c := pilosa.NewCluster() c.ReplicaN = 1 c.Hasher = NewModHasher() + c.Path = path + c.Topology = pilosa.NewTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ diff --git a/test/handler.go b/test/handler.go index aa5a56acb..b8fce4c65 100644 --- a/test/handler.go +++ b/test/handler.go @@ -3,7 +3,6 @@ package test import ( "context" "encoding/json" - "errors" "io" "io/ioutil" "net/http" @@ -12,7 +11,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -77,6 +75,7 @@ func NewServer() *Server { return s } +/* TODO travis: fix this test // LocalStatus returns the state of the local node as well as the // holder (indexes/frames) according to the local node. func (s *Server) LocalStatus() (proto.Message, error) { @@ -104,6 +103,7 @@ func (s *Server) ClusterStatus() (proto.Message, error) { // So just return its status return s.LocalStatus() } +*/ // HandleRemoteStatus just need to implement a nop to complete the Interface func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } From a8871ada6ac5921e1969888470c269ed1f086ad5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 25 Oct 2017 00:26:11 -0500 Subject: [PATCH 002/311] Convert Host to URI. Fix all compile errors. --- client.go | 15 +- client_test.go | 27 +- cluster.go | 287 ++++++++-------- cluster_internal_test.go | 20 +- cluster_test.go | 22 +- config.go | 2 +- ctl/backup_test.go | 4 +- ctl/export_test.go | 4 +- ctl/import_test.go | 8 +- ctl/restore_test.go | 4 +- event.go | 2 +- executor.go | 20 +- executor_test.go | 40 ++- fragment.go | 10 +- gossip/gossip.go | 21 +- handler.go | 10 +- holder.go | 14 +- holder_test.go | 19 +- internal/private.pb.go | 709 ++++++++++++++++++++++++++------------- internal/private.proto | 23 +- pilosa.go | 4 +- server.go | 54 +-- server/server.go | 11 +- server/server_test.go | 14 +- test/cluster.go | 19 +- test/executor.go | 5 +- test/handler.go | 8 +- uri.go | 63 +++- 28 files changed, 911 insertions(+), 528 deletions(-) diff --git a/client.go b/client.go index fbd0d3717..5edd45468 100644 --- a/client.go +++ b/client.go @@ -32,6 +32,7 @@ import ( "time" "crypto/tls" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -336,7 +337,7 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, // Import to each node. for _, node := range nodes { if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) } } @@ -441,7 +442,7 @@ func (c *Client) ImportValue(ctx context.Context, index, frame, field string, sl // Import to each node. for _, node := range nodes { if err := c.importValueNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) } } @@ -529,7 +530,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice node := nodes[i] if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil { - e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err) + e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) continue } else { return nil @@ -710,7 +711,7 @@ func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, return nil, ErrFragmentNotFound } else if resp.StatusCode != http.StatusOK { resp.Body.Close() - return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.Host, resp.StatusCode) + return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) } return resp.Body, nil @@ -789,7 +790,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, // Return error if response not OK. if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: host=%s, code=%d", node.Host, resp.StatusCode) + return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode) } } @@ -1224,8 +1225,8 @@ func uriPathToURL(uri *URI, path string) url.URL { func nodePathToURL(node *Node, path string) url.URL { return url.URL{ - Scheme: node.Scheme, - Host: node.Host, + Scheme: node.URI.Scheme(), + Host: node.URI.Host(), Path: path, } } diff --git a/client_test.go b/client_test.go index 70ea6c669..e5dffc9a5 100644 --- a/client_test.go +++ b/client_test.go @@ -37,7 +37,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { server[i] = test.NewServer() server[i].Handler.URI = server[i].HostURI() server[i].Handler.Cluster = c - server[i].Handler.Cluster.Nodes[i].Host = server[i].Host() + server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI() server[i].Handler.Holder = hldr[i].Holder } return server, hldr @@ -56,24 +56,21 @@ func TestClient_MultiNode(t *testing.T) { s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(nil) e.Holder = hldr[0].Holder - e.Scheme = cluster.Nodes[0].Scheme - e.Host = cluster.Nodes[0].Host + e.URI = cluster.Nodes[0].URI e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(nil) e.Holder = hldr[1].Holder - e.Scheme = cluster.Nodes[1].Scheme - e.Host = cluster.Nodes[1].Host + e.URI = cluster.Nodes[1].URI e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(nil) e.Holder = hldr[2].Holder - e.Scheme = cluster.Nodes[2].Scheme - e.Host = cluster.Nodes[2].Host + e.URI = cluster.Nodes[2].URI e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } @@ -81,7 +78,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host()) + owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false for _, ownNum := range owns { if ownNum == num { @@ -212,7 +209,7 @@ func TestClient_Import(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder // Send import request. @@ -263,7 +260,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder // Send import request. @@ -312,7 +309,7 @@ func TestClient_ImportValue(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder // Send import request. @@ -350,7 +347,7 @@ func TestClient_BackupRestore(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder c := test.MustNewClient(s.Host()) @@ -415,7 +412,7 @@ func TestClient_BackupInverseView(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder c := test.MustNewClient(s.Host()) @@ -452,7 +449,7 @@ func TestClient_BackupInvalidView(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder c := test.MustNewClient(s.Host()) @@ -481,7 +478,7 @@ func TestClient_FragmentBlocks(t *testing.T) { defer s.Close() s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder // Retrieve blocks. diff --git a/cluster.go b/cluster.go index 6dcc0b102..c3b02ecd2 100644 --- a/cluster.go +++ b/cluster.go @@ -53,8 +53,9 @@ const ( // Node represents a node in the cluster. type Node struct { - Scheme string `json:"scheme"` - Host string `json:"host"` // HostPort + //Scheme string `json:"scheme"` + //Host string `json:"host"` // HostPort + URI URI // TODO: add json tags: `json:"uri"` status *internal.NodeStatus `json:"status"` } @@ -72,16 +73,6 @@ func (n *Node) SetState(s string) { n.status.State = s } -// URI returns the pilosa.URI corresponding to this node -func (n *Node) URI() (*URI, error) { - uri, err := NewURIFromAddress(n.Host) - if err != nil { - return nil, err - } - uri.SetScheme(n.Scheme) - return uri, nil -} - // Nodes represents a list of nodes. type Nodes []*Node @@ -95,10 +86,10 @@ func (a Nodes) Contains(n *Node) bool { return false } -// ContainsHost returns true if host matches one of the node's host. -func (a Nodes) ContainsHost(host string) bool { +// ContainsURI returns true if host matches one of the node's uri. +func (a Nodes) ContainsURI(uri URI) bool { for _, n := range a { - if n.Host == host { + if n.URI == uri { return true } } @@ -116,24 +107,24 @@ func (a Nodes) Filter(n *Node) []*Node { return other } -// FilterHost returns a new list of nodes with host removed. -func (a Nodes) FilterHost(host string) []*Node { +// FilterURI returns a new list of nodes with URI removed. +func (a Nodes) FilterURI(uri URI) []*Node { other := make([]*Node, 0, len(a)) for _, node := range a { - if node.Host != host { + if node.URI != uri { other = append(other, node) } } return other } -// Hosts returns a list of all hostnames. -func (a Nodes) Hosts() []string { - hosts := make([]string, len(a)) +// URIs returns a list of all uris. +func (a Nodes) URIs() []URI { + uris := make([]URI, len(a)) for i, n := range a { - hosts[i] = n.Host + uris[i] = n.URI } - return hosts + return uris } // Clone returns a shallow copy of nodes. @@ -149,11 +140,11 @@ type ByHost []*Node func (h ByHost) Len() int { return len(h) } func (h ByHost) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h ByHost) Less(i, j int) bool { return h[i].Host < h[j].Host } +func (h ByHost) Less(i, j int) bool { return h[i].URI.String() < h[j].URI.String() } // Cluster represents a collection of nodes. type Cluster struct { - URI *URI + URI URI Nodes []*Node // TODO phase this out? NodeSet NodeSet @@ -178,11 +169,11 @@ type Cluster struct { // Required for cluster Resize. State string - Coordinator string + Coordinator URI IndexReporter IndexReporter Broadcaster Broadcaster - joiningHosts chan string + joiningURIs chan URI mu sync.RWMutex jobs map[int64]*ResizeJob @@ -204,9 +195,9 @@ func NewCluster() *Cluster { ReplicaN: DefaultReplicaN, EventReceiver: NopEventReceiver, - joiningHosts: make(chan string, 10), // buffered channel - jobs: make(map[int64]*ResizeJob), - closing: make(chan struct{}), + joiningURIs: make(chan URI, 10), // buffered channel + jobs: make(map[int64]*ResizeJob), + closing: make(chan struct{}), LogOutput: os.Stderr, } @@ -219,16 +210,17 @@ func (c *Cluster) logger() *log.Logger { // IsCoordinator is true if this node is the coordinator. func (c *Cluster) IsCoordinator() bool { - return c.Coordinator == c.URI.HostPort() + return c.Coordinator == c.URI } // AddHost adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddHost(host string) error { +func (c *Cluster) AddHost(uri URI) error { // add to cluster - _, added := c.AddNode(host) + _, added := c.AddNode(uri) if !added { + fmt.Println("NOT added") return nil } @@ -236,17 +228,19 @@ func (c *Cluster) AddHost(host string) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.AddHost(host) { + if !c.Topology.AddURI(uri) { + fmt.Println("call top.AddHost()") return nil } // save topology + fmt.Println("calll c.saveTop()") return c.saveTopology() } -// HostList returns the list of hosts in the cluster. -func (c *Cluster) HostList() []string { - return Nodes(c.Nodes).Hosts() +// URISet returns the list of uris in the cluster. +func (c *Cluster) URISet() []URI { + return Nodes(c.Nodes).URIs() } func (c *Cluster) setState(state string) { @@ -256,15 +250,14 @@ func (c *Cluster) setState(state string) { } func (c *Cluster) localNode() *Node { - return c.NodeByHost(c.URI.HostPort()) + return c.NodeByURI(c.URI) } // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ - State: c.State, - HostList: c.HostList(), - //NodeStatuses: encodeNodeStatuses(c.Nodes), // TODO travis: remove this? + State: c.State, + URISet: encodeURIs(c.URISet()), } } @@ -279,25 +272,25 @@ func encodeNodeStatuses(a []*Node) []*internal.NodeStatus { } */ -// NodeByHost returns a node reference by host. -func (c *Cluster) NodeByHost(host string) *Node { +// NodeByURI returns a node reference by uri. +func (c *Cluster) NodeByURI(uri URI) *Node { for _, n := range c.Nodes { - if n.Host == host { + if n.URI == uri { return n } } return nil } -// AddNode adds a node to the cluster, sorted by host. +// AddNode adds a node to the cluster, sorted by uri. // Returns a pointer to the node and true if the node was added. -func (c *Cluster) AddNode(host string) (*Node, bool) { - n := c.NodeByHost(host) +func (c *Cluster) AddNode(uri URI) (*Node, bool) { + n := c.NodeByURI(uri) if n != nil { return n, false } - n = &Node{Host: host} + n = &Node{URI: uri} c.Nodes = append(c.Nodes, n) // All hosts must be merged in the same order on all nodes in the cluster. @@ -332,7 +325,7 @@ func fragsDiff(a, b []frag) []frag { return ret } -type fragsByHost map[string][]frag +type fragsByHost map[URI][]frag func (a fragsByHost) add(b fragsByHost) fragsByHost { for k, v := range b { @@ -369,7 +362,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { return std.add(inv) } -// fragCombos returns a map (by host) of lists of fragments for a given index +// fragCombos returns a map (by uri) of lists of fragments for a given index // by creating every combination of frame/view specified in `frameViews` up to maxSlice. func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { t := make(fragsByHost) @@ -379,7 +372,7 @@ func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFram // for each frame/view combination: for frame, views := range frameViews { for _, view := range views { - t[n.Host] = append(t[n.Host], frag{frame, view, i}) + t[n.URI] = append(t[n.URI], frag{frame, view, i}) } } } @@ -389,12 +382,12 @@ func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFram // DataDiff returns a list of ResizeSources - for each host in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[string][]*internal.ResizeSource { - m := make(map[string][]*internal.ResizeSource) +func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[URI][]*internal.ResizeSource { + m := make(map[URI][]*internal.ResizeSource) // Initialize the map with all the nodes in `to`. for _, n := range to.Nodes { - m[n.Host] = nil + m[n.URI] = nil } // For now, we want our source to be confined to the primary fragment @@ -417,10 +410,10 @@ func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[string][]*internal.Resiz srcFrags := srcCluster.fragsByHost(idx) // srcHostsByFrag is the inverse representation of srcFrags. - srcHostsByFrag := make(map[frag]string) - for host, frags := range srcFrags { + srcHostsByFrag := make(map[frag]URI) + for uri, frags := range srcFrags { for _, frag := range frags { - srcHostsByFrag[frag] = host + srcHostsByFrag[frag] = uri } } @@ -439,7 +432,7 @@ func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[string][]*internal.Resiz m[host] = []*internal.ResizeSource{} for _, frag := range diff { src := &internal.ResizeSource{ - Host: srcHostsByFrag[frag], + URI: (srcHostsByFrag[frag]).Encode(), Index: idx.Name(), Frame: frag.frame, View: frag.view, @@ -470,8 +463,8 @@ func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node { } // OwnsFragment returns true if a host owns a fragment. -func (c *Cluster) OwnsFragment(host string, index string, slice uint64) bool { - return Nodes(c.FragmentNodes(index, slice)).ContainsHost(host) +func (c *Cluster) OwnsFragment(uri URI, index string, slice uint64) bool { + return Nodes(c.FragmentNodes(index, slice)).ContainsURI(uri) } // PartitionNodes returns a list of nodes that own a partition. @@ -498,13 +491,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { } // OwnsSlices find the set of slices owned by the node per Index -func (c *Cluster) OwnsSlices(index string, maxSlice uint64, host string) []uint64 { +func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { p := c.Partition(index, i) // Determine primary owner node. nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[nodeIndex].Host == host { + if c.Nodes[nodeIndex].URI == uri { slices = append(slices, i) } } @@ -550,11 +543,13 @@ func (c *Cluster) Open() error { return fmt.Errorf("considerTopology: %v", err) } // Add the local node to the cluster and update state. - c.AddHost(c.URI.HostPort()) + fmt.Println("IS Coord") + c.AddHost(c.URI) c.setState(state) } else { // Add the local node to the cluster. - c.AddHost(c.URI.HostPort()) + fmt.Println("NOT Coord") + c.AddHost(c.URI) } // Start the EventReceiver. @@ -583,15 +578,15 @@ func (c *Cluster) Close() error { } func (c *Cluster) needTopologyAgreement() bool { - return c.State == NodeStateStarting && !SlicesAreEqual(c.Topology.HostList, Nodes(c.Nodes).Hosts()) + return c.State == NodeStateStarting && !URISlicesAreEqual(c.Topology.URISet, c.URISet()) } func (c *Cluster) haveTopologyAgreement() bool { - return SlicesAreEqual(c.Topology.HostList, Nodes(c.Nodes).Hosts()) + return URISlicesAreEqual(c.Topology.URISet, c.URISet()) } -func (c *Cluster) handleJoiningHost(host string) error { - j, err := c.GenerateResizeJob(host) +func (c *Cluster) handleJoiningHost(uri URI) error { + j, err := c.GenerateResizeJob(uri) if err != nil { return err } @@ -607,8 +602,8 @@ func (c *Cluster) handleJoiningHost(host string) error { switch jobResult { case ResizeJobStateDone: c.CompleteCurrentJob(ResizeJobStateDone) - // Add host to the cluster. - return c.AddHost(host) + // Add uri to the cluster. + return c.AddHost(uri) case ResizeJobStateAborted: c.CompleteCurrentJob(ResizeJobStateAborted) } @@ -622,25 +617,25 @@ func (c *Cluster) setStateAndBroadcast(state string) error { } func (c *Cluster) listenForJoins() { - var hostJoined bool + var uriJoined bool for { // Handle all pending joins before changing state back to NORMAL. select { - case host := <-c.joiningHosts: - err := c.handleJoiningHost(host) + case uri := <-c.joiningURIs: + err := c.handleJoiningHost(uri) if err != nil { c.logger().Printf("handleJoiningHost error: err=%s", err) continue } - hostJoined = true + uriJoined = true continue default: } // Only change state to NORMAL if we have successfully added at least one host. - if hostJoined { + if uriJoined { // Put the cluster back to state NORMAL and broadcast. if err := c.setStateAndBroadcast(NodeStateNormal); err != nil { c.logger().Printf("setStateAndBroadcast error: err=%s", err) @@ -651,13 +646,13 @@ func (c *Cluster) listenForJoins() { select { case <-c.closing: return - case host := <-c.joiningHosts: + case host := <-c.joiningURIs: err := c.handleJoiningHost(host) if err != nil { c.logger().Printf("handleJoiningHost error: err=%s", err) continue } - hostJoined = true + uriJoined = true continue } } @@ -666,11 +661,11 @@ func (c *Cluster) listenForJoins() { // GenerateResizeJob creates a new ResizeJob based on the new host being // added. It also saves a reference to the ResizeJob in the `jobs` map // for future lookup by JobID. -func (c *Cluster) GenerateResizeJob(addHost string) (*ResizeJob, error) { +func (c *Cluster) GenerateResizeJob(addURI URI) (*ResizeJob, error) { c.mu.Lock() defer c.mu.Unlock() - j := c.generateResizeJob(addHost) + j := c.generateResizeJob(addURI) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -688,9 +683,9 @@ func (c *Cluster) GenerateResizeJob(addHost string) (*ResizeJob, error) { // the difference between Cluster and a new Cluster containing addHost. // Broadcaster is associated to the ResizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. -func (c *Cluster) generateResizeJob(addHost string) *ResizeJob { +func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { - j := NewResizeJob(addHost, Nodes(c.Nodes).Hosts()) + j := NewResizeJob(addURI, Nodes(c.Nodes).URIs()) j.Broadcaster = c.Broadcaster // toCluster is a clone of Cluster with the new node added for comparison. @@ -699,7 +694,7 @@ func (c *Cluster) generateResizeJob(addHost string) *ResizeJob { toCluster.Hasher = c.Hasher toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN - toCluster.AddNode(addHost) + toCluster.AddNode(addURI) // Add to the ResizeJob the instructions for each index. for _, idx := range c.IndexReporter.Indexes() { @@ -707,16 +702,16 @@ func (c *Cluster) generateResizeJob(addHost string) *ResizeJob { // a host in toCluster. dataDiff := c.DataDiff(toCluster, idx) - for host, sources := range dataDiff { + for uri, sources := range dataDiff { // If a host doesn't need to request data, mark it as complete. if len(sources) == 0 { - j.Hosts[host] = true + j.URIs[uri] = true continue } instr := &internal.ResizeInstruction{ JobID: j.ID, - Host: host, - Coordinator: c.Coordinator, + URI: uri.Encode(), + Coordinator: encodeURI(c.Coordinator), Sources: sources, } j.Instructions = append(j.Instructions, instr) @@ -745,7 +740,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { for _, src := range instr.Sources { /************************************************************/ // TODO travis: get the data files from other nodes. - fmt.Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.Host) + fmt.Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.URI) for i := 0; i <= 4; i++ { fmt.Printf(" %d", i) time.Sleep(1 * time.Second) @@ -756,11 +751,11 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { complete := &internal.ResizeInstructionComplete{ JobID: instr.JobID, - Host: instr.Host, + URI: instr.URI, } node := &Node{ - Host: instr.Coordinator, + URI: decodeURI(instr.Coordinator), } if err := c.Broadcaster.SendTo(node, complete); err != nil { c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) @@ -778,10 +773,12 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc return fmt.Errorf("ResizeJob %d is no longer running", j.ID) } - // Mark host complete. - j.Hosts[complete.Host] = true + uri := decodeURI(complete.URI) - if !j.hostsArePending() { + // Mark host complete. + j.URIs[uri] = true + + if !j.urisArePending() { j.result <- ResizeJobStateDone } @@ -799,7 +796,7 @@ func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } type ResizeJob struct { ID int64 - Hosts map[string]bool + URIs map[URI]bool Instructions []*internal.ResizeInstruction Broadcaster Broadcaster @@ -810,22 +807,22 @@ type ResizeJob struct { } // NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(addHost string, existingHosts []string) *ResizeJob { +func NewResizeJob(addURI URI, existingURIs []URI) *ResizeJob { - // Build a map of hosts to track their resize status. - hosts := make(map[string]bool) + // Build a map of uris to track their resize status. + uris := make(map[URI]bool) // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. - for _, h := range existingHosts { - hosts[h] = false + for _, u := range existingURIs { + uris[u] = false } // Include the added node in the map for tracking. - hosts[addHost] = false + uris[addURI] = false return &ResizeJob{ ID: rand.Int63(), - Hosts: hosts, + URIs: uris, result: make(chan string), } } @@ -857,7 +854,7 @@ func (j *ResizeJob) Run() error { j.setState(ResizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. - if !j.hostsArePending() { + if !j.urisArePending() { j.result <- ResizeJobStateDone return nil } @@ -880,9 +877,9 @@ func (j *ResizeJob) isComplete() bool { } } -// hostsArePending returns true if any host is still working on the resize. -func (j *ResizeJob) hostsArePending() bool { - for _, complete := range j.Hosts { +// urisArePending returns true if any uri is still working on the resize. +func (j *ResizeJob) urisArePending() bool { + for _, complete := range j.URIs { if !complete { return true } @@ -896,7 +893,7 @@ func (j *ResizeJob) distributeResizeInstructions() error { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. node := &Node{ - Host: instr.Host, + URI: decodeURI(instr.URI), } if err := j.Broadcaster.SendTo(node, instr); err != nil { return err @@ -905,40 +902,50 @@ func (j *ResizeJob) distributeResizeInstructions() error { return nil } +type URISet []URI + +func (u URISet) ToHostPortStrings() []string { + other := make([]string, 0, len(u)) + for _, uri := range u { + other = append(other, uri.HostPort()) + } + return other +} + // Topology represents the list of hosts in the cluster. type Topology struct { - mu sync.RWMutex - HostList []string + mu sync.RWMutex + URISet []URI } func NewTopology() *Topology { return &Topology{} } -// ContainsHost returns true if host matches one of the topology's hosts. -func (t *Topology) ContainsHost(host string) bool { +// ContainsURI returns true if uri matches one of the topology's uris. +func (t *Topology) ContainsURI(uri URI) bool { t.mu.RLock() defer t.mu.RUnlock() - return t.containsHost(host) + return t.containsURI(uri) } -func (t *Topology) containsHost(host string) bool { - for _, thost := range t.HostList { - if thost == host { +func (t *Topology) containsURI(uri URI) bool { + for _, turi := range t.URISet { + if turi == uri { return true } } return false } -// AddHost adds the host to the topology and returns true if added. -func (t *Topology) AddHost(host string) bool { +// AddHost adds the uri to the topology and returns true if added. +func (t *Topology) AddURI(uri URI) bool { t.mu.Lock() defer t.mu.Unlock() - if t.containsHost(host) { + if t.containsURI(uri) { return false } - t.HostList = append(t.HostList, host) + t.URISet = append(t.URISet, uri) return true } @@ -956,13 +963,18 @@ func (c *Cluster) loadTopology() error { if err := proto.Unmarshal(buf, &pb); err != nil { return err } - c.Topology = decodeTopology(&pb) + top, err := decodeTopology(&pb) + if err != nil { + return err + } + c.Topology = top return nil } // saveTopology writes the current topology to disk. func (c *Cluster) saveTopology() error { + fmt.Println("saveTopology", filepath.Join(c.Path, ".topology")) if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { return err } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { @@ -976,33 +988,34 @@ func encodeTopology(topology *Topology) *internal.Topology { return nil } return &internal.Topology{ - HostList: topology.HostList, + URISet: encodeURIs(topology.URISet), } } -func decodeTopology(topology *internal.Topology) *Topology { +func decodeTopology(topology *internal.Topology) (*Topology, error) { if topology == nil { - return nil + return nil, nil } + t := &Topology{ - HostList: topology.HostList, + URISet: decodeURIs(topology.URISet), } - return t + return t, nil } func (c *Cluster) considerTopology() (string, error) { // If there is no .topology file, it's safe to go to state NORMAL. - if len(c.Topology.HostList) == 0 { + if len(c.Topology.URISet) == 0 { return NodeStateNormal, nil } // The local node (coordinator) must be in the .topology. - if !c.Topology.ContainsHost(c.Coordinator) { - return "", fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.HostList) + if !c.Topology.ContainsURI(c.Coordinator) { + return "", fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.URISet) } // If local node is the only thing in .topology, continue to state NORMAL. - if len(c.Topology.HostList) == 1 { + if len(c.Topology.URISet) == 1 { return NodeStateNormal, nil } @@ -1014,7 +1027,7 @@ func (c *Cluster) considerTopology() (string, error) { // ReceiveEvent represents an implementation of EventHandler. func (c *Cluster) ReceiveEvent(e *NodeEvent) error { // Ignore events sent from this node. - if e.Host == c.URI.HostPort() { + if e.URI == c.URI { return nil } @@ -1027,11 +1040,12 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. - if !c.Topology.ContainsHost(e.Host) { - return fmt.Errorf("host is not in topology: %v", e.Host) + if !c.Topology.ContainsURI(e.URI) { + return fmt.Errorf("host is not in topology: %v", e.URI) } - if err := c.AddHost(e.Host); err != nil { + uri := e.URI + if err := c.AddHost(uri); err != nil { return err } @@ -1045,13 +1059,14 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { } // Don't do anything else if the cluster already contains the node. - if c.NodeByHost(e.Host) != nil { + if c.NodeByURI(e.URI) != nil { return nil } // If the index does not yet have data, go ahead and add the node. if !c.IndexReporter.HasData() { - if err := c.AddHost(e.Host); err != nil { + uri := e.URI + if err := c.AddHost(uri); err != nil { return err } return c.setStateAndBroadcast(NodeStateNormal) @@ -1062,7 +1077,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { if err := c.setStateAndBroadcast(NodeStateResizing); err != nil { return err } - c.joiningHosts <- e.Host + c.joiningURIs <- e.URI case NodeLeave: // TODO: implement this @@ -1079,8 +1094,8 @@ func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { return nil } - for _, host := range cs.HostList { - c.AddHost(host) + for _, uri := range decodeURIs(cs.URISet) { + c.AddHost(uri) } c.setState(cs.State) diff --git a/cluster_internal_test.go b/cluster_internal_test.go index b35670609..648164ca7 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -23,8 +23,16 @@ import ( func TestFragCombos(t *testing.T) { c := NewCluster() - c.AddNode("host0") - c.AddNode("host1") + uri0, err := NewURIFromAddress("host0") + if err != nil { + t.Fatal(err) + } + uri1, err := NewURIFromAddress("host1") + if err != nil { + t.Fatal(err) + } + c.AddNode(*uri0) + c.AddNode(*uri1) tests := []struct { idx string @@ -37,8 +45,8 @@ func TestFragCombos(t *testing.T) { maxSlice: uint64(2), frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, expected: fragsByHost{ - "host0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, - "host1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, + URI{"http", "host0", 10101}: []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, + URI{"http", "host1", 10101}: []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, }, }, { @@ -46,8 +54,8 @@ func TestFragCombos(t *testing.T) { maxSlice: uint64(3), frameViews: viewsByFrame{"f": []string{"v0"}}, expected: fragsByHost{ - "host0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, - "host1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, + URI{"http", "host0", 10101}: []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, + URI{"http", "host1", 10101}: []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, }, }, } diff --git a/cluster_test.go b/cluster_test.go index 4cc54286f..0be636181 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -22,7 +22,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" ) @@ -30,9 +29,9 @@ import ( func TestCluster_Owners(t *testing.T) { c := pilosa.Cluster{ Nodes: []*pilosa.Node{ - {Host: "serverA:1000"}, - {Host: "serverB:1000"}, - {Host: "serverC:1000"}, + {URI: test.NewURIFromHostPort("serverA", 1000)}, + {URI: test.NewURIFromHostPort("serverB", 1000)}, + {URI: test.NewURIFromHostPort("serverC", 1000)}, }, Hasher: test.NewModHasher(), ReplicaN: 2, @@ -127,23 +126,25 @@ func TestCluster_NodeStates(t *testing.T) { // Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { c := test.NewCluster(5) - slices := c.OwnsSlices("test", 10, "host2") + slices := c.OwnsSlices("test", 10, test.NewURIFromHostPort("host2", 0)) if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { t.Fatalf("unexpected slices for node's index: %v", slices) } } +// TODO travis: fix these tests +/* func TestCluster_Nodes(t *testing.T) { nodes := []*pilosa.Node{ - &pilosa.Node{Host: "node0"}, - &pilosa.Node{Host: "node1"}, - &pilosa.Node{Host: "node2"}, + {URI: test.NewURIFromHostPort("node0", 0)}, + {URI: test.NewURIFromHostPort("node1", 0)}, + {URI: test.NewURIFromHostPort("node2", 0)}, } - t.Run("Hosts", func(t *testing.T) { - actual := pilosa.Nodes(nodes).Hosts() + t.Run("URISet", func(t *testing.T) { + actual := pilosa.Nodes(nodes).URIs() expected := []string{"node0", "node1", "node2"} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) @@ -315,3 +316,4 @@ func TestCluster_Resize(t *testing.T) { } }) } +*/ diff --git a/config.go b/config.go index e14dc9851..918c0415a 100644 --- a/config.go +++ b/config.go @@ -29,7 +29,7 @@ const ( // DefaultHost is the default hostname to use. DefaultHost = "localhost" - // DefaultPort is the default port use with the hostname. + // DefaultPort is the default port to use with the hostname. DefaultPort = "10101" // DefaultClusterType sets the node intercommunication method. diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 3db4b3b73..bb541b344 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,9 +50,9 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = uri + s.Handler.URI = *uri s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder cm := NewBackupCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") diff --git a/ctl/export_test.go b/ctl/export_test.go index 401b902e9..9e4381d39 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,9 +63,9 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = uri + s.Handler.URI = *uri s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder cm.Host = s.Host() diff --git a/ctl/import_test.go b/ctl/import_test.go index 5979bdbee..af38292cd 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,9 +69,9 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = uri + s.Handler.URI = *uri s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder cm.Host = s.Host() @@ -109,9 +109,9 @@ func TestImportCommand_RunValue(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = uri + s.Handler.URI = *uri s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder cm.Host = s.Host() diff --git a/ctl/restore_test.go b/ctl/restore_test.go index 9dd2d3661..60a0be9f6 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,9 +52,9 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = uri + s.Handler.URI = *uri s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder cm := NewRestoreCommand(stdin, stdout, stderr) diff --git a/event.go b/event.go index 420cb2073..0b7485f76 100644 --- a/event.go +++ b/event.go @@ -27,7 +27,7 @@ const ( // NodeEvent is a single event related to node activity in the cluster. type NodeEvent struct { Event NodeEventType - Host string // HostPort + URI URI } // EventHandler is the interface for the pilosa object which knows how to diff --git a/executor.go b/executor.go index 64a068149..00b12c9c2 100644 --- a/executor.go +++ b/executor.go @@ -43,8 +43,7 @@ type Executor struct { Holder *Holder // Local hostname & cluster configuration. - Scheme string - Host string + URI URI Cluster *Cluster // Client used for remote HTTP requests. @@ -962,7 +961,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql ret := false for _, node := range e.Cluster.FragmentNodes(index, slice) { // Update locally if host matches. - if node.Host == e.Host { + if node.URI == e.URI { val, err := f.ClearBit(view, rowID, colID, nil) if err != nil { return false, err @@ -1067,7 +1066,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C for _, node := range e.Cluster.FragmentNodes(index, slice) { // Update locally if host matches. - if node.Host == e.Host { + if node.URI == e.URI { val, err := f.SetBit(view, rowID, colID, timestamp) if err != nil { return false, err @@ -1146,7 +1145,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1204,7 +1203,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1291,7 +1290,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1350,7 +1349,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1384,7 +1383,6 @@ func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Qu // Create HTTP request. u := nodePathToURL(node, fmt.Sprintf("/index/%s/query", index)) - u.Scheme = e.Scheme req, err := http.NewRequest("POST", (&u).String(), bytes.NewReader(buf)) if err != nil { return nil, err @@ -1492,7 +1490,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.NodeByHost(e.Host)} + nodes = []*Node{e.Cluster.NodeByURI(e.URI)} } // Start mapping across all primary owners. @@ -1548,7 +1546,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod resp := mapResponse{node: n, slices: nodeSlices} // Send local slices to mapper, otherwise remote exec. - if n.Host == e.Host { + if n.URI == e.URI { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) diff --git a/executor_test.go b/executor_test.go index bf7550b25..34fe21e9a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -870,8 +870,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Scheme = "http" - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -914,7 +918,13 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to return a count. s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -944,7 +954,13 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments. var remoteCalled bool @@ -990,7 +1006,13 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments. var remoteCalled bool @@ -1037,7 +1059,13 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. var remoteExecN int diff --git a/fragment.go b/fragment.go index 67f9ff000..172fc22a3 100644 --- a/fragment.go +++ b/fragment.go @@ -1677,7 +1677,7 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - Host string + URI URI Cluster *Cluster ClientOptions *ClientOptions @@ -1707,14 +1707,14 @@ func (s *FragmentSyncer) SyncFragment() error { blockSets := make([][]FragmentBlock, 0, len(nodes)) for _, node := range nodes { // Read local blocks. - if node.Host == s.Host { + if node.URI == s.URI { b := s.Fragment.Blocks() blockSets = append(blockSets, b) continue } // Retrieve remote blocks. - client, err := NewClient(node.Host, s.ClientOptions) + client, err := NewClientFromURI(&node.URI, s.ClientOptions) if err != nil { return err } @@ -1784,7 +1784,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { var pairSets []PairSet var clients []*Client for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { - if s.Host == node.Host { + if s.URI == node.URI { continue } @@ -1793,7 +1793,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client, err := NewClient(node.Host, s.ClientOptions) + client, err := NewClientFromURI(&node.URI, s.ClientOptions) if err != nil { return err } diff --git a/gossip/gossip.go b/gossip/gossip.go index a579cf0b8..e17588e51 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -48,7 +48,9 @@ type GossipNodeSet struct { func (g *GossipNodeSet) Nodes() []*pilosa.Node { a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { - a = append(a, &pilosa.Node{Scheme: "gossip", Host: n.Name}) + uri, _ := pilosa.NewURIFromAddress(n.Name) + // TODO don't swallow the error above + a = append(a, &pilosa.Node{URI: *uri}) } return a } @@ -78,9 +80,15 @@ func (g *GossipNodeSet) Open() error { RetransmitMult: 3, } + uri, err := pilosa.NewURIFromAddress(g.config.gossipSeed) + if err != nil { + return err + } + // attach to gossip seed node - nodes := []*pilosa.Node{&pilosa.Node{Scheme: "gossip", Host: g.config.gossipSeed}} //TODO: support a list of seeds - err = g.joinWithRetry(pilosa.Nodes(nodes).Hosts()) + nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds + + err = g.joinWithRetry(pilosa.URISet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) if err != nil { return err } @@ -205,7 +213,7 @@ func (g *GossipNodeSet) SendTo(to *pilosa.Node, pb proto.Message) error { // Get the memberlist.Node from the pilosa.Node. for _, node := range mlist.Members() { - if node.Name == to.Host { + if node.Name == to.URI.String() { return mlist.SendToTCP(node, msg) } } @@ -322,9 +330,12 @@ func (g *GossipEventReceiver) listen() { continue } + uri, _ := pilosa.NewURIFromAddress(e.Node.Name) + // TODO: don't swallow this error + ne := &pilosa.NodeEvent{ Event: nodeEventType, - Host: e.Node.Name, + URI: *uri, } g.eventHandler.ReceiveEvent(ne) } diff --git a/handler.go b/handler.go index 016e3a348..0251461ae 100644 --- a/handler.go +++ b/handler.go @@ -56,7 +56,7 @@ type Handler struct { StatusHandler StatusHandler // Local hostname & cluster configuration. - URI *URI + URI URI Cluster *Cluster ClientOptions *ClientOptions @@ -1168,7 +1168,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) { + if !h.Cluster.OwnsFragment(h.URI, req.Index, req.Slice) { mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) http.Error(w, mesg, http.StatusPreconditionFailed) return @@ -1238,7 +1238,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) { + if !h.Cluster.OwnsFragment(h.URI, req.Index, req.Slice) { mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) http.Error(w, mesg, http.StatusPreconditionFailed) return @@ -1304,7 +1304,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), index, slice) { + if !h.Cluster.OwnsFragment(h.URI, index, slice) { mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice) http.Error(w, mesg, http.StatusPreconditionFailed) return @@ -1536,7 +1536,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // Loop over each slice and import it if this node owns it. for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), indexName, slice) { + if !h.Cluster.OwnsFragment(h.URI, indexName, slice) { continue } diff --git a/holder.go b/holder.go index d98a5d52b..909233159 100644 --- a/holder.go +++ b/holder.go @@ -434,7 +434,7 @@ func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.Lstd type HolderSyncer struct { Holder *Holder - URI *URI + URI URI Cluster *Cluster ClientOptions *ClientOptions @@ -485,7 +485,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.URI.HostPort(), di.Name, slice) { + if !s.Cluster.OwnsFragment(s.URI, di.Name, slice) { continue } @@ -521,8 +521,8 @@ func (s *HolderSyncer) syncIndex(index string) error { } // Sync with every other host. - for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewClient(node.Host, s.ClientOptions) + for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { + client, err := NewClientFromURI(&node.URI, s.ClientOptions) if err != nil { return err } @@ -566,8 +566,8 @@ func (s *HolderSyncer) syncFrame(index, name string) error { } // Sync with every other host. - for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewClient(node.Host, s.ClientOptions) + for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { + client, err := NewClientFromURI(&node.URI, s.ClientOptions) if err != nil { return err } @@ -621,7 +621,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ Fragment: frag, - Host: s.URI.HostPort(), + URI: s.URI, Cluster: s.Cluster, Closing: s.Closing, ClientOptions: s.ClientOptions, diff --git a/holder_test.go b/holder_test.go index 6b89bf46e..0427c577e 100644 --- a/holder_test.go +++ b/holder_test.go @@ -347,16 +347,21 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(nil) e.Holder = hldr1.Holder - e.Scheme = cluster.Nodes[1].Scheme - e.Host = cluster.Nodes[1].Host + e.URI = cluster.Nodes[1].URI e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].Host = "localhost:0" - cluster.Nodes[1].Host = test.MustParseURLHost(s.URL) + + uri, err := pilosa.NewURIFromAddress(s.URL) + if err != nil { + t.Fatal(err) + } + + cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[1].URI = *uri // Create frames on nodes. for _, hldr := range []*test.Holder{hldr0, hldr1} { @@ -408,13 +413,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr0.Index("y").SetRemoteMaxSlice(3) // Set up syncer. - uri, err := cluster.Nodes[0].URI() - if err != nil { - t.Fatal(err) - } syncer := pilosa.HolderSyncer{ Holder: hldr0.Holder, - URI: uri, + URI: cluster.Nodes[0].URI, Cluster: cluster, } diff --git a/internal/private.pb.go b/internal/private.pb.go index 0901f65d6..707d89cf5 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -27,6 +27,7 @@ InputDefinitionAction CreateInputDefinitionMessage DeleteInputDefinitionMessage + URI NodeStatus ClusterStatus FrameSchema @@ -628,23 +629,55 @@ func (m *DeleteInputDefinitionMessage) GetName() string { return "" } +type URI struct { + Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` + Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` + Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` +} + +func (m *URI) Reset() { *m = URI{} } +func (m *URI) String() string { return proto.CompactTextString(m) } +func (*URI) ProtoMessage() {} +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } + +func (m *URI) GetScheme() string { + if m != nil { + return m.Scheme + } + return "" +} + +func (m *URI) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *URI) GetPort() uint32 { + if m != nil { + return m.Port + } + return 0 +} + type NodeStatus struct { - Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` - HostList []string `protobuf:"bytes,4,rep,name=HostList" json:"HostList,omitempty"` + URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` + URISet []*URI `protobuf:"bytes,4,rep,name=URISet" json:"URISet,omitempty"` } func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } -func (m *NodeStatus) GetHost() string { +func (m *NodeStatus) GetURI() *URI { if m != nil { - return m.Host + return m.URI } - return "" + return nil } func (m *NodeStatus) GetState() string { @@ -661,22 +694,22 @@ func (m *NodeStatus) GetIndexes() []*Index { return nil } -func (m *NodeStatus) GetHostList() []string { +func (m *NodeStatus) GetURISet() []*URI { if m != nil { - return m.HostList + return m.URISet } return nil } type ClusterStatus struct { - State string `protobuf:"bytes,1,opt,name=State,proto3" json:"State,omitempty"` - HostList []string `protobuf:"bytes,2,rep,name=HostList" json:"HostList,omitempty"` + State string `protobuf:"bytes,1,opt,name=State,proto3" json:"State,omitempty"` + URISet []*URI `protobuf:"bytes,2,rep,name=URISet" json:"URISet,omitempty"` } func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } func (m *ClusterStatus) GetState() string { if m != nil { @@ -685,9 +718,9 @@ func (m *ClusterStatus) GetState() string { return "" } -func (m *ClusterStatus) GetHostList() []string { +func (m *ClusterStatus) GetURISet() []*URI { if m != nil { - return m.HostList + return m.URISet } return nil } @@ -699,7 +732,7 @@ type FrameSchema struct { func (m *FrameSchema) Reset() { *m = FrameSchema{} } func (m *FrameSchema) String() string { return proto.CompactTextString(m) } func (*FrameSchema) ProtoMessage() {} -func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *FrameSchema) GetFields() []*Field { if m != nil { @@ -718,7 +751,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *Field) GetName() string { if m != nil { @@ -757,7 +790,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -782,15 +815,15 @@ func (m *DeleteViewMessage) GetView() string { type ResizeInstruction struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` - Coordinator string `protobuf:"bytes,3,opt,name=Coordinator,proto3" json:"Coordinator,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + Coordinator *URI `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` } func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -799,18 +832,18 @@ func (m *ResizeInstruction) GetJobID() int64 { return 0 } -func (m *ResizeInstruction) GetHost() string { +func (m *ResizeInstruction) GetURI() *URI { if m != nil { - return m.Host + return m.URI } - return "" + return nil } -func (m *ResizeInstruction) GetCoordinator() string { +func (m *ResizeInstruction) GetCoordinator() *URI { if m != nil { return m.Coordinator } - return "" + return nil } func (m *ResizeInstruction) GetSources() []*ResizeSource { @@ -821,7 +854,7 @@ func (m *ResizeInstruction) GetSources() []*ResizeSource { } type ResizeSource struct { - Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` + URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` @@ -831,13 +864,13 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } -func (m *ResizeSource) GetHost() string { +func (m *ResizeSource) GetURI() *URI { if m != nil { - return m.Host + return m.URI } - return "" + return nil } func (m *ResizeSource) GetIndex() string { @@ -869,15 +902,15 @@ func (m *ResizeSource) GetSlice() uint64 { } type ResizeInstructionComplete struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` } func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComplete{} } func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{26} + return fileDescriptorPrivate, []int{27} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -887,25 +920,25 @@ func (m *ResizeInstructionComplete) GetJobID() int64 { return 0 } -func (m *ResizeInstructionComplete) GetHost() string { +func (m *ResizeInstructionComplete) GetURI() *URI { if m != nil { - return m.Host + return m.URI } - return "" + return nil } type Topology struct { - HostList []string `protobuf:"bytes,1,rep,name=HostList" json:"HostList,omitempty"` + URISet []*URI `protobuf:"bytes,1,rep,name=URISet" json:"URISet,omitempty"` } func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } -func (m *Topology) GetHostList() []string { +func (m *Topology) GetURISet() []*URI { if m != nil { - return m.HostList + return m.URISet } return nil } @@ -930,6 +963,7 @@ func init() { proto.RegisterType((*InputDefinitionAction)(nil), "internal.InputDefinitionAction") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") + proto.RegisterType((*URI)(nil), "internal.URI") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") @@ -1724,6 +1758,41 @@ func (m *DeleteInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *URI) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *URI) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Scheme) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) + i += copy(dAtA[i:], m.Scheme) + } + if len(m.Host) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + if m.Port != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) + } + return i, nil +} + func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1739,11 +1808,15 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Host) > 0 { + if m.URI != nil { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i += copy(dAtA[i:], m.Host) + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n14, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n14 } if len(m.State) > 0 { dAtA[i] = 0x12 @@ -1763,19 +1836,16 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.HostList) > 0 { - for _, s := range m.HostList { + if len(m.URISet) > 0 { + for _, msg := range m.URISet { dAtA[i] = 0x22 i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + i += n } } return i, nil @@ -1802,19 +1872,16 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if len(m.HostList) > 0 { - for _, s := range m.HostList { + if len(m.URISet) > 0 { + for _, msg := range m.URISet { dAtA[i] = 0x12 i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + i += n } } return i, nil @@ -1946,17 +2013,25 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) } - if len(m.Host) > 0 { + if m.URI != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i += copy(dAtA[i:], m.Host) + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n15, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n15 } - if len(m.Coordinator) > 0 { + if m.Coordinator != nil { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Coordinator))) - i += copy(dAtA[i:], m.Coordinator) + i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) + n16, err := m.Coordinator.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -1988,11 +2063,15 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Host) > 0 { + if m.URI != nil { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i += copy(dAtA[i:], m.Host) + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n17, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n17 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2040,11 +2119,15 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) } - if len(m.Host) > 0 { + if m.URI != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i += copy(dAtA[i:], m.Host) + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n18, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n18 } return i, nil } @@ -2064,19 +2147,16 @@ func (m *Topology) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.HostList) > 0 { - for _, s := range m.HostList { + if len(m.URISet) > 0 { + for _, msg := range m.URISet { dAtA[i] = 0xa i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + i += n } } return i, nil @@ -2435,11 +2515,28 @@ func (m *DeleteInputDefinitionMessage) Size() (n int) { return n } +func (m *URI) Size() (n int) { + var l int + _ = l + l = len(m.Scheme) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Port != 0 { + n += 1 + sovPrivate(uint64(m.Port)) + } + return n +} + func (m *NodeStatus) Size() (n int) { var l int _ = l - l = len(m.Host) - if l > 0 { + if m.URI != nil { + l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } l = len(m.State) @@ -2452,9 +2549,9 @@ func (m *NodeStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if len(m.HostList) > 0 { - for _, s := range m.HostList { - l = len(s) + if len(m.URISet) > 0 { + for _, e := range m.URISet { + l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } } @@ -2468,9 +2565,9 @@ func (m *ClusterStatus) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.HostList) > 0 { - for _, s := range m.HostList { - l = len(s) + if len(m.URISet) > 0 { + for _, e := range m.URISet { + l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } } @@ -2533,12 +2630,12 @@ func (m *ResizeInstruction) Size() (n int) { if m.JobID != 0 { n += 1 + sovPrivate(uint64(m.JobID)) } - l = len(m.Host) - if l > 0 { + if m.URI != nil { + l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Coordinator) - if l > 0 { + if m.Coordinator != nil { + l = m.Coordinator.Size() n += 1 + l + sovPrivate(uint64(l)) } if len(m.Sources) > 0 { @@ -2553,8 +2650,8 @@ func (m *ResizeInstruction) Size() (n int) { func (m *ResizeSource) Size() (n int) { var l int _ = l - l = len(m.Host) - if l > 0 { + if m.URI != nil { + l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } l = len(m.Index) @@ -2581,8 +2678,8 @@ func (m *ResizeInstructionComplete) Size() (n int) { if m.JobID != 0 { n += 1 + sovPrivate(uint64(m.JobID)) } - l = len(m.Host) - if l > 0 { + if m.URI != nil { + l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } return n @@ -2591,9 +2688,9 @@ func (m *ResizeInstructionComplete) Size() (n int) { func (m *Topology) Size() (n int) { var l int _ = l - if len(m.HostList) > 0 { - for _, s := range m.HostList { - l = len(s) + if len(m.URISet) > 0 { + for _, e := range m.URISet { + l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } } @@ -5295,6 +5392,133 @@ func (m *DeleteInputDefinitionMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *URI) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: URI: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: URI: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Scheme", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Scheme = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) + } + m.Port = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Port |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5326,9 +5550,9 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -5338,20 +5562,24 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Host = string(dAtA[iNdEx:postIndex]) + if m.URI == nil { + m.URI = &URI{} + } + if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 2: if wireType != 2 { @@ -5415,9 +5643,9 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HostList", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URISet", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -5427,20 +5655,22 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.HostList = append(m.HostList, string(dAtA[iNdEx:postIndex])) + m.URISet = append(m.URISet, &URI{}) + if err := m.URISet[len(m.URISet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -5523,9 +5753,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HostList", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URISet", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -5535,20 +5765,22 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.HostList = append(m.HostList, string(dAtA[iNdEx:postIndex])) + m.URISet = append(m.URISet, &URI{}) + if err := m.URISet[len(m.URISet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -5985,9 +6217,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -5997,26 +6229,30 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Host = string(dAtA[iNdEx:postIndex]) + if m.URI == nil { + m.URI = &URI{} + } + if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Coordinator", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6026,20 +6262,24 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Coordinator = string(dAtA[iNdEx:postIndex]) + if m.Coordinator == nil { + m.Coordinator = &URI{} + } + if err := m.Coordinator.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 4: if wireType != 2 { @@ -6124,9 +6364,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6136,20 +6376,24 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Host = string(dAtA[iNdEx:postIndex]) + if m.URI == nil { + m.URI = &URI{} + } + if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 2: if wireType != 2 { @@ -6328,9 +6572,9 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6340,20 +6584,24 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Host = string(dAtA[iNdEx:postIndex]) + if m.URI == nil { + m.URI = &URI{} + } + if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -6407,9 +6655,9 @@ func (m *Topology) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HostList", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field URISet", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6419,20 +6667,22 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.HostList = append(m.HostList, string(dAtA[iNdEx:postIndex])) + m.URISet = append(m.URISet, &URI{}) + if err := m.URISet[len(m.URISet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -6563,72 +6813,75 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1063 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0x66, 0x3c, 0xe3, 0xc4, 0x2e, 0x6f, 0x12, 0x67, 0x58, 0x22, 0x27, 0x8a, 0x8c, 0xd5, 0x87, - 0xdd, 0x10, 0x89, 0x08, 0x05, 0x69, 0xc5, 0xdf, 0x81, 0x5d, 0xdb, 0xab, 0x0c, 0xac, 0x17, 0x68, - 0x47, 0xcb, 0x0d, 0xa9, 0x63, 0x37, 0xd9, 0x51, 0xc6, 0xd3, 0x66, 0xa6, 0x27, 0x89, 0x57, 0x82, - 0x23, 0x77, 0x6e, 0x48, 0x1c, 0x79, 0x19, 0x8e, 0x3c, 0x02, 0x0a, 0x17, 0xde, 0x00, 0x89, 0x13, - 0xea, 0xea, 0x9e, 0x1f, 0xff, 0x66, 0xb3, 0xb7, 0xa9, 0xaf, 0xab, 0xab, 0xbe, 0xaa, 0xae, 0xaa, - 0x29, 0xd8, 0x18, 0x47, 0xfe, 0x25, 0x93, 0xfc, 0x68, 0x1c, 0x09, 0x29, 0xdc, 0x8a, 0x1f, 0x4a, - 0x1e, 0x85, 0x2c, 0x20, 0x5f, 0x41, 0xd5, 0x0b, 0x87, 0xfc, 0xba, 0xc7, 0x25, 0x73, 0x5b, 0x50, - 0x6b, 0x8b, 0x20, 0x19, 0x85, 0xcf, 0xd8, 0x19, 0x0f, 0x1a, 0x56, 0xcb, 0x3a, 0xa8, 0xd2, 0x22, - 0xa4, 0x34, 0x4e, 0xfd, 0x11, 0xff, 0x26, 0x61, 0xa1, 0x4c, 0x46, 0x8d, 0x92, 0xd6, 0x28, 0x40, - 0xe4, 0x3f, 0x0b, 0xaa, 0x4f, 0x23, 0x36, 0xe2, 0x68, 0x71, 0x0f, 0x2a, 0x54, 0x5c, 0x15, 0xcd, - 0x65, 0xb2, 0xfb, 0x00, 0x36, 0xbd, 0xf0, 0x92, 0x47, 0x31, 0xef, 0x86, 0xec, 0x2c, 0xe0, 0x43, - 0x34, 0x57, 0xa1, 0x33, 0xa8, 0xbb, 0x0f, 0xd5, 0x36, 0x1b, 0xbc, 0xe4, 0xa7, 0x93, 0x31, 0x6f, - 0xd8, 0x68, 0x24, 0x07, 0xb2, 0xd3, 0xbe, 0xff, 0x8a, 0x37, 0x9c, 0x96, 0x75, 0xb0, 0x41, 0x73, - 0x60, 0x96, 0x6f, 0x79, 0x8e, 0xaf, 0x4b, 0xe0, 0x1e, 0x65, 0xe1, 0x79, 0xc6, 0x61, 0x0d, 0x39, - 0x4c, 0x61, 0xee, 0x43, 0x58, 0x7b, 0xea, 0xf3, 0x60, 0x18, 0x37, 0xd6, 0x5b, 0xf6, 0x41, 0xed, - 0x78, 0xeb, 0x28, 0xcd, 0xdf, 0x11, 0xe2, 0xd4, 0x1c, 0x13, 0x02, 0x9b, 0xde, 0x68, 0x2c, 0x22, - 0x49, 0x79, 0x3c, 0x16, 0x61, 0xcc, 0xdd, 0x3a, 0xd8, 0xdd, 0x28, 0x32, 0xb1, 0xab, 0x4f, 0xf2, - 0x13, 0xd4, 0x9f, 0x04, 0x62, 0x70, 0xd1, 0x61, 0x92, 0x51, 0xfe, 0x43, 0xc2, 0x63, 0xe9, 0xde, - 0x87, 0x32, 0xbe, 0x82, 0xd1, 0xd3, 0x82, 0x42, 0x31, 0x93, 0x26, 0xcd, 0x5a, 0x50, 0x28, 0xde, - 0xc7, 0x54, 0x38, 0x54, 0x0b, 0x0a, 0xed, 0x07, 0xfe, 0x40, 0xa7, 0xc0, 0xa1, 0x5a, 0x70, 0x5d, - 0x70, 0x5e, 0xf8, 0xfc, 0xca, 0xc4, 0x8d, 0xdf, 0xc4, 0x83, 0xed, 0x82, 0x7f, 0x43, 0x73, 0x07, - 0xd6, 0xa8, 0xb8, 0xf2, 0x3a, 0x71, 0xc3, 0x6a, 0xd9, 0x07, 0x0e, 0x35, 0x12, 0x66, 0x17, 0x9f, - 0x5f, 0x1d, 0x95, 0xf0, 0x28, 0x07, 0xc8, 0x2e, 0x94, 0x31, 0xd5, 0x2a, 0xca, 0xfc, 0xae, 0xfa, - 0x24, 0xbf, 0x59, 0xb0, 0xdd, 0x63, 0xd7, 0x48, 0x23, 0xce, 0xdc, 0x9c, 0x40, 0x35, 0x03, 0x51, - 0xbb, 0x76, 0x7c, 0x98, 0xe7, 0x72, 0x4e, 0x3f, 0x47, 0xba, 0xa1, 0x8c, 0x26, 0x34, 0xbf, 0xbc, - 0xf7, 0x19, 0x6c, 0x4e, 0x1f, 0x2a, 0x0e, 0x17, 0x7c, 0x92, 0x66, 0xfa, 0x82, 0x4f, 0x54, 0x4e, - 0x2e, 0x59, 0x90, 0xe8, 0xfc, 0x39, 0x54, 0x0b, 0x9f, 0x94, 0x3e, 0xb2, 0xc8, 0x77, 0xe0, 0xb6, - 0x23, 0xce, 0x24, 0x47, 0x03, 0x3d, 0x1e, 0xc7, 0xec, 0x9c, 0x2f, 0x7f, 0x05, 0x9d, 0xd9, 0x52, - 0x31, 0xb3, 0xfb, 0x50, 0xf5, 0x62, 0x53, 0xa8, 0xf8, 0x12, 0x15, 0x9a, 0x03, 0xe4, 0x10, 0xdc, - 0x0e, 0x0f, 0xb8, 0xe4, 0xa6, 0xb7, 0x56, 0xd8, 0x27, 0xfd, 0x94, 0xcb, 0xed, 0xba, 0xee, 0x43, - 0x70, 0x54, 0x5b, 0x21, 0x95, 0xda, 0xf1, 0xdb, 0x79, 0xea, 0xb2, 0x1e, 0xa6, 0xa8, 0x40, 0xfc, - 0xd4, 0xa8, 0x69, 0xc5, 0x5b, 0x02, 0x5c, 0x50, 0x66, 0xa9, 0x2b, 0x7b, 0xd6, 0x55, 0xd6, 0xdc, - 0xc6, 0xd5, 0xe7, 0x69, 0xac, 0x6f, 0xea, 0x8a, 0x74, 0x0c, 0xaa, 0xca, 0xf5, 0xb9, 0x3a, 0xd5, - 0x77, 0xf0, 0x7b, 0x79, 0xc8, 0xb3, 0x3c, 0xfe, 0xb1, 0x8c, 0xcb, 0xbb, 0x99, 0x99, 0xc9, 0x9c, - 0x9a, 0x58, 0x69, 0x61, 0x99, 0x0e, 0xcb, 0x64, 0x9c, 0x03, 0xca, 0x6b, 0xdc, 0x70, 0xe6, 0xe6, - 0x80, 0xc2, 0xa9, 0x39, 0x56, 0xed, 0x64, 0x8a, 0xbc, 0xac, 0xdb, 0x49, 0x4b, 0x6e, 0x17, 0xea, - 0x5e, 0x38, 0x4e, 0x64, 0x87, 0x7f, 0xef, 0x87, 0xbe, 0xf4, 0x45, 0x18, 0x37, 0xd6, 0xd0, 0xd4, - 0x6e, 0x91, 0xd1, 0x94, 0x06, 0x9d, 0xbb, 0x42, 0x7e, 0xb6, 0x60, 0x6b, 0x06, 0x5c, 0x12, 0x74, - 0xca, 0xb7, 0xb4, 0x9a, 0xef, 0xa3, 0x6c, 0xc0, 0xd9, 0xa8, 0xd8, 0x5c, 0xca, 0x66, 0x7a, 0xde, - 0xfd, 0x6e, 0xc1, 0xfd, 0x45, 0x0a, 0x0b, 0xd9, 0x34, 0x01, 0xbe, 0x8e, 0xfc, 0x11, 0x8b, 0x26, - 0x5f, 0xf2, 0x89, 0x99, 0xf5, 0x05, 0xc4, 0xfd, 0x16, 0x76, 0x66, 0x6c, 0x3d, 0x1e, 0xe8, 0x14, - 0x69, 0x52, 0xef, 0x2e, 0x25, 0xa5, 0xf5, 0xe8, 0x92, 0xeb, 0xe4, 0x5f, 0x0b, 0xde, 0x59, 0x78, - 0x94, 0xd7, 0xa3, 0x55, 0x2c, 0xfd, 0x43, 0xa8, 0xbf, 0x50, 0xa3, 0xa2, 0xc3, 0x63, 0xe9, 0x87, - 0x4c, 0x69, 0x9a, 0x82, 0x9d, 0xc3, 0x5d, 0x0f, 0x2a, 0x88, 0xf5, 0xd8, 0xd8, 0xd0, 0x7c, 0xff, - 0x16, 0x9a, 0x47, 0xa9, 0xbe, 0x9e, 0x69, 0xd9, 0x75, 0x45, 0x06, 0xa7, 0x6e, 0x3a, 0xc2, 0x51, - 0xd8, 0xfb, 0x14, 0x36, 0xa6, 0x2e, 0xdc, 0x69, 0xce, 0x09, 0xd8, 0x4f, 0x67, 0xcb, 0x14, 0x93, - 0xd5, 0x5d, 0xfa, 0x31, 0x40, 0xae, 0x6a, 0x06, 0xc0, 0x8a, 0xfa, 0x2c, 0x28, 0x93, 0x13, 0xd8, - 0x4f, 0x07, 0xdf, 0x1d, 0x1c, 0xa6, 0xd5, 0x52, 0xca, 0xab, 0x85, 0xfc, 0x08, 0xf0, 0x5c, 0x0c, - 0x79, 0x5f, 0x32, 0x99, 0xc4, 0x4a, 0xe3, 0x44, 0xc4, 0x32, 0xad, 0x27, 0xf5, 0x8d, 0x83, 0x59, - 0x32, 0x99, 0x0d, 0x13, 0x14, 0xdc, 0xf7, 0x60, 0x1d, 0x8d, 0xf2, 0xb4, 0x6c, 0xb6, 0x66, 0x7a, - 0x9d, 0xa6, 0xe7, 0xaa, 0xd5, 0x95, 0xa1, 0x67, 0x7e, 0x2c, 0xb1, 0xa1, 0xab, 0x34, 0x93, 0xc9, - 0x63, 0xd8, 0x68, 0x07, 0x49, 0x2c, 0x79, 0x64, 0x18, 0x64, 0xde, 0xac, 0xa2, 0xb7, 0xa2, 0x89, - 0xd2, 0x8c, 0x89, 0x47, 0x50, 0xc3, 0x7a, 0xea, 0x0f, 0x5e, 0xf2, 0x11, 0x2b, 0x2c, 0x11, 0xd6, - 0xea, 0x25, 0xa2, 0x0f, 0xe5, 0xe5, 0x4d, 0xe4, 0x82, 0x83, 0x7b, 0x90, 0x49, 0x15, 0xae, 0x40, - 0x75, 0xb0, 0x7b, 0xbe, 0x7e, 0x28, 0x9b, 0xaa, 0x4f, 0x44, 0xd8, 0x35, 0x16, 0x92, 0x42, 0x98, - 0xfa, 0xcb, 0x6c, 0xeb, 0x87, 0x51, 0x3b, 0xc0, 0x9b, 0xfc, 0x0f, 0xd2, 0x55, 0xc2, 0x2e, 0xac, - 0x12, 0xbf, 0x58, 0xb0, 0x4d, 0x79, 0xec, 0xbf, 0xe2, 0x5e, 0x18, 0xcb, 0x28, 0xc9, 0x9a, 0xea, - 0x0b, 0x71, 0xe6, 0x75, 0xd0, 0xaa, 0x4d, 0xb5, 0x90, 0xbd, 0x60, 0xa9, 0xf0, 0x82, 0xb8, 0x6f, - 0x8a, 0x68, 0xa8, 0x9a, 0x49, 0x44, 0xc6, 0x74, 0x11, 0x72, 0x3f, 0x80, 0xf5, 0xbe, 0x48, 0xa2, - 0x41, 0x36, 0x72, 0x77, 0xf2, 0xac, 0x69, 0xcf, 0xfa, 0x98, 0xa6, 0x6a, 0xe4, 0x1a, 0xee, 0x15, - 0x0f, 0x96, 0x55, 0x8e, 0x8e, 0xbb, 0xb4, 0x30, 0x6e, 0x7b, 0x51, 0xdc, 0x4e, 0x1e, 0x77, 0xbe, - 0x12, 0x94, 0x0b, 0x2b, 0x01, 0xe9, 0xc2, 0xee, 0x5c, 0x32, 0xda, 0x62, 0x34, 0x56, 0x59, 0x7f, - 0xfd, 0xa4, 0x90, 0x07, 0x50, 0x39, 0x15, 0x63, 0x11, 0x88, 0xf3, 0xc9, 0x54, 0x79, 0x59, 0xd3, - 0xe5, 0xf5, 0xa4, 0xfe, 0xc7, 0x4d, 0xd3, 0xfa, 0xf3, 0xa6, 0x69, 0xfd, 0x75, 0xd3, 0xb4, 0x7e, - 0xfd, 0xbb, 0xf9, 0xd6, 0xd9, 0x1a, 0x2e, 0xf7, 0x1f, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xed, - 0x3a, 0xc0, 0x34, 0xed, 0x0b, 0x00, 0x00, + // 1110 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0x23, 0x45, + 0x13, 0x7e, 0xc7, 0x63, 0x3b, 0x76, 0x79, 0x9d, 0x38, 0xfd, 0x2e, 0x91, 0x13, 0x45, 0xde, 0xa8, + 0x25, 0xd8, 0x10, 0x89, 0x00, 0x41, 0x5a, 0xf1, 0x75, 0x80, 0x8d, 0xb3, 0xca, 0xc0, 0x66, 0x59, + 0xda, 0xc9, 0x72, 0x43, 0xea, 0xd8, 0x4d, 0x76, 0x94, 0xf1, 0xb4, 0x99, 0x69, 0x27, 0xf1, 0x1e, + 0xb8, 0xc1, 0x3f, 0x40, 0x42, 0xe2, 0xc8, 0x99, 0xff, 0xc1, 0x91, 0x9f, 0x80, 0xc2, 0x85, 0x7f, + 0x80, 0xc4, 0x09, 0x75, 0x75, 0xcf, 0x87, 0x3f, 0x43, 0x72, 0x9b, 0x7a, 0xba, 0xba, 0xea, 0xe9, + 0xa7, 0xab, 0xca, 0x6d, 0xa8, 0x0f, 0x22, 0xff, 0x82, 0x2b, 0xb1, 0x3b, 0x88, 0xa4, 0x92, 0xa4, + 0xe2, 0x87, 0x4a, 0x44, 0x21, 0x0f, 0xe8, 0x17, 0x50, 0xf5, 0xc2, 0x9e, 0xb8, 0x3a, 0x12, 0x8a, + 0x93, 0x2d, 0xa8, 0xed, 0xcb, 0x60, 0xd8, 0x0f, 0x9f, 0xf2, 0x53, 0x11, 0x34, 0x9d, 0x2d, 0x67, + 0xbb, 0xca, 0xf2, 0x90, 0xf6, 0x38, 0xf6, 0xfb, 0xe2, 0xcb, 0x21, 0x0f, 0xd5, 0xb0, 0xdf, 0x2c, + 0x18, 0x8f, 0x1c, 0x44, 0xff, 0x71, 0xa0, 0xfa, 0x24, 0xe2, 0x7d, 0x81, 0x11, 0x37, 0xa0, 0xc2, + 0xe4, 0x65, 0x3e, 0x5c, 0x6a, 0x93, 0x37, 0x60, 0xd9, 0x0b, 0x2f, 0x44, 0x14, 0x8b, 0x83, 0x90, + 0x9f, 0x06, 0xa2, 0x87, 0xe1, 0x2a, 0x6c, 0x02, 0x25, 0x9b, 0x50, 0xdd, 0xe7, 0xdd, 0x97, 0xe2, + 0x78, 0x34, 0x10, 0x4d, 0x17, 0x83, 0x64, 0x40, 0xba, 0xda, 0xf1, 0x5f, 0x89, 0x66, 0x71, 0xcb, + 0xd9, 0xae, 0xb3, 0x0c, 0x98, 0xe4, 0x5b, 0x9a, 0xe2, 0x4b, 0x28, 0xdc, 0x63, 0x3c, 0x3c, 0x4b, + 0x39, 0x94, 0x91, 0xc3, 0x18, 0x46, 0x1e, 0x42, 0xf9, 0x89, 0x2f, 0x82, 0x5e, 0xdc, 0x5c, 0xda, + 0x72, 0xb7, 0x6b, 0x7b, 0x2b, 0xbb, 0x89, 0x7e, 0xbb, 0x88, 0x33, 0xbb, 0x4c, 0x29, 0x2c, 0x7b, + 0xfd, 0x81, 0x8c, 0x14, 0x13, 0xf1, 0x40, 0x86, 0xb1, 0x20, 0x0d, 0x70, 0x0f, 0xa2, 0xc8, 0x9e, + 0x5d, 0x7f, 0xd2, 0xef, 0xa0, 0xf1, 0x38, 0x90, 0xdd, 0xf3, 0x36, 0x57, 0x9c, 0x89, 0x6f, 0x87, + 0x22, 0x56, 0xe4, 0x3e, 0x94, 0xf0, 0x16, 0xac, 0x9f, 0x31, 0x34, 0x8a, 0x4a, 0x5a, 0x99, 0x8d, + 0xa1, 0x51, 0xdc, 0x8f, 0x52, 0x14, 0x99, 0x31, 0x34, 0xda, 0x09, 0xfc, 0xae, 0x91, 0xa0, 0xc8, + 0x8c, 0x41, 0x08, 0x14, 0x5f, 0xf8, 0xe2, 0xd2, 0x9e, 0x1b, 0xbf, 0xa9, 0x07, 0xab, 0xb9, 0xfc, + 0x96, 0xe6, 0x1a, 0x94, 0x99, 0xbc, 0xf4, 0xda, 0x71, 0xd3, 0xd9, 0x72, 0xb7, 0x8b, 0xcc, 0x5a, + 0xa8, 0x2e, 0x5e, 0xbf, 0x5e, 0x2a, 0xe0, 0x52, 0x06, 0xd0, 0x75, 0x28, 0xa1, 0xd4, 0xfa, 0x94, + 0xd9, 0x5e, 0xfd, 0x49, 0x7f, 0x76, 0x60, 0xf5, 0x88, 0x5f, 0x21, 0x8d, 0x38, 0x4d, 0x73, 0x08, + 0xd5, 0x14, 0x44, 0xef, 0xda, 0xde, 0x4e, 0xa6, 0xe5, 0x94, 0x7f, 0x86, 0x1c, 0x84, 0x2a, 0x1a, + 0xb1, 0x6c, 0xf3, 0xc6, 0xc7, 0xb0, 0x3c, 0xbe, 0xa8, 0x39, 0x9c, 0x8b, 0x51, 0xa2, 0xf4, 0xb9, + 0x18, 0x69, 0x4d, 0x2e, 0x78, 0x30, 0x34, 0xfa, 0x15, 0x99, 0x31, 0x3e, 0x2c, 0xbc, 0xef, 0xd0, + 0xaf, 0x81, 0xec, 0x47, 0x82, 0x2b, 0x81, 0x01, 0x8e, 0x44, 0x1c, 0xf3, 0x33, 0x31, 0xff, 0x16, + 0x8c, 0xb2, 0x85, 0xbc, 0xb2, 0x9b, 0x50, 0xf5, 0x62, 0x5b, 0xa8, 0x78, 0x13, 0x15, 0x96, 0x01, + 0x74, 0x07, 0x48, 0x5b, 0x04, 0x42, 0x09, 0xdb, 0x5b, 0x0b, 0xe2, 0xd3, 0x4e, 0xc2, 0xe5, 0x66, + 0x5f, 0xf2, 0x10, 0x8a, 0xba, 0xad, 0x90, 0x4a, 0x6d, 0xef, 0xff, 0x99, 0x74, 0x69, 0x0f, 0x33, + 0x74, 0xa0, 0x7e, 0x12, 0xd4, 0xb6, 0xe2, 0x0d, 0x07, 0x9c, 0x51, 0x66, 0x49, 0x2a, 0x77, 0x32, + 0x55, 0xda, 0xdc, 0x36, 0xd5, 0x27, 0xc9, 0x59, 0xef, 0x9a, 0x8a, 0xb6, 0x2d, 0xaa, 0xcb, 0xf5, + 0x99, 0x5e, 0x35, 0x7b, 0xf0, 0x7b, 0xfe, 0x91, 0x27, 0x79, 0xfc, 0xe5, 0xd8, 0x94, 0xb7, 0x0b, + 0x33, 0xa1, 0x9c, 0x9e, 0x58, 0x49, 0x61, 0xd9, 0x0e, 0x4b, 0x6d, 0x9c, 0x03, 0x3a, 0x6b, 0xdc, + 0x2c, 0x4e, 0xcd, 0x01, 0x8d, 0x33, 0xbb, 0xac, 0xdb, 0xc9, 0x16, 0x79, 0xc9, 0xb4, 0x93, 0xb1, + 0xc8, 0x01, 0x34, 0xbc, 0x70, 0x30, 0x54, 0x6d, 0xf1, 0x8d, 0x1f, 0xfa, 0xca, 0x97, 0x61, 0xdc, + 0x2c, 0x63, 0xa8, 0xf5, 0x3c, 0xa3, 0x31, 0x0f, 0x36, 0xb5, 0x85, 0xfe, 0xe0, 0xc0, 0xca, 0x04, + 0x38, 0xe7, 0xd0, 0x09, 0xdf, 0xc2, 0x62, 0xbe, 0x8f, 0xd2, 0x01, 0xe7, 0xa2, 0x63, 0x6b, 0x2e, + 0x9b, 0xf1, 0x79, 0xf7, 0x8b, 0x03, 0xf7, 0x67, 0x39, 0xcc, 0x64, 0xd3, 0x02, 0x78, 0x1e, 0xf9, + 0x7d, 0x1e, 0x8d, 0x3e, 0x17, 0x23, 0x3b, 0xeb, 0x73, 0x08, 0xf9, 0x0a, 0xd6, 0x26, 0x62, 0x7d, + 0xda, 0x35, 0x12, 0x19, 0x52, 0x0f, 0xe6, 0x92, 0x32, 0x7e, 0x6c, 0xce, 0x76, 0xfa, 0xb7, 0x03, + 0xaf, 0xcd, 0x5c, 0xca, 0xea, 0xd1, 0xc9, 0x97, 0xfe, 0x0e, 0x34, 0x5e, 0xe8, 0x51, 0xd1, 0x16, + 0xb1, 0xf2, 0x43, 0xae, 0x3d, 0x6d, 0xc1, 0x4e, 0xe1, 0xc4, 0x83, 0x0a, 0x62, 0x47, 0x7c, 0x60, + 0x69, 0xbe, 0x75, 0x03, 0xcd, 0xdd, 0xc4, 0xdf, 0xcc, 0xb4, 0x74, 0xbb, 0x26, 0x83, 0x53, 0x37, + 0x19, 0xe1, 0x68, 0x6c, 0x7c, 0x04, 0xf5, 0xb1, 0x0d, 0xb7, 0x9a, 0x73, 0x12, 0x36, 0x93, 0xd9, + 0x32, 0xc6, 0x64, 0x71, 0x97, 0x7e, 0x00, 0x90, 0xb9, 0xda, 0x01, 0xb0, 0xa0, 0x3e, 0x73, 0xce, + 0xf4, 0x10, 0x36, 0x93, 0xc1, 0x77, 0x8b, 0x84, 0x49, 0xb5, 0x14, 0xb2, 0x6a, 0xa1, 0x07, 0xe0, + 0x9e, 0x30, 0x0f, 0x3b, 0xa9, 0xfb, 0x52, 0xa4, 0x57, 0x64, 0x2d, 0xbd, 0xe5, 0x50, 0xc6, 0x2a, + 0xd9, 0xa2, 0xbf, 0x35, 0xf6, 0x5c, 0x46, 0x0a, 0x19, 0xd7, 0x19, 0x7e, 0xd3, 0x1f, 0x1d, 0x80, + 0x67, 0xb2, 0x27, 0x3a, 0x8a, 0xab, 0x61, 0x4c, 0x1e, 0x60, 0x54, 0x8c, 0x55, 0xdb, 0xab, 0x67, + 0x67, 0x3a, 0x61, 0x1e, 0xc3, 0x7c, 0x7a, 0xda, 0x2b, 0xae, 0xd2, 0x09, 0x85, 0x06, 0x79, 0x13, + 0x96, 0x90, 0xa9, 0x48, 0x6a, 0x71, 0x65, 0x62, 0x80, 0xb0, 0x64, 0x9d, 0xbc, 0x0e, 0xe5, 0x13, + 0xe6, 0x75, 0x84, 0xb2, 0x33, 0x62, 0x22, 0x89, 0x5d, 0xa4, 0x4f, 0xa1, 0xbe, 0x1f, 0x0c, 0x63, + 0x25, 0x22, 0xcb, 0x2c, 0x4d, 0xec, 0xe4, 0x13, 0x67, 0xd1, 0x0a, 0x8b, 0xa2, 0x3d, 0x82, 0x1a, + 0x96, 0x2e, 0x8a, 0xc3, 0x73, 0xef, 0x15, 0x67, 0xf1, 0x7b, 0xa5, 0x03, 0xa5, 0xf9, 0xfd, 0x4a, + 0xa0, 0x88, 0x4f, 0x2e, 0x2b, 0x31, 0xbe, 0xb6, 0x1a, 0xe0, 0x1e, 0xf9, 0xa6, 0x26, 0x5c, 0xa6, + 0x3f, 0x11, 0xe1, 0x57, 0x58, 0xb3, 0x1a, 0xe1, 0xfa, 0x07, 0x6d, 0xd5, 0xd4, 0x80, 0x7e, 0x6e, + 0xdc, 0xe5, 0xa7, 0x27, 0x79, 0xb5, 0xb8, 0xb9, 0x57, 0xcb, 0xaf, 0x0e, 0xac, 0x32, 0x11, 0xfb, + 0xaf, 0x84, 0x17, 0xc6, 0x2a, 0x1a, 0xa6, 0xfd, 0xfb, 0x99, 0x3c, 0xf5, 0xda, 0x18, 0xd5, 0x65, + 0xc6, 0x48, 0x2e, 0xb9, 0x30, 0xf7, 0x92, 0xdf, 0xd6, 0xef, 0x5c, 0x19, 0xf5, 0x74, 0x13, 0xcb, + 0xc8, 0x56, 0xf8, 0x84, 0x63, 0xde, 0x83, 0xbc, 0x03, 0x4b, 0x1d, 0x39, 0x8c, 0xba, 0xe9, 0xe4, + 0x5f, 0xcb, 0x9c, 0x0d, 0x2b, 0xb3, 0xcc, 0x12, 0x37, 0xfa, 0xbd, 0x03, 0xf7, 0xf2, 0x2b, 0xff, + 0xa9, 0xf2, 0x8c, 0x42, 0x85, 0x99, 0x0a, 0xb9, 0xb3, 0x14, 0x2a, 0x66, 0x0a, 0x65, 0xef, 0x94, + 0x52, 0xee, 0x9d, 0x42, 0x19, 0xac, 0x4f, 0xc9, 0xb6, 0x2f, 0xfb, 0x03, 0x7d, 0x3f, 0x77, 0x94, + 0x8f, 0xbe, 0x0b, 0x95, 0x63, 0x39, 0x90, 0x81, 0x3c, 0x1b, 0xe5, 0x0a, 0xd4, 0x59, 0x50, 0xa0, + 0x8f, 0x1b, 0xbf, 0x5d, 0xb7, 0x9c, 0xdf, 0xaf, 0x5b, 0xce, 0x1f, 0xd7, 0x2d, 0xe7, 0xa7, 0x3f, + 0x5b, 0xff, 0x3b, 0x2d, 0xe3, 0x3f, 0x91, 0xf7, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x7e, 0xc9, + 0xe9, 0x9b, 0x9a, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 4268692c1..1b1bdd6fb 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -112,17 +112,22 @@ message DeleteInputDefinitionMessage { string Name = 2; } +message URI { + string Scheme = 1; + string Host = 2; + uint32 Port = 3; +} + message NodeStatus { - string Host = 1; + URI URI = 1; string State = 2; repeated Index Indexes = 3; - repeated string HostList = 4; + repeated URI URISet = 4; } message ClusterStatus { string State = 1; - repeated string HostList = 2; - //repeated NodeStatus NodeStatuses = 3; + repeated URI URISet = 2; } message FrameSchema { @@ -144,13 +149,13 @@ message DeleteViewMessage { message ResizeInstruction { int64 JobID = 1; - string Host = 2; - string Coordinator = 3; + URI URI = 2; + URI Coordinator = 3; repeated ResizeSource Sources = 4; } message ResizeSource { - string Host = 1; + URI URI = 1; string Index = 2; string Frame = 3; string View = 4; @@ -159,10 +164,10 @@ message ResizeSource { message ResizeInstructionComplete { int64 JobID = 1; - string Host = 2; + URI URI = 2; } message Topology { - repeated string HostList = 1; + repeated URI URISet = 1; } diff --git a/pilosa.go b/pilosa.go index bdab539cd..af4186513 100644 --- a/pilosa.go +++ b/pilosa.go @@ -162,8 +162,8 @@ func StringInSlice(a string, list []string) bool { return false } -// SlicesAreEqual determines if two string slices are equal. -func SlicesAreEqual(a, b []string) bool { +// URISlicesAreEqual determines if two string slices are equal. +func URISlicesAreEqual(a, b []URI) bool { if a == nil && b == nil { return true diff --git a/server.go b/server.go index 4ecffe974..4e4a507fd 100644 --- a/server.go +++ b/server.go @@ -60,7 +60,7 @@ type Server struct { // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". Network string - URI *URI + URI URI Cluster *Cluster // Background monitoring intervals. @@ -135,19 +135,21 @@ func (s *Server) Open() error { // Set Cluster URI. s.Cluster.URI = s.URI - // Create local node if no cluster is specified. - if len(s.Cluster.Nodes) == 0 { - s.Cluster.Nodes = []*Node{ - {Scheme: s.URI.Scheme(), Host: s.URI.HostPort()}, + /* + // Create local node if no cluster is specified. + if len(s.Cluster.Nodes) == 0 { + s.Cluster.Nodes = []*Node{ + {Scheme: s.URI.Scheme(), Host: s.URI.HostPort()}, + } } - } - // TODO: nodes aren't here yet. May need to merge new stats code anyway. - for i, n := range s.Cluster.Nodes { - if s.Cluster.NodeByHost(n.Host) != nil { - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) + // TODO: nodes aren't here yet. May need to merge new stats code anyway. + for i, n := range s.Cluster.Nodes { + if s.Cluster.NodeByHost(n.Host) != nil { + s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) + } } - } + */ // Open holder. s.Holder.LogOutput = s.LogOutput @@ -171,8 +173,7 @@ func (s *Server) Open() error { // Create executor for executing queries. e := NewExecutor(&ClientOptions{TLS: s.TLS}) e.Holder = s.Holder - e.Scheme = s.URI.Scheme() - e.Host = s.URI.HostPort() + e.URI = s.URI e.Cluster = s.Cluster e.MaxWritesPerRequest = s.MaxWritesPerRequest @@ -288,8 +289,8 @@ func (s *Server) monitorMaxSlices() { oldmaxslices := s.Holder.MaxSlices() for _, node := range s.Cluster.Nodes { - if s.URI.HostPort() != node.Host { - maxSlices, _ := s.checkMaxSlices(node.Scheme, node.Host) + if s.URI != node.URI { + maxSlices, _ := s.checkMaxSlices(node.URI) for index, newmax := range maxSlices { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation @@ -404,16 +405,16 @@ func (s *Server) LocalStatus() (proto.Message, error) { } ns := internal.NodeStatus{ - Host: s.URI.HostPort(), - State: s.State(), - Indexes: EncodeIndexes(s.Holder.Indexes()), - HostList: s.Cluster.HostList(), + URI: encodeURI(s.URI), + State: s.State(), + Indexes: EncodeIndexes(s.Holder.Indexes()), + URISet: encodeURIs(s.Cluster.URISet()), } // TODO: get rid of this // Append Slice list per this Node's indexes for _, index := range ns.Indexes { - index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI.HostPort()) + index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI) } return &ns, nil @@ -440,15 +441,16 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // Ignore status updates from self. - if s.URI.HostPort() == ns.Host { + if s.URI == decodeURI(ns.URI) { return nil } - fmt.Printf("mergeRemoteStatus on (%s) from (%s)\n", s.URI.HostPort(), ns.Host) + fmt.Printf("mergeRemoteStatus on (%s) from (%s)\n", s.URI, ns.URI) // Update Node.state. // Node can be nil if a merge occurs (via gossip) before the coordinator has // a chance to broadcast the existence of the node. - if node := s.Cluster.NodeByHost(ns.Host); node != nil { + uri := decodeURI(ns.URI) + if node := s.Cluster.NodeByURI(uri); node != nil { node.SetStatus(ns) } @@ -475,11 +477,11 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return nil } -func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) { +func (s *Server) checkMaxSlices(uri URI) (map[string]uint64, error) { // Create HTTP request. req, err := http.NewRequest("GET", (&url.URL{ - Scheme: scheme, - Host: hostPort, + Scheme: uri.Scheme(), + Host: uri.HostPort(), Path: "/slices/max", }).String(), nil) diff --git a/server/server.go b/server/server.go index 7fafce371..10220c1d4 100644 --- a/server/server.go +++ b/server/server.go @@ -102,7 +102,7 @@ func (m *Command) Run(args ...string) (err error) { return fmt.Errorf("server.Open: %v", err) } - m.Server.Logger().Printf("Listening as %s\n", m.Server.URI.Normalize()) + m.Server.Logger().Printf("Listening as %s\n", m.Server.URI) return nil } @@ -114,10 +114,11 @@ func (m *Command) SetupServer() error { } uri, err := pilosa.AddressWithDefaults(m.Config.Bind) + if err != nil { return err } - m.Server.URI = uri + m.Server.URI = *uri cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN @@ -181,11 +182,11 @@ func (m *Command) SetupServer() error { } // Set the coordinator node. - uri, err = pilosa.AddressWithDefaults(m.Config.Cluster.Coordinator) + curi, err := pilosa.AddressWithDefaults(m.Config.Cluster.Coordinator) if err != nil { return err } - m.Server.Cluster.Coordinator = uri.HostPort() + m.Server.Cluster.Coordinator = *curi // Set internal port (string). gossipPortStr := pilosa.DefaultGossipPort @@ -220,8 +221,8 @@ func (m *Command) SetupServer() error { // get the host portion of addr to use for binding gossipHost := uri.Host() - gossipNodeSet := gossip.NewGossipNodeSet(uri.HostPort(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + gossipNodeSet := gossip.NewGossipNodeSet(uri.String(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet diff --git a/server/server_test.go b/server/server_test.go index 0eb19be21..041026759 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -288,8 +288,8 @@ func TestMain_FrameRestore(t *testing.T) { // Update cluster config. m0.Server.Cluster.Nodes = []*pilosa.Node{ - {Scheme: "http", Host: m0.Server.URI.HostPort()}, - {Scheme: "http", Host: m1.Server.URI.HostPort()}, + {URI: m0.Server.URI}, + {URI: m1.Server.URI}, } m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes @@ -427,8 +427,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Update cluster config m0.Server.Cluster.Nodes = []*pilosa.Node{ - {Host: m0.Server.URI.HostPort()}, - {Host: m1.Server.URI.HostPort()}, + {URI: m0.Server.URI}, + {URI: m1.Server.URI}, } m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes @@ -445,9 +445,9 @@ func TestMain_SendReceiveMessage(t *testing.T) { } gossipSeed := gossipHost + ":" + freePorts[0] - topology := &pilosa.Topology{HostList: []string{m0.Server.URI.HostPort(), m1.Server.URI.HostPort()}} + topology := &pilosa.Topology{URISet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} - m0.Server.Cluster.Coordinator = m0.Server.URI.HostPort() + m0.Server.Cluster.Coordinator = m0.Server.URI m0.Server.Cluster.Topology = topology m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() @@ -478,7 +478,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal(err) } - m1.Server.Cluster.Coordinator = m0.Server.URI.HostPort() + m1.Server.Cluster.Coordinator = m0.Server.URI m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() gossipNodeSet1 := gossip.NewGossipNodeSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) diff --git a/test/cluster.go b/test/cluster.go index 60d44384f..604aff14c 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -22,8 +22,7 @@ func NewCluster(n int) *pilosa.Cluster { for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ - Scheme: "http", - Host: fmt.Sprintf("host%d", i), + URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) } @@ -47,3 +46,19 @@ type ConstHasher struct { func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } + +// NewURI is a test URI creator that intentionally swallows errors. +func NewURI(scheme, host string, port uint16) pilosa.URI { + uri := pilosa.DefaultURI() + uri.SetScheme(scheme) + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +func NewURIFromHostPort(host string, port uint16) pilosa.URI { + uri := pilosa.DefaultURI() + uri.SetHost(host) + uri.SetPort(port) + return *uri +} diff --git a/test/executor.go b/test/executor.go index 73445a1cd..237d25916 100644 --- a/test/executor.go +++ b/test/executor.go @@ -13,13 +13,12 @@ type Executor struct { } // NewExecutor returns a new instance of Executor. -// The executor always matches the hostname of the first cluster node. +// The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { e := &Executor{Executor: pilosa.NewExecutor(nil)} e.Holder = holder e.Cluster = cluster - e.Scheme = cluster.Nodes[0].Scheme - e.Host = cluster.Nodes[0].Host + e.URI = cluster.Nodes[0].URI return e } diff --git a/test/handler.go b/test/handler.go index b8fce4c65..44ca6d6e0 100644 --- a/test/handler.go +++ b/test/handler.go @@ -64,13 +64,13 @@ func NewServer() *Server { if err != nil { panic(err) } - s.Handler.URI = uri + s.Handler.URI = *uri // Handler test messages can no-op. s.Handler.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Cluster.Nodes[0].URI = *uri return s } @@ -111,12 +111,12 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } // Host returns the hostname of the running server. func (s *Server) Host() string { return MustParseURLHost(s.URL) } -func (s *Server) HostURI() *pilosa.URI { +func (s *Server) HostURI() pilosa.URI { uri, err := pilosa.NewURIFromAddress(s.URL) if err != nil { panic(err) } - return uri + return *uri } // MustParseURLHost parses rawurl and returns the hostname. Panic on error. diff --git a/uri.go b/uri.go index de6e4bae6..24d902f70 100644 --- a/uri.go +++ b/uri.go @@ -20,6 +20,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/pilosa/pilosa/internal" ) var schemeRegexp = regexp.MustCompile("^[+a-z]+$") @@ -67,11 +69,7 @@ func NewURIFromHostPort(host string, port uint16) (*URI, error) { // NewURIFromAddress parses the passed address and returns a URI. func NewURIFromAddress(address string) (*URI, error) { - uri, err := parseAddress(address) - if err != nil { - return nil, err - } - return uri, err + return parseAddress(address) } // Scheme returns the scheme of this URI. @@ -134,14 +132,17 @@ func (u *URI) Normalize() string { return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port) } +// String returns the address as a string. +func (u URI) String() string { + return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port) +} + // Equals returns true if the checked URI is equivalent to this URI. func (u URI) Equals(other *URI) bool { if other == nil { return false } - return u.scheme == other.scheme && - u.host == other.host && - u.port == other.port + return u == *other } // The following methods are required to implement pflag Value interface. @@ -188,3 +189,49 @@ func parseAddress(address string) (uri *URI, err error) { } return uri, nil } + +// Encode converts o into its internal representation. +func (u URI) Encode() *internal.URI { + return encodeURI(u) +} + +func encodeURI(u URI) *internal.URI { + return &internal.URI{ + Scheme: u.scheme, + Host: u.host, + Port: uint32(u.port), + } +} + +func decodeURI(i *internal.URI) URI { + if i == nil { + return URI{} + } + return URI{ + scheme: i.Scheme, + host: i.Host, + port: uint16(i.Port), + } +} + +func encodeURIs(a []URI) []*internal.URI { + if len(a) == 0 { + return nil + } + other := make([]*internal.URI, len(a)) + for i := range a { + other[i] = encodeURI(a[i]) + } + return other +} + +func decodeURIs(a []*internal.URI) []URI { + if len(a) == 0 { + return nil + } + other := make([]URI, len(a)) + for i := range a { + other[i] = decodeURI(a[i]) + } + return other +} From deed9adfe498e2b6475c7a897cf2ed9bbd4c6e4f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 25 Oct 2017 09:05:43 -0500 Subject: [PATCH 003/311] Fix existing tests --- client.go | 2 +- cluster.go | 6 +- cluster_test.go | 145 ++++++++++++++++++++++-------------------------- handler_test.go | 7 +-- test/handler.go | 33 +++-------- uri.go | 30 ++++++++++ 6 files changed, 110 insertions(+), 113 deletions(-) diff --git a/client.go b/client.go index 5edd45468..a579e527f 100644 --- a/client.go +++ b/client.go @@ -1226,7 +1226,7 @@ func uriPathToURL(uri *URI, path string) url.URL { func nodePathToURL(node *Node, path string) url.URL { return url.URL{ Scheme: node.URI.Scheme(), - Host: node.URI.Host(), + Host: node.URI.HostPort(), Path: path, } } diff --git a/cluster.go b/cluster.go index c3b02ecd2..4f389fb4b 100644 --- a/cluster.go +++ b/cluster.go @@ -55,7 +55,7 @@ const ( type Node struct { //Scheme string `json:"scheme"` //Host string `json:"host"` // HostPort - URI URI // TODO: add json tags: `json:"uri"` + URI URI `json:"uri"` status *internal.NodeStatus `json:"status"` } @@ -220,7 +220,6 @@ func (c *Cluster) AddHost(uri URI) error { // add to cluster _, added := c.AddNode(uri) if !added { - fmt.Println("NOT added") return nil } @@ -229,12 +228,10 @@ func (c *Cluster) AddHost(uri URI) error { return fmt.Errorf("Cluster.Topology is nil") } if !c.Topology.AddURI(uri) { - fmt.Println("call top.AddHost()") return nil } // save topology - fmt.Println("calll c.saveTop()") return c.saveTopology() } @@ -974,7 +971,6 @@ func (c *Cluster) loadTopology() error { // saveTopology writes the current topology to disk. func (c *Cluster) saveTopology() error { - fmt.Println("saveTopology", filepath.Join(c.Path, ".topology")) if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { return err } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { diff --git a/cluster_test.go b/cluster_test.go index 0be636181..674be6919 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -22,6 +22,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" ) @@ -91,38 +92,6 @@ func TestHasher(t *testing.T) { } } -/* TODO travis: fix test -// Ensure cluster can compare its Nodes and Members -func TestCluster_NodeStates(t *testing.T) { - c := pilosa.Cluster{ - Nodes: []*pilosa.Node{ - {Host: "serverA:1000"}, - {Host: "serverB:1000"}, - {Host: "serverC:1000"}, - }, - NodeSet: &pilosa.StaticNodeSet{}, - } - - err := c.NodeSet.(*pilosa.StaticNodeSet).Join([]*pilosa.Node{ - &pilosa.Node{Host: "serverA:1000"}, - &pilosa.Node{Host: "serverC:1000"}, - &pilosa.Node{Host: "serverD:1000"}, - }) - if err != nil { - t.Fatalf("unexpected gossiper nodes: %s", err) - } - - // Verify a DOWN node is reported, and extraneous nodes are ignored - if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{ - "serverA:1000": pilosa.NodeStateUp, - "serverB:1000": pilosa.NodeStateDown, - "serverC:1000": pilosa.NodeStateUp, - }) { - t.Fatalf("unexpected node state: %s", spew.Sdump(a)) - } -} -*/ - // Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { c := test.NewCluster(5) @@ -133,35 +102,37 @@ func TestCluster_OwnsSlices(t *testing.T) { } } -// TODO travis: fix these tests -/* func TestCluster_Nodes(t *testing.T) { + uri0 := test.NewURIFromHostPort("node0", 0) + uri1 := test.NewURIFromHostPort("node1", 0) + uri2 := test.NewURIFromHostPort("node2", 0) + uri3 := test.NewURIFromHostPort("node3", 0) nodes := []*pilosa.Node{ - {URI: test.NewURIFromHostPort("node0", 0)}, - {URI: test.NewURIFromHostPort("node1", 0)}, - {URI: test.NewURIFromHostPort("node2", 0)}, + {URI: uri0}, + {URI: uri1}, + {URI: uri2}, } t.Run("URISet", func(t *testing.T) { actual := pilosa.Nodes(nodes).URIs() - expected := []string{"node0", "node1", "node2"} + expected := []pilosa.URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("Filter", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).Hosts() - expected := []string{"node0", "node2"} + actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs() + expected := []pilosa.URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) - t.Run("FilterHost", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterHost("node1")).Hosts() - expected := []string{"node0", "node2"} + t.Run("FilterURI", func(t *testing.T) { + actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs() + expected := []pilosa.URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -178,9 +149,9 @@ func TestCluster_Nodes(t *testing.T) { } }) - t.Run("ContainsHost", func(t *testing.T) { - actualTrue := pilosa.Nodes(nodes).ContainsHost("node1") - actualFalse := pilosa.Nodes(nodes).ContainsHost("nodeX") + t.Run("ContainsURI", func(t *testing.T) { + actualTrue := pilosa.Nodes(nodes).ContainsURI(uri1) + actualFalse := pilosa.Nodes(nodes).ContainsURI(uri3) if !reflect.DeepEqual(actualTrue, true) { t.Errorf("expected: %v, but got: %v", true, actualTrue) } @@ -191,8 +162,8 @@ func TestCluster_Nodes(t *testing.T) { t.Run("Clone", func(t *testing.T) { clone := pilosa.Nodes(nodes).Clone() - actual := pilosa.Nodes(clone).Hosts() - expected := []string{"node0", "node1", "node2"} + actual := pilosa.Nodes(clone).URIs() + expected := []pilosa.URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -200,19 +171,21 @@ func TestCluster_Nodes(t *testing.T) { } func TestCluster_Coordinator(t *testing.T) { + uri1 := test.NewURIFromHostPort("node1", 0) + uri2 := test.NewURIFromHostPort("node2", 0) c1 := *pilosa.NewCluster() - c1.Host = "host0:port0" - c1.Coordinator = "host0:port0" + c1.URI = uri1 + c1.Coordinator = uri1 c2 := *pilosa.NewCluster() - c2.Host = "host1:port1" - c2.Coordinator = "host0:port0" + c2.URI = uri2 + c2.Coordinator = uri1 t.Run("IsCoordinator", func(t *testing.T) { if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.Host) + t.Errorf("!IsCoordinator error: %v", c1.URI) } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.Host) + t.Errorf("IsCoordinator error: %v", c2.URI) } }) } @@ -220,34 +193,39 @@ func TestCluster_Coordinator(t *testing.T) { func TestCluster_Topology(t *testing.T) { c1 := test.NewCluster(1) + uri1 := test.NewURIFromHostPort("node1", 0) + uri2 := test.NewURIFromHostPort("node2", 0) + base := test.NewURIFromHostPort("host0", 0) + invalid := test.NewURIFromHostPort("invalid", 0) + t.Run("AddHost", func(t *testing.T) { - err := c1.AddHost("abc") + err := c1.AddHost(uri1) if err != nil { t.Fatal(err) } // add the same host. - err = c1.AddHost("abc") + err = c1.AddHost(uri1) if err != nil { t.Fatal(err) } - err = c1.AddHost("xyz") + err = c1.AddHost(uri2) if err != nil { t.Fatal(err) } - actual := pilosa.Nodes(c1.Nodes).Hosts() - expected := []string{"abc", "host0", "xyz"} + actual := pilosa.Nodes(c1.Nodes).URIs() + expected := []pilosa.URI{base, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) - t.Run("ContainsHost", func(t *testing.T) { - if !c1.Topology.ContainsHost("abc") { - t.Errorf("!ContainsHost error: %v", "abc") - } else if c1.Topology.ContainsHost("invalidHost") { - t.Errorf("ContainsHost error: %v", "invalidHost") + t.Run("ContainsURI", func(t *testing.T) { + if !c1.Topology.ContainsURI(uri1) { + t.Errorf("!ContainsHost error: %v", uri1) + } else if c1.Topology.ContainsURI(invalid) { + t.Errorf("ContainsHost error: %v", invalid) } }) } @@ -290,23 +268,33 @@ func TestCluster_Resize(t *testing.T) { c2 := test.NewCluster(4) c2.ReplicaN = 2 - expected := map[string][]*internal.ResizeSource{ - "host0": []*internal.ResizeSource{ - {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 5}, + u0 := test.NewURIFromHostPort("host0", 0) + u1 := test.NewURIFromHostPort("host1", 0) + u2 := test.NewURIFromHostPort("host2", 0) + u3 := test.NewURIFromHostPort("host3", 0) + + uri0 := u0.Encode() + uri1 := u1.Encode() + uri2 := u2.Encode() + //uri3 := u3.Encode() + + expected := map[pilosa.URI][]*internal.ResizeSource{ + u0: []*internal.ResizeSource{ + {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 5}, }, - "host1": []*internal.ResizeSource{ - {Host: "host2", Index: "i", Frame: "f", View: "v", Slice: 0}, - {Host: "host2", Index: "i", Frame: "f", View: "inverse", Slice: 0}, + u1: []*internal.ResizeSource{ + {URI: uri2, Index: "i", Frame: "f", View: "v", Slice: 0}, + {URI: uri2, Index: "i", Frame: "f", View: "inverse", Slice: 0}, }, - "host2": []*internal.ResizeSource{ - {Host: "host0", Index: "i", Frame: "f", View: "inverse", Slice: 3}, + u2: []*internal.ResizeSource{ + {URI: uri0, Index: "i", Frame: "f", View: "inverse", Slice: 3}, }, - "host3": []*internal.ResizeSource{ - {Host: "host0", Index: "i", Frame: "f", View: "v", Slice: 1}, - {Host: "host1", Index: "i", Frame: "f", View: "v", Slice: 2}, - {Host: "host0", Index: "i", Frame: "f", View: "inverse", Slice: 1}, - {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 2}, - {Host: "host1", Index: "i", Frame: "f", View: "inverse", Slice: 5}, + u3: []*internal.ResizeSource{ + {URI: uri0, Index: "i", Frame: "f", View: "v", Slice: 1}, + {URI: uri1, Index: "i", Frame: "f", View: "v", Slice: 2}, + {URI: uri0, Index: "i", Frame: "f", View: "inverse", Slice: 1}, + {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 2}, + {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 5}, }, } @@ -316,4 +304,3 @@ func TestCluster_Resize(t *testing.T) { } }) } -*/ diff --git a/handler_test.go b/handler_test.go index 42048a224..0a540d059 100644 --- a/handler_test.go +++ b/handler_test.go @@ -74,7 +74,6 @@ func TestHandler_NotFound(t *testing.T) { } } -/* TODO travis: fix test // Ensure the handler can return the schema. func TestHandler_Schema(t *testing.T) { hldr := test.MustOpenHolder() @@ -107,6 +106,7 @@ func TestHandler_Schema(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -147,11 +147,10 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" { + } else if body := w.Body.String(); body != `{"status":{"State":"NORMAL","URISet":[{"Scheme":"http","Host":"localhost","Port":10101}]}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } -*/ // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { @@ -1185,7 +1184,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" { + } else if w.Body.String() != `[{"uri":{"scheme":"http","host":"host2"}},{"uri":{"scheme":"http","host":"host0"}}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } diff --git a/test/handler.go b/test/handler.go index 44ca6d6e0..6d08bc6c4 100644 --- a/test/handler.go +++ b/test/handler.go @@ -11,6 +11,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -75,35 +76,19 @@ func NewServer() *Server { return s } -/* TODO travis: fix this test -// LocalStatus returns the state of the local node as well as the -// holder (indexes/frames) according to the local node. +// LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - if s.Handler.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Host: s.Handler.Handler.URI.HostPort(), - State: pilosa.NodeStateUp, - Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.URI.HostPort()) - } - - return &ns, nil + return nil, nil } -// ClusterStatus returns the NodeState for all nodes in the cluster. +// ClusterStatus exists so that test.Server implements StatusHandler. func (s *Server) ClusterStatus() (proto.Message, error) { - // Assuming we are only testing this with one Node - // So just return its status - return s.LocalStatus() + uri := pilosa.DefaultURI() + return &internal.ClusterStatus{ + State: pilosa.NodeStateNormal, + URISet: []*internal.URI{uri.Encode()}, + }, nil } -*/ // HandleRemoteStatus just need to implement a nop to complete the Interface func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } diff --git a/uri.go b/uri.go index 24d902f70..23aba4dfc 100644 --- a/uri.go +++ b/uri.go @@ -15,6 +15,7 @@ package pilosa import ( + "encoding/json" "errors" "fmt" "regexp" @@ -235,3 +236,32 @@ func decodeURIs(a []*internal.URI) []URI { } return other } + +// MarshalJSON marshals URI into a JSON-encoded byte slice. +func (u *URI) MarshalJSON() ([]byte, error) { + var output struct { + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + } + output.Scheme = u.scheme + output.Host = u.host + output.Port = u.port + + return json.Marshal(output) +} + +func (u *URI) UnmarshalJSON(b []byte) error { + var input struct { + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + } + if err := json.Unmarshal(b, &input); err != nil { + return err + } + u.scheme = input.Scheme + u.host = input.Host + u.port = input.Port + return nil +} From cd8542ca300e64e3389ca40135802d2f932a05cf Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 26 Oct 2017 11:11:05 -0500 Subject: [PATCH 004/311] Remove FrameSchema. Move Fields to the Frame struct. --- frame.go | 232 ++++++++++++-------------------- handler.go | 4 +- index.go | 14 +- index_test.go | 20 ++- internal/private.pb.go | 294 +++++++++++------------------------------ internal/private.proto | 4 - 6 files changed, 174 insertions(+), 394 deletions(-) diff --git a/frame.go b/frame.go index c8ad8cf32..06d081423 100644 --- a/frame.go +++ b/frame.go @@ -43,12 +43,10 @@ const ( // Frame represents a container for views. type Frame struct { - mu sync.RWMutex - path string - index string - name string - timeQuantum TimeQuantum - schema *FrameSchema + mu sync.RWMutex + path string + index string + name string views map[string]*View @@ -58,14 +56,14 @@ type Frame struct { broadcaster Broadcaster Stats StatsClient - // Frame settings. + // Frame options. rowLabel string - cacheType string inverseEnabled bool + cacheType string + cacheSize uint32 + timeQuantum TimeQuantum rangeEnabled bool - - // Cache size for ranked frames - cacheSize uint32 + fields []*Field LogOutput io.Writer } @@ -78,10 +76,9 @@ func NewFrame(path, index, name string) (*Frame, error) { } return &Frame{ - path: path, - index: index, - name: name, - schema: &FrameSchema{}, + path: path, + index: index, + name: name, views: make(map[string]*View), rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), @@ -91,9 +88,11 @@ func NewFrame(path, index, name string) (*Frame, error) { rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, - rangeEnabled: DefaultRangeEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, + //timeQuantum + rangeEnabled: DefaultRangeEnabled, + //fields LogOutput: ioutil.Discard, }, nil @@ -230,7 +229,7 @@ func (f *Frame) options() FrameOptions { CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, - Fields: f.schema.Fields, + Fields: f.fields, } } @@ -244,8 +243,6 @@ func (f *Frame) Open() error { if err := f.loadMeta(); err != nil { return err - } else if err := f.loadSchema(); err != nil { - return err } if err := f.openViews(); err != nil { @@ -304,12 +301,13 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.timeQuantum = "" f.rowLabel = DefaultRowLabel - f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled - f.rangeEnabled = DefaultRangeEnabled + f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize + f.timeQuantum = "" + f.rangeEnabled = DefaultRangeEnabled + //f.fields return nil } else if err != nil { return err @@ -320,17 +318,16 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled - f.rangeEnabled = pb.RangeEnabled - f.cacheSize = pb.CacheSize - - // Copy cache type. f.cacheType = pb.CacheType if f.cacheType == "" { f.cacheType = DefaultCacheType } + f.cacheSize = pb.CacheSize + f.timeQuantum = TimeQuantum(pb.TimeQuantum) + f.rangeEnabled = pb.RangeEnabled + f.fields = decodeFields(pb.Fields) return nil } @@ -352,35 +349,6 @@ func (f *Frame) saveMeta() error { return nil } -// loadSchema reads the schema for the frame. -func (f *Frame) loadSchema() error { - buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema")) - if os.IsNotExist(err) { - f.schema = &FrameSchema{} - return nil - } else if err != nil { - return err - } - - var pb internal.FrameSchema - if err := proto.Unmarshal(buf, &pb); err != nil { - return err - } - f.schema = decodeFrameSchema(&pb) - - return nil -} - -// saveSchema writes the current schema to disk. -func (f *Frame) saveSchema() error { - if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil { - return err - } else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil { - return err - } - return nil -} - // Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() @@ -402,16 +370,11 @@ func (f *Frame) Close() error { return nil } -// Schema returns the frame's current schema. -func (f *Frame) Schema() *FrameSchema { +// Field returns a field by name. +func (f *Frame) Field(name string) *Field { f.mu.RLock() defer f.mu.RUnlock() - return f.schema -} - -// Field returns a field from the schema by name. -func (f *Frame) Field(name string) *Field { - for _, field := range f.Schema().Fields { + for _, field := range f.fields { if field.Name == name { return field } @@ -419,7 +382,24 @@ func (f *Frame) Field(name string) *Field { return nil } -// CreateField creates a new field on the schema. +// Fields returns the fields on the frame. +func (f *Frame) Fields() []*Field { + f.mu.RLock() + defer f.mu.RUnlock() + return f.fields +} + +// HasField returns true if a field exists on the frame. +func (f *Frame) HasField(name string) bool { + for _, fld := range f.fields { + if fld.Name == name { + return true + } + } + return false +} + +// CreateField creates a new field on the frame. func (f *Frame) CreateField(field *Field) error { f.mu.Lock() defer f.mu.Unlock() @@ -429,18 +409,35 @@ func (f *Frame) CreateField(field *Field) error { return ErrFrameFieldsNotAllowed } - // Copy schema and append field. - schema := f.schema.Clone() - if err := schema.AddField(field); err != nil { + // Append field. + if err := f.addField(field); err != nil { return err } - f.schema = schema - f.saveSchema() + f.saveMeta() + return nil +} + +// addField adds a single field to fields. +func (f *Frame) addField(field *Field) error { + if err := ValidateField(field); err != nil { + return err + } else if f.HasField(field.Name) { + return ErrFieldExists + } + + // Add field to list. + f.fields = append(f.fields, field) + + // Sort fields by name. + sort.Slice(f.fields, func(i, j int) bool { + return f.fields[i].Name < f.fields[j].Name + }) + return nil } // GetFields returns a list of all the fields in the frame. -func (f *Frame) GetFields() (*FrameSchema, error) { +func (f *Frame) GetFields() ([]*Field, error) { f.mu.RLock() defer f.mu.RUnlock() @@ -449,12 +446,12 @@ func (f *Frame) GetFields() (*FrameSchema, error) { return nil, ErrFrameFieldsNotAllowed } - err := f.loadSchema() + err := f.loadMeta() if err != nil { return nil, err } - return f.schema, nil + return f.fields, nil } // DeleteField deletes an existing field on the schema. @@ -467,12 +464,10 @@ func (f *Frame) DeleteField(name string) error { return ErrFrameFieldsNotAllowed } - // Copy schema and remove field. - schema := f.schema.Clone() - if err := schema.DeleteField(name); err != nil { + // Remove field. + if err := f.deleteField(name); err != nil { return err } - f.schema = schema // Remove views. viewName := ViewFieldPrefix + name @@ -489,6 +484,18 @@ func (f *Frame) DeleteField(name string) error { return nil } +// deleteField removes a single field from fields. +func (f *Frame) deleteField(name string) error { + for i, field := range f.fields { + if field.Name == name { + copy(f.fields[i:], f.fields[i+1:]) + f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil + return nil + } + } + return ErrFieldNotFound +} + // TimeQuantum returns the time quantum for the frame. func (f *Frame) TimeQuantum() TimeQuantum { f.mu.Lock() @@ -1021,77 +1028,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { } } -// FrameSchema represents the list of fields on a frame. -type FrameSchema struct { - Fields []*Field -} - -// Clone returns a copy of s. -func (s *FrameSchema) Clone() *FrameSchema { - other := &FrameSchema{Fields: make([]*Field, len(s.Fields))} - copy(other.Fields, s.Fields) - return other -} - -// HasField returns true if a field exists on the schema. -func (s *FrameSchema) HasField(name string) bool { - for _, f := range s.Fields { - if f.Name == name { - return true - } - } - return false -} - -// AddField adds a single field to the schema. -func (s *FrameSchema) AddField(field *Field) error { - if err := ValidateField(field); err != nil { - return err - } else if s.HasField(field.Name) { - return ErrFieldExists - } - - // Add field to list. - s.Fields = append(s.Fields, field) - - // Sort fields by name. - sort.Slice(s.Fields, func(i, j int) bool { - return s.Fields[i].Name < s.Fields[j].Name - }) - - return nil -} - -// DeleteField removes a single field from the schema. -func (s *FrameSchema) DeleteField(name string) error { - for i, field := range s.Fields { - if field.Name == name { - copy(s.Fields[i:], s.Fields[i+1:]) - s.Fields, s.Fields[len(s.Fields)-1] = s.Fields[:len(s.Fields)-1], nil - return nil - } - } - return ErrFieldNotFound -} - -func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema { - if schema == nil { - return nil - } - return &internal.FrameSchema{ - Fields: encodeFields(schema.Fields), - } -} - -func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema { - if schema == nil { - return nil - } - return &FrameSchema{ - Fields: decodeFields(schema.Fields), - } -} - // List of field data types. const ( FieldTypeInt = "int" diff --git a/handler.go b/handler.go index 0251461ae..ef5655f43 100644 --- a/handler.go +++ b/handler.go @@ -861,7 +861,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { return } - schema, err := frame.GetFields() + fields, err := frame.GetFields() if err == ErrFrameFieldsNotAllowed { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -871,7 +871,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { } // Encode response. - if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: schema.Fields}); err != nil { + if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { h.logger().Printf("response encoding error: %s", err) } } diff --git a/index.go b/index.go index 3e6cb2518..00e08002f 100644 --- a/index.go +++ b/index.go @@ -494,18 +494,12 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { f.inverseEnabled = opt.InverseEnabled f.rangeEnabled = opt.RangeEnabled - if err := f.saveMeta(); err != nil { - f.Close() - return nil, err - } - f.rangeEnabled = opt.RangeEnabled - // Set schema & save. - f.schema = &FrameSchema{ - Fields: opt.Fields, - } - if err := f.saveSchema(); err != nil { + // Set fields. + f.fields = opt.Fields + + if err := f.saveMeta(); err != nil { f.Close() return nil, err } diff --git a/index_test.go b/index_test.go index a7cff68f7..077b2ded6 100644 --- a/index_test.go +++ b/index_test.go @@ -106,25 +106,21 @@ func TestIndex_CreateFrame(t *testing.T) { }, }); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, + } else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, }) { - t.Fatalf("unexpected schema: %#v", f.Schema()) + t.Fatalf("unexpected fields: %#v", f.Fields()) } // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, }) { - t.Fatalf("unexpected schema after reopen: %#v", f.Schema()) + t.Fatalf("unexpected fields after reopen: %#v", f.Fields()) } }) diff --git a/internal/private.pb.go b/internal/private.pb.go index 707d89cf5..bab2ad1ba 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -30,7 +30,6 @@ URI NodeStatus ClusterStatus - FrameSchema Field DeleteViewMessage ResizeInstruction @@ -725,22 +724,6 @@ func (m *ClusterStatus) GetURISet() []*URI { return nil } -type FrameSchema struct { - Fields []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"` -} - -func (m *FrameSchema) Reset() { *m = FrameSchema{} } -func (m *FrameSchema) String() string { return proto.CompactTextString(m) } -func (*FrameSchema) ProtoMessage() {} -func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } - -func (m *FrameSchema) GetFields() []*Field { - if m != nil { - return m.Fields - } - return nil -} - type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -751,7 +734,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *Field) GetName() string { if m != nil { @@ -790,7 +773,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -823,7 +806,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -864,7 +847,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *ResizeSource) GetURI() *URI { if m != nil { @@ -910,7 +893,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{27} + return fileDescriptorPrivate, []int{26} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -934,7 +917,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *Topology) GetURISet() []*URI { if m != nil { @@ -966,7 +949,6 @@ func init() { proto.RegisterType((*URI)(nil), "internal.URI") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") - proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") proto.RegisterType((*Field)(nil), "internal.Field") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") @@ -1887,36 +1869,6 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *FrameSchema) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *FrameSchema) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } - return i, nil -} - func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2574,18 +2526,6 @@ func (m *ClusterStatus) Size() (n int) { return n } -func (m *FrameSchema) Size() (n int) { - var l int - _ = l - if len(m.Fields) > 0 { - for _, e := range m.Fields { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - } - return n -} - func (m *Field) Size() (n int) { var l int _ = l @@ -5803,87 +5743,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } -func (m *FrameSchema) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: FrameSchema: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: FrameSchema: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Fields = append(m.Fields, &Field{}) - if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *Field) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -6813,75 +6672,74 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1110 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0x23, 0x45, - 0x13, 0x7e, 0xc7, 0x63, 0x3b, 0x76, 0x79, 0x9d, 0x38, 0xfd, 0x2e, 0x91, 0x13, 0x45, 0xde, 0xa8, - 0x25, 0xd8, 0x10, 0x89, 0x00, 0x41, 0x5a, 0xf1, 0x75, 0x80, 0x8d, 0xb3, 0xca, 0xc0, 0x66, 0x59, - 0xda, 0xc9, 0x72, 0x43, 0xea, 0xd8, 0x4d, 0x76, 0x94, 0xf1, 0xb4, 0x99, 0x69, 0x27, 0xf1, 0x1e, - 0xb8, 0xc1, 0x3f, 0x40, 0x42, 0xe2, 0xc8, 0x99, 0xff, 0xc1, 0x91, 0x9f, 0x80, 0xc2, 0x85, 0x7f, - 0x80, 0xc4, 0x09, 0x75, 0x75, 0xcf, 0x87, 0x3f, 0x43, 0x72, 0x9b, 0x7a, 0xba, 0xba, 0xea, 0xe9, - 0xa7, 0xab, 0xca, 0x6d, 0xa8, 0x0f, 0x22, 0xff, 0x82, 0x2b, 0xb1, 0x3b, 0x88, 0xa4, 0x92, 0xa4, - 0xe2, 0x87, 0x4a, 0x44, 0x21, 0x0f, 0xe8, 0x17, 0x50, 0xf5, 0xc2, 0x9e, 0xb8, 0x3a, 0x12, 0x8a, - 0x93, 0x2d, 0xa8, 0xed, 0xcb, 0x60, 0xd8, 0x0f, 0x9f, 0xf2, 0x53, 0x11, 0x34, 0x9d, 0x2d, 0x67, - 0xbb, 0xca, 0xf2, 0x90, 0xf6, 0x38, 0xf6, 0xfb, 0xe2, 0xcb, 0x21, 0x0f, 0xd5, 0xb0, 0xdf, 0x2c, - 0x18, 0x8f, 0x1c, 0x44, 0xff, 0x71, 0xa0, 0xfa, 0x24, 0xe2, 0x7d, 0x81, 0x11, 0x37, 0xa0, 0xc2, - 0xe4, 0x65, 0x3e, 0x5c, 0x6a, 0x93, 0x37, 0x60, 0xd9, 0x0b, 0x2f, 0x44, 0x14, 0x8b, 0x83, 0x90, - 0x9f, 0x06, 0xa2, 0x87, 0xe1, 0x2a, 0x6c, 0x02, 0x25, 0x9b, 0x50, 0xdd, 0xe7, 0xdd, 0x97, 0xe2, - 0x78, 0x34, 0x10, 0x4d, 0x17, 0x83, 0x64, 0x40, 0xba, 0xda, 0xf1, 0x5f, 0x89, 0x66, 0x71, 0xcb, - 0xd9, 0xae, 0xb3, 0x0c, 0x98, 0xe4, 0x5b, 0x9a, 0xe2, 0x4b, 0x28, 0xdc, 0x63, 0x3c, 0x3c, 0x4b, - 0x39, 0x94, 0x91, 0xc3, 0x18, 0x46, 0x1e, 0x42, 0xf9, 0x89, 0x2f, 0x82, 0x5e, 0xdc, 0x5c, 0xda, - 0x72, 0xb7, 0x6b, 0x7b, 0x2b, 0xbb, 0x89, 0x7e, 0xbb, 0x88, 0x33, 0xbb, 0x4c, 0x29, 0x2c, 0x7b, - 0xfd, 0x81, 0x8c, 0x14, 0x13, 0xf1, 0x40, 0x86, 0xb1, 0x20, 0x0d, 0x70, 0x0f, 0xa2, 0xc8, 0x9e, - 0x5d, 0x7f, 0xd2, 0xef, 0xa0, 0xf1, 0x38, 0x90, 0xdd, 0xf3, 0x36, 0x57, 0x9c, 0x89, 0x6f, 0x87, - 0x22, 0x56, 0xe4, 0x3e, 0x94, 0xf0, 0x16, 0xac, 0x9f, 0x31, 0x34, 0x8a, 0x4a, 0x5a, 0x99, 0x8d, - 0xa1, 0x51, 0xdc, 0x8f, 0x52, 0x14, 0x99, 0x31, 0x34, 0xda, 0x09, 0xfc, 0xae, 0x91, 0xa0, 0xc8, - 0x8c, 0x41, 0x08, 0x14, 0x5f, 0xf8, 0xe2, 0xd2, 0x9e, 0x1b, 0xbf, 0xa9, 0x07, 0xab, 0xb9, 0xfc, - 0x96, 0xe6, 0x1a, 0x94, 0x99, 0xbc, 0xf4, 0xda, 0x71, 0xd3, 0xd9, 0x72, 0xb7, 0x8b, 0xcc, 0x5a, - 0xa8, 0x2e, 0x5e, 0xbf, 0x5e, 0x2a, 0xe0, 0x52, 0x06, 0xd0, 0x75, 0x28, 0xa1, 0xd4, 0xfa, 0x94, - 0xd9, 0x5e, 0xfd, 0x49, 0x7f, 0x76, 0x60, 0xf5, 0x88, 0x5f, 0x21, 0x8d, 0x38, 0x4d, 0x73, 0x08, - 0xd5, 0x14, 0x44, 0xef, 0xda, 0xde, 0x4e, 0xa6, 0xe5, 0x94, 0x7f, 0x86, 0x1c, 0x84, 0x2a, 0x1a, - 0xb1, 0x6c, 0xf3, 0xc6, 0xc7, 0xb0, 0x3c, 0xbe, 0xa8, 0x39, 0x9c, 0x8b, 0x51, 0xa2, 0xf4, 0xb9, - 0x18, 0x69, 0x4d, 0x2e, 0x78, 0x30, 0x34, 0xfa, 0x15, 0x99, 0x31, 0x3e, 0x2c, 0xbc, 0xef, 0xd0, - 0xaf, 0x81, 0xec, 0x47, 0x82, 0x2b, 0x81, 0x01, 0x8e, 0x44, 0x1c, 0xf3, 0x33, 0x31, 0xff, 0x16, - 0x8c, 0xb2, 0x85, 0xbc, 0xb2, 0x9b, 0x50, 0xf5, 0x62, 0x5b, 0xa8, 0x78, 0x13, 0x15, 0x96, 0x01, - 0x74, 0x07, 0x48, 0x5b, 0x04, 0x42, 0x09, 0xdb, 0x5b, 0x0b, 0xe2, 0xd3, 0x4e, 0xc2, 0xe5, 0x66, - 0x5f, 0xf2, 0x10, 0x8a, 0xba, 0xad, 0x90, 0x4a, 0x6d, 0xef, 0xff, 0x99, 0x74, 0x69, 0x0f, 0x33, - 0x74, 0xa0, 0x7e, 0x12, 0xd4, 0xb6, 0xe2, 0x0d, 0x07, 0x9c, 0x51, 0x66, 0x49, 0x2a, 0x77, 0x32, - 0x55, 0xda, 0xdc, 0x36, 0xd5, 0x27, 0xc9, 0x59, 0xef, 0x9a, 0x8a, 0xb6, 0x2d, 0xaa, 0xcb, 0xf5, - 0x99, 0x5e, 0x35, 0x7b, 0xf0, 0x7b, 0xfe, 0x91, 0x27, 0x79, 0xfc, 0xe5, 0xd8, 0x94, 0xb7, 0x0b, - 0x33, 0xa1, 0x9c, 0x9e, 0x58, 0x49, 0x61, 0xd9, 0x0e, 0x4b, 0x6d, 0x9c, 0x03, 0x3a, 0x6b, 0xdc, - 0x2c, 0x4e, 0xcd, 0x01, 0x8d, 0x33, 0xbb, 0xac, 0xdb, 0xc9, 0x16, 0x79, 0xc9, 0xb4, 0x93, 0xb1, - 0xc8, 0x01, 0x34, 0xbc, 0x70, 0x30, 0x54, 0x6d, 0xf1, 0x8d, 0x1f, 0xfa, 0xca, 0x97, 0x61, 0xdc, - 0x2c, 0x63, 0xa8, 0xf5, 0x3c, 0xa3, 0x31, 0x0f, 0x36, 0xb5, 0x85, 0xfe, 0xe0, 0xc0, 0xca, 0x04, - 0x38, 0xe7, 0xd0, 0x09, 0xdf, 0xc2, 0x62, 0xbe, 0x8f, 0xd2, 0x01, 0xe7, 0xa2, 0x63, 0x6b, 0x2e, - 0x9b, 0xf1, 0x79, 0xf7, 0x8b, 0x03, 0xf7, 0x67, 0x39, 0xcc, 0x64, 0xd3, 0x02, 0x78, 0x1e, 0xf9, - 0x7d, 0x1e, 0x8d, 0x3e, 0x17, 0x23, 0x3b, 0xeb, 0x73, 0x08, 0xf9, 0x0a, 0xd6, 0x26, 0x62, 0x7d, - 0xda, 0x35, 0x12, 0x19, 0x52, 0x0f, 0xe6, 0x92, 0x32, 0x7e, 0x6c, 0xce, 0x76, 0xfa, 0xb7, 0x03, - 0xaf, 0xcd, 0x5c, 0xca, 0xea, 0xd1, 0xc9, 0x97, 0xfe, 0x0e, 0x34, 0x5e, 0xe8, 0x51, 0xd1, 0x16, - 0xb1, 0xf2, 0x43, 0xae, 0x3d, 0x6d, 0xc1, 0x4e, 0xe1, 0xc4, 0x83, 0x0a, 0x62, 0x47, 0x7c, 0x60, - 0x69, 0xbe, 0x75, 0x03, 0xcd, 0xdd, 0xc4, 0xdf, 0xcc, 0xb4, 0x74, 0xbb, 0x26, 0x83, 0x53, 0x37, - 0x19, 0xe1, 0x68, 0x6c, 0x7c, 0x04, 0xf5, 0xb1, 0x0d, 0xb7, 0x9a, 0x73, 0x12, 0x36, 0x93, 0xd9, - 0x32, 0xc6, 0x64, 0x71, 0x97, 0x7e, 0x00, 0x90, 0xb9, 0xda, 0x01, 0xb0, 0xa0, 0x3e, 0x73, 0xce, - 0xf4, 0x10, 0x36, 0x93, 0xc1, 0x77, 0x8b, 0x84, 0x49, 0xb5, 0x14, 0xb2, 0x6a, 0xa1, 0x07, 0xe0, - 0x9e, 0x30, 0x0f, 0x3b, 0xa9, 0xfb, 0x52, 0xa4, 0x57, 0x64, 0x2d, 0xbd, 0xe5, 0x50, 0xc6, 0x2a, - 0xd9, 0xa2, 0xbf, 0x35, 0xf6, 0x5c, 0x46, 0x0a, 0x19, 0xd7, 0x19, 0x7e, 0xd3, 0x1f, 0x1d, 0x80, - 0x67, 0xb2, 0x27, 0x3a, 0x8a, 0xab, 0x61, 0x4c, 0x1e, 0x60, 0x54, 0x8c, 0x55, 0xdb, 0xab, 0x67, - 0x67, 0x3a, 0x61, 0x1e, 0xc3, 0x7c, 0x7a, 0xda, 0x2b, 0xae, 0xd2, 0x09, 0x85, 0x06, 0x79, 0x13, - 0x96, 0x90, 0xa9, 0x48, 0x6a, 0x71, 0x65, 0x62, 0x80, 0xb0, 0x64, 0x9d, 0xbc, 0x0e, 0xe5, 0x13, - 0xe6, 0x75, 0x84, 0xb2, 0x33, 0x62, 0x22, 0x89, 0x5d, 0xa4, 0x4f, 0xa1, 0xbe, 0x1f, 0x0c, 0x63, - 0x25, 0x22, 0xcb, 0x2c, 0x4d, 0xec, 0xe4, 0x13, 0x67, 0xd1, 0x0a, 0x8b, 0xa2, 0x3d, 0x82, 0x1a, - 0x96, 0x2e, 0x8a, 0xc3, 0x73, 0xef, 0x15, 0x67, 0xf1, 0x7b, 0xa5, 0x03, 0xa5, 0xf9, 0xfd, 0x4a, - 0xa0, 0x88, 0x4f, 0x2e, 0x2b, 0x31, 0xbe, 0xb6, 0x1a, 0xe0, 0x1e, 0xf9, 0xa6, 0x26, 0x5c, 0xa6, - 0x3f, 0x11, 0xe1, 0x57, 0x58, 0xb3, 0x1a, 0xe1, 0xfa, 0x07, 0x6d, 0xd5, 0xd4, 0x80, 0x7e, 0x6e, - 0xdc, 0xe5, 0xa7, 0x27, 0x79, 0xb5, 0xb8, 0xb9, 0x57, 0xcb, 0xaf, 0x0e, 0xac, 0x32, 0x11, 0xfb, - 0xaf, 0x84, 0x17, 0xc6, 0x2a, 0x1a, 0xa6, 0xfd, 0xfb, 0x99, 0x3c, 0xf5, 0xda, 0x18, 0xd5, 0x65, - 0xc6, 0x48, 0x2e, 0xb9, 0x30, 0xf7, 0x92, 0xdf, 0xd6, 0xef, 0x5c, 0x19, 0xf5, 0x74, 0x13, 0xcb, - 0xc8, 0x56, 0xf8, 0x84, 0x63, 0xde, 0x83, 0xbc, 0x03, 0x4b, 0x1d, 0x39, 0x8c, 0xba, 0xe9, 0xe4, - 0x5f, 0xcb, 0x9c, 0x0d, 0x2b, 0xb3, 0xcc, 0x12, 0x37, 0xfa, 0xbd, 0x03, 0xf7, 0xf2, 0x2b, 0xff, - 0xa9, 0xf2, 0x8c, 0x42, 0x85, 0x99, 0x0a, 0xb9, 0xb3, 0x14, 0x2a, 0x66, 0x0a, 0x65, 0xef, 0x94, - 0x52, 0xee, 0x9d, 0x42, 0x19, 0xac, 0x4f, 0xc9, 0xb6, 0x2f, 0xfb, 0x03, 0x7d, 0x3f, 0x77, 0x94, - 0x8f, 0xbe, 0x0b, 0x95, 0x63, 0x39, 0x90, 0x81, 0x3c, 0x1b, 0xe5, 0x0a, 0xd4, 0x59, 0x50, 0xa0, - 0x8f, 0x1b, 0xbf, 0x5d, 0xb7, 0x9c, 0xdf, 0xaf, 0x5b, 0xce, 0x1f, 0xd7, 0x2d, 0xe7, 0xa7, 0x3f, - 0x5b, 0xff, 0x3b, 0x2d, 0xe3, 0x3f, 0x91, 0xf7, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x7e, 0xc9, - 0xe9, 0x9b, 0x9a, 0x0c, 0x00, 0x00, + // 1096 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0x66, 0x3c, 0xb6, 0x63, 0x57, 0xd6, 0x89, 0xd3, 0x2c, 0x91, 0x13, 0x45, 0xde, 0xa8, 0x25, + 0xd8, 0x10, 0x89, 0x00, 0x41, 0x42, 0xfc, 0x1d, 0x60, 0xe3, 0xac, 0x32, 0xb0, 0x59, 0x96, 0x76, + 0xb2, 0xdc, 0x90, 0x3a, 0x4e, 0x93, 0x1d, 0x65, 0x3c, 0x6d, 0x66, 0xda, 0x49, 0xbc, 0x07, 0x6e, + 0xf0, 0x06, 0x48, 0x48, 0x1c, 0x39, 0xf3, 0x1e, 0x1c, 0x79, 0x04, 0x14, 0x2e, 0xbc, 0x01, 0x12, + 0x27, 0xd4, 0xd5, 0x3d, 0x3f, 0x1e, 0xff, 0x84, 0xe4, 0x36, 0xf5, 0x75, 0x75, 0xd5, 0xd7, 0x5f, + 0x57, 0x95, 0xdb, 0xd0, 0x18, 0x44, 0xfe, 0x05, 0x57, 0x62, 0x67, 0x10, 0x49, 0x25, 0x49, 0xcd, + 0x0f, 0x95, 0x88, 0x42, 0x1e, 0xd0, 0x2f, 0xa1, 0xee, 0x85, 0xa7, 0xe2, 0xea, 0x50, 0x28, 0x4e, + 0x36, 0x61, 0x71, 0x4f, 0x06, 0xc3, 0x7e, 0xf8, 0x84, 0x9f, 0x88, 0xa0, 0xe5, 0x6c, 0x3a, 0x5b, + 0x75, 0x96, 0x87, 0xb4, 0xc7, 0x91, 0xdf, 0x17, 0x5f, 0x0d, 0x79, 0xa8, 0x86, 0xfd, 0x56, 0xc9, + 0x78, 0xe4, 0x20, 0xfa, 0xaf, 0x03, 0xf5, 0xc7, 0x11, 0xef, 0x0b, 0x8c, 0xb8, 0x0e, 0x35, 0x26, + 0x2f, 0xf3, 0xe1, 0x52, 0x9b, 0xbc, 0x01, 0x4b, 0x5e, 0x78, 0x21, 0xa2, 0x58, 0xec, 0x87, 0xfc, + 0x24, 0x10, 0xa7, 0x18, 0xae, 0xc6, 0x0a, 0x28, 0xd9, 0x80, 0xfa, 0x1e, 0xef, 0xbd, 0x10, 0x47, + 0xa3, 0x81, 0x68, 0xb9, 0x18, 0x24, 0x03, 0xd2, 0xd5, 0xae, 0xff, 0x52, 0xb4, 0xca, 0x9b, 0xce, + 0x56, 0x83, 0x65, 0x40, 0x91, 0x6f, 0x65, 0x82, 0x2f, 0xa1, 0x70, 0x8f, 0xf1, 0xf0, 0x2c, 0xe5, + 0x50, 0x45, 0x0e, 0x63, 0x18, 0x79, 0x08, 0xd5, 0xc7, 0xbe, 0x08, 0x4e, 0xe3, 0xd6, 0xc2, 0xa6, + 0xbb, 0xb5, 0xb8, 0xbb, 0xbc, 0x93, 0xe8, 0xb7, 0x83, 0x38, 0xb3, 0xcb, 0x94, 0xc2, 0x92, 0xd7, + 0x1f, 0xc8, 0x48, 0x31, 0x11, 0x0f, 0x64, 0x18, 0x0b, 0xd2, 0x04, 0x77, 0x3f, 0x8a, 0xec, 0xd9, + 0xf5, 0x27, 0xfd, 0x1e, 0x9a, 0x8f, 0x02, 0xd9, 0x3b, 0xef, 0x70, 0xc5, 0x99, 0xf8, 0x6e, 0x28, + 0x62, 0x45, 0xee, 0x43, 0x05, 0x6f, 0xc1, 0xfa, 0x19, 0x43, 0xa3, 0xa8, 0xa4, 0x95, 0xd9, 0x18, + 0x1a, 0xc5, 0xfd, 0x28, 0x45, 0x99, 0x19, 0x43, 0xa3, 0xdd, 0xc0, 0xef, 0x19, 0x09, 0xca, 0xcc, + 0x18, 0x84, 0x40, 0xf9, 0xb9, 0x2f, 0x2e, 0xed, 0xb9, 0xf1, 0x9b, 0x7a, 0xb0, 0x92, 0xcb, 0x6f, + 0x69, 0xae, 0x42, 0x95, 0xc9, 0x4b, 0xaf, 0x13, 0xb7, 0x9c, 0x4d, 0x77, 0xab, 0xcc, 0xac, 0x85, + 0xea, 0xe2, 0xf5, 0xeb, 0xa5, 0x12, 0x2e, 0x65, 0x00, 0x5d, 0x83, 0x0a, 0x4a, 0xad, 0x4f, 0x99, + 0xed, 0xd5, 0x9f, 0xf4, 0x17, 0x07, 0x56, 0x0e, 0xf9, 0x15, 0xd2, 0x88, 0xd3, 0x34, 0x07, 0x50, + 0x4f, 0x41, 0xf4, 0x5e, 0xdc, 0xdd, 0xce, 0xb4, 0x9c, 0xf0, 0xcf, 0x90, 0xfd, 0x50, 0x45, 0x23, + 0x96, 0x6d, 0x5e, 0xff, 0x04, 0x96, 0xc6, 0x17, 0x35, 0x87, 0x73, 0x31, 0x4a, 0x94, 0x3e, 0x17, + 0x23, 0xad, 0xc9, 0x05, 0x0f, 0x86, 0x46, 0xbf, 0x32, 0x33, 0xc6, 0x47, 0xa5, 0x0f, 0x1c, 0xfa, + 0x0d, 0x90, 0xbd, 0x48, 0x70, 0x25, 0x30, 0xc0, 0xa1, 0x88, 0x63, 0x7e, 0x26, 0x66, 0xdf, 0x82, + 0x51, 0xb6, 0x94, 0x57, 0x76, 0x03, 0xea, 0x5e, 0x6c, 0x0b, 0x15, 0x6f, 0xa2, 0xc6, 0x32, 0x80, + 0x6e, 0x03, 0xe9, 0x88, 0x40, 0x28, 0x61, 0x7b, 0x6b, 0x4e, 0x7c, 0xda, 0x4d, 0xb8, 0xdc, 0xec, + 0x4b, 0x1e, 0x42, 0x59, 0xb7, 0x15, 0x52, 0x59, 0xdc, 0x7d, 0x35, 0x93, 0x2e, 0xed, 0x61, 0x86, + 0x0e, 0xd4, 0x4f, 0x82, 0xda, 0x56, 0xbc, 0xe1, 0x80, 0x53, 0xca, 0x2c, 0x49, 0xe5, 0x16, 0x53, + 0xa5, 0xcd, 0x6d, 0x53, 0x7d, 0x9a, 0x9c, 0xf5, 0xae, 0xa9, 0x68, 0xc7, 0xa2, 0xba, 0x5c, 0x9f, + 0xea, 0x55, 0xb3, 0x07, 0xbf, 0x67, 0x1f, 0xb9, 0xc8, 0xe3, 0x6f, 0xc7, 0xa6, 0xbc, 0x5d, 0x98, + 0x82, 0x72, 0x7a, 0x62, 0x25, 0x85, 0x65, 0x3b, 0x2c, 0xb5, 0x71, 0x0e, 0xe8, 0xac, 0x71, 0xab, + 0x3c, 0x31, 0x07, 0x34, 0xce, 0xec, 0xb2, 0x6e, 0x27, 0x5b, 0xe4, 0x15, 0xd3, 0x4e, 0xc6, 0x22, + 0xfb, 0xd0, 0xf4, 0xc2, 0xc1, 0x50, 0x75, 0xc4, 0xb7, 0x7e, 0xe8, 0x2b, 0x5f, 0x86, 0x71, 0xab, + 0x8a, 0xa1, 0xd6, 0xf2, 0x8c, 0xc6, 0x3c, 0xd8, 0xc4, 0x16, 0xfa, 0xa3, 0x03, 0xcb, 0x05, 0x70, + 0xc6, 0xa1, 0x13, 0xbe, 0xa5, 0xf9, 0x7c, 0xdf, 0x4f, 0x07, 0x9c, 0x8b, 0x8e, 0xed, 0x99, 0x6c, + 0xc6, 0xe7, 0xdd, 0xaf, 0x0e, 0xdc, 0x9f, 0xe6, 0x30, 0x95, 0x4d, 0x1b, 0xe0, 0x59, 0xe4, 0xf7, + 0x79, 0x34, 0xfa, 0x42, 0x8c, 0xec, 0xac, 0xcf, 0x21, 0xe4, 0x6b, 0x58, 0x2d, 0xc4, 0xfa, 0xac, + 0x67, 0x24, 0x32, 0xa4, 0x1e, 0xcc, 0x24, 0x65, 0xfc, 0xd8, 0x8c, 0xed, 0xf4, 0x1f, 0x07, 0x5e, + 0x9b, 0xba, 0x94, 0xd5, 0xa3, 0x93, 0x2f, 0xfd, 0x6d, 0x68, 0x3e, 0xd7, 0xa3, 0xa2, 0x23, 0x62, + 0xe5, 0x87, 0x5c, 0x7b, 0xda, 0x82, 0x9d, 0xc0, 0x89, 0x07, 0x35, 0xc4, 0x0e, 0xf9, 0xc0, 0xd2, + 0x7c, 0xeb, 0x06, 0x9a, 0x3b, 0x89, 0xbf, 0x99, 0x69, 0xe9, 0x76, 0x4d, 0x06, 0xa7, 0x6e, 0x32, + 0xc2, 0xd1, 0x58, 0xff, 0x18, 0x1a, 0x63, 0x1b, 0x6e, 0x35, 0xe7, 0x24, 0x6c, 0x24, 0xb3, 0x65, + 0x8c, 0xc9, 0xfc, 0x2e, 0xfd, 0x10, 0x20, 0x73, 0xb5, 0x03, 0x60, 0x4e, 0x7d, 0xe6, 0x9c, 0xe9, + 0x01, 0x6c, 0x24, 0x83, 0xef, 0x16, 0x09, 0x93, 0x6a, 0x29, 0x65, 0xd5, 0x42, 0xf7, 0xc1, 0x3d, + 0x66, 0x1e, 0x76, 0x52, 0xef, 0x85, 0x48, 0xaf, 0xc8, 0x5a, 0x7a, 0xcb, 0x81, 0x8c, 0x55, 0xb2, + 0x45, 0x7f, 0x6b, 0xec, 0x99, 0x8c, 0x14, 0x32, 0x6e, 0x30, 0xfc, 0xa6, 0x3f, 0x39, 0x00, 0x4f, + 0xe5, 0xa9, 0xe8, 0x2a, 0xae, 0x86, 0x31, 0x79, 0x80, 0x51, 0x31, 0xd6, 0xe2, 0x6e, 0x23, 0x3b, + 0xd3, 0x31, 0xf3, 0x18, 0xe6, 0xd3, 0xd3, 0x5e, 0x71, 0x95, 0x4e, 0x28, 0x34, 0xc8, 0x9b, 0xb0, + 0x80, 0x4c, 0x45, 0x52, 0x8b, 0xcb, 0x85, 0x01, 0xc2, 0x92, 0x75, 0xf2, 0x3a, 0x54, 0x8f, 0x99, + 0xd7, 0x15, 0xca, 0xce, 0x88, 0x42, 0x12, 0xbb, 0x48, 0x9f, 0x40, 0x63, 0x2f, 0x18, 0xc6, 0x4a, + 0x44, 0x96, 0x59, 0x9a, 0xd8, 0xc9, 0x27, 0xce, 0xa2, 0x95, 0xe6, 0x45, 0xeb, 0x42, 0x65, 0x76, + 0xdf, 0x11, 0x28, 0xe3, 0xd3, 0xc9, 0x4a, 0x85, 0xaf, 0xa6, 0x26, 0xb8, 0x87, 0xbe, 0xb9, 0x5b, + 0x97, 0xe9, 0x4f, 0x44, 0xf8, 0x15, 0xd6, 0x9e, 0x46, 0xb8, 0xfe, 0x61, 0x5a, 0x31, 0x77, 0xa9, + 0x9f, 0x0d, 0x77, 0xf9, 0x09, 0x49, 0x5e, 0x1f, 0x6e, 0xee, 0xf5, 0xf1, 0x9b, 0x03, 0x2b, 0x4c, + 0xc4, 0xfe, 0x4b, 0xe1, 0x85, 0xb1, 0x8a, 0x86, 0x69, 0x1f, 0x7e, 0x2e, 0x4f, 0xbc, 0x0e, 0x46, + 0x75, 0x99, 0x31, 0x92, 0xcb, 0x2a, 0xcd, 0xbc, 0xac, 0xb7, 0xf5, 0x7b, 0x55, 0x46, 0xa7, 0xba, + 0x19, 0x65, 0x64, 0x2b, 0xb5, 0xe0, 0x98, 0xf7, 0x20, 0xef, 0xc0, 0x42, 0x57, 0x0e, 0xa3, 0x5e, + 0x3a, 0xc1, 0x57, 0x33, 0x67, 0xc3, 0xca, 0x2c, 0xb3, 0xc4, 0x8d, 0xfe, 0xe0, 0xc0, 0xbd, 0xfc, + 0xca, 0xff, 0xaa, 0x20, 0xa3, 0x50, 0x69, 0xaa, 0x42, 0xee, 0x34, 0x85, 0xca, 0x99, 0x42, 0xd9, + 0x7b, 0xa3, 0x92, 0x7b, 0x6f, 0x50, 0x06, 0x6b, 0x13, 0xb2, 0xed, 0xc9, 0xfe, 0x40, 0xdf, 0xcf, + 0x1d, 0xe5, 0xa3, 0xef, 0x42, 0xed, 0x48, 0x0e, 0x64, 0x20, 0xcf, 0x46, 0xb9, 0x42, 0x73, 0xe6, + 0x14, 0xda, 0xa3, 0xe6, 0xef, 0xd7, 0x6d, 0xe7, 0x8f, 0xeb, 0xb6, 0xf3, 0xe7, 0x75, 0xdb, 0xf9, + 0xf9, 0xaf, 0xf6, 0x2b, 0x27, 0x55, 0xfc, 0x47, 0xf1, 0xde, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, + 0xd7, 0x59, 0xea, 0x61, 0x62, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 1b1bdd6fb..fdb5fe082 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -130,10 +130,6 @@ message ClusterStatus { repeated URI URISet = 2; } -message FrameSchema { - repeated Field Fields = 1; -} - message Field { string Name = 1; string Type = 2; From 2bd0677df9ab0fb238b1ecbcc43ca55332b6c588 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 26 Oct 2017 23:33:34 -0500 Subject: [PATCH 005/311] Remove MaxSlice polling. Add Schema to proto. Update MaxSlices in proto to include both standard and inverse. Update LocalStatus (shared in gossip) to include MaxSlices and Schema. Encode InputDefinitions with Index. TODO: - make sure InputDefinitions are considered in LocalStatus merge. - decide what to do about `/slices/max` endpoint. client.backup is using it. --- cluster.go | 39 +- cmd/server_test.go | 4 - config.go | 2 - ctl/server.go | 1 - gossip/gossip.go | 3 +- handler.go | 4 +- holder.go | 17 + index.go | 23 +- input_definition.go | 59 ++- internal/private.pb.go | 905 +++++++++++++++++++++++------------------ internal/private.proto | 22 +- server.go | 169 +++----- server/server.go | 13 - 13 files changed, 657 insertions(+), 604 deletions(-) diff --git a/cluster.go b/cluster.go index 4f389fb4b..8bb9b5ff2 100644 --- a/cluster.go +++ b/cluster.go @@ -53,24 +53,7 @@ const ( // Node represents a node in the cluster. type Node struct { - //Scheme string `json:"scheme"` - //Host string `json:"host"` // HostPort URI URI `json:"uri"` - - status *internal.NodeStatus `json:"status"` -} - -// SetStatus sets the NodeStatus. -func (n *Node) SetStatus(s *internal.NodeStatus) { - n.status = s -} - -// SetState sets the Node.status.state. -func (n *Node) SetState(s string) { - if n.status == nil { - n.status = &internal.NodeStatus{} - } - n.status.State = s } // Nodes represents a list of nodes. @@ -242,13 +225,12 @@ func (c *Cluster) URISet() []URI { func (c *Cluster) setState(state string) { c.State = state - localNode := c.localNode() - localNode.SetState(state) } -func (c *Cluster) localNode() *Node { - return c.NodeByURI(c.URI) -} +// localNode is not being used. +//func (c *Cluster) localNode() *Node { +// return c.NodeByURI(c.URI) +//} // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { @@ -258,17 +240,6 @@ func (c *Cluster) Status() *internal.ClusterStatus { } } -/* -// encodeNodeStatuses converts a into its internal representation. -func encodeNodeStatuses(a []*Node) []*internal.NodeStatus { - other := make([]*internal.NodeStatus, len(a)) - for i := range a { - other[i] = a[i].status - } - return other -} -*/ - // NodeByURI returns a node reference by uri. func (c *Cluster) NodeByURI(uri URI) *Node { for _, n := range c.Nodes { @@ -609,7 +580,7 @@ func (c *Cluster) handleJoiningHost(uri URI) error { func (c *Cluster) setStateAndBroadcast(state string) error { c.setState(state) - // Broadcast status changes to the cluster. + // Broadcast cluster status changes to the cluster. return c.Broadcaster.SendSync(c.Status()) } diff --git a/cmd/server_test.go b/cmd/server_test.go index 53841a5d9..b09962663 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -51,7 +51,6 @@ func TestServerConfig(t *testing.T) { bind = "localhost:0" [cluster] - poll-interval = "45s" type = "static" replicas = 2 hosts = [ @@ -64,7 +63,6 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) - v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182)) return v.Error() }, }, @@ -99,7 +97,6 @@ func TestServerConfig(t *testing.T) { bind = "localhost:19444" data-dir = "` + actualDataDir + `" [cluster] - poll-interval = "2m0s" hosts = [ "localhost:19444", ] @@ -115,7 +112,6 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) - v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Minute*2)) v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11)) v.Check(cmd.Server.CPUProfile, profFile.Name()) v.Check(cmd.Server.CPUTime, time.Minute) diff --git a/config.go b/config.go index 918c0415a..25ae85cd8 100644 --- a/config.go +++ b/config.go @@ -78,7 +78,6 @@ type Config struct { ReplicaN int `toml:"replicas"` Type string `toml:"type"` Hosts []string `toml:"hosts"` - PollInterval Duration `toml:"poll-interval"` LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` @@ -113,7 +112,6 @@ func NewConfig() *Config { } c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Type = DefaultClusterType - c.Cluster.PollInterval = Duration(DefaultPollingInterval) c.Cluster.Hosts = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) c.Metric.Service = DefaultMetrics diff --git a/ctl/server.go b/ctl/server.go index 5c9fd245e..a2531682d 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -35,7 +35,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Long Query Time.") flags.StringVarP(&srv.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") diff --git a/gossip/gossip.go b/gossip/gossip.go index e17588e51..2cd2e09e0 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -148,8 +148,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) g.config.memberlistConfig.AdvertisePort = gossipPort - // TODO travis: pause node status (remove this next line) - g.config.memberlistConfig.PushPullInterval = 0 * time.Millisecond + //g.config.memberlistConfig.PushPullInterval = 15 * time.Second // Default is 15s in DefaultLocalConfig. g.config.memberlistConfig.Delegate = g g.config.memberlistConfig.SecretKey = secretKey g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) diff --git a/handler.go b/handler.go index ef5655f43..ffe189e35 100644 --- a/handler.go +++ b/handler.go @@ -132,7 +132,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") + //router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") // TODO: this is being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -305,6 +305,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } +/* func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { var ms map[string]uint64 if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { @@ -327,6 +328,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { MaxSlices: ms, }) } +*/ type sliceMaxResponse struct { MaxSlices map[string]uint64 `json:"maxSlices"` diff --git a/holder.go b/holder.go index 909233159..ae314a274 100644 --- a/holder.go +++ b/holder.go @@ -26,6 +26,8 @@ import ( "sync" "syscall" "time" + + "github.com/pilosa/pilosa/internal" ) const ( @@ -185,6 +187,21 @@ func (h *Holder) Schema() []*IndexInfo { return a } +// EncodeMaxSlices creates and internal representation of max slices. +func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { + return &internal.MaxSlices{ + Standard: h.MaxSlices(), + Inverse: h.MaxInverseSlices(), + } +} + +// EncodeSchema creates and internal representation of schema. +func (h *Holder) EncodeSchema() *internal.Schema { + return &internal.Schema{ + Indexes: EncodeIndexes(h.Indexes()), + } +} + // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } diff --git a/index.go b/index.go index 00e08002f..734ea7ed5 100644 --- a/index.go +++ b/index.go @@ -392,6 +392,20 @@ func (i *Index) Frames() []*Frame { return a } +// InputDefinitions returns a list of all inputDefinitions in the index. +func (i *Index) InputDefinitions() []*InputDefinition { + i.mu.RLock() + defer i.mu.RUnlock() + + a := make([]*InputDefinition, 0, len(i.inputDefinitions)) + for _, d := range i.inputDefinitions { + a = append(a, d) + } + //sort.Sort(inputDefintionSlice(a)) // TODO + + return a +} + // RecalculateCaches recalculates caches on every frame in the index. func (i *Index) RecalculateCaches() { for _, frame := range i.Frames() { @@ -617,12 +631,10 @@ func EncodeIndexes(a []*Index) []*internal.Index { // encodeIndex converts d into its internal representation. func encodeIndex(d *Index) *internal.Index { - io := d.options() return &internal.Index{ - Name: d.name, - Meta: io.Encode(), - MaxSlice: d.MaxSlice(), - Frames: encodeFrames(d.Frames()), + Name: d.name, + Frames: encodeFrames(d.Frames()), + InputDefinitions: encodeInputDefinitions(d.InputDefinitions()), } } @@ -716,7 +728,6 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { if err != nil { return nil, err } - inputDef.broadcaster = i.broadcaster return inputDef, nil } diff --git a/input_definition.go b/input_definition.go index 13b27bd17..98ca8ca09 100644 --- a/input_definition.go +++ b/input_definition.go @@ -36,12 +36,11 @@ var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleR // InputDefinition represents a container for the data input definition. type InputDefinition struct { - name string - path string - index string - broadcaster Broadcaster - frames []InputFrame - fields []InputDefinitionField + name string + path string + index string + frames []InputFrame + fields []InputDefinitionField } // NewInputDefinition returns a new instance of InputDefinition. @@ -86,16 +85,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name for _, fr := range pb.Frames { - frameMeta := fr.Meta inputFrame := InputFrame{ - Name: fr.Name, - Options: FrameOptions{ - RowLabel: frameMeta.RowLabel, - InverseEnabled: frameMeta.InverseEnabled, - CacheSize: frameMeta.CacheSize, - CacheType: frameMeta.CacheType, - TimeQuantum: TimeQuantum(frameMeta.TimeQuantum), - }, + Name: fr.Name, + Options: *decodeFrameOptions(fr.Meta), } i.frames = append(i.frames, inputFrame) } @@ -327,6 +319,43 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { return &def } +// encodeInputDefinitions converts a into its internal representation. +func encodeInputDefinitions(a []*InputDefinition) []*internal.InputDefinition { + other := make([]*internal.InputDefinition, len(a)) + for i := range a { + other[i] = encodeInputDefinition(a[i]) + } + return other +} + +// encodeInputDefinition converts i into its internal representation. +func encodeInputDefinition(i *InputDefinition) *internal.InputDefinition { + //fo := f.options() + return &internal.InputDefinition{ + Name: i.name, + Frames: encodeInputFrames(i.frames), + Fields: encodeInputDefinitionFields(i.fields), + } +} + +// encodeInputFrames converts a into its internal representation. +func encodeInputFrames(a []InputFrame) []*internal.Frame { + other := make([]*internal.Frame, len(a)) + for i := range a { + other[i] = a[i].Encode() + } + return other +} + +// encodeInputDefinitionFields converts a into its internal representation. +func encodeInputDefinitionFields(a []InputDefinitionField) []*internal.InputDefinitionField { + other := make([]*internal.InputDefinitionField, len(a)) + for i := range a { + other[i] = a[i].Encode() + } + return other +} + // AddFrame manually add frame to input definition. func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) diff --git a/internal/private.pb.go b/internal/private.pb.go index bab2ad1ba..6993a7070 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -14,13 +14,14 @@ BlockDataRequest BlockDataResponse Cache - MaxSlicesResponse + MaxSlices CreateSliceMessage DeleteIndexMessage CreateIndexMessage CreateFrameMessage DeleteFrameMessage Frame + Schema Index InputDefinition InputDefinitionField @@ -248,18 +249,26 @@ func (m *Cache) GetIDs() []uint64 { return nil } -type MaxSlicesResponse struct { - MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +type MaxSlices struct { + Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxSlicesResponse) Reset() { *m = MaxSlicesResponse{} } -func (m *MaxSlicesResponse) String() string { return proto.CompactTextString(m) } -func (*MaxSlicesResponse) ProtoMessage() {} -func (*MaxSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } +func (m *MaxSlices) Reset() { *m = MaxSlices{} } +func (m *MaxSlices) String() string { return proto.CompactTextString(m) } +func (*MaxSlices) ProtoMessage() {} +func (*MaxSlices) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } -func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { +func (m *MaxSlices) GetStandard() map[string]uint64 { if m != nil { - return m.MaxSlices + return m.Standard + } + return nil +} + +func (m *MaxSlices) GetInverse() map[string]uint64 { + if m != nil { + return m.Inverse } return nil } @@ -393,8 +402,9 @@ func (m *DeleteFrameMessage) GetFrame() string { } type Frame struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } func (m *Frame) Reset() { *m = Frame{} } @@ -416,19 +426,42 @@ func (m *Frame) GetMeta() *FrameMeta { return nil } +func (m *Frame) GetViews() []string { + if m != nil { + return m.Views + } + return nil +} + +type Schema struct { + Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` +} + +func (m *Schema) Reset() { *m = Schema{} } +func (m *Schema) String() string { return proto.CompactTextString(m) } +func (*Schema) ProtoMessage() {} +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } + +func (m *Schema) GetIndexes() []*Index { + if m != nil { + return m.Indexes + } + return nil +} + type Index struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` - Slices []uint64 `protobuf:"varint,5,rep,packed,name=Slices" json:"Slices,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + // IndexMeta Meta = 2; + // uint64 MaxSlice = 3; + Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + // repeated uint64 Slices = 5; InputDefinitions []*InputDefinition `protobuf:"bytes,6,rep,name=InputDefinitions" json:"InputDefinitions,omitempty"` } func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Index) GetName() string { if m != nil { @@ -437,20 +470,6 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetMeta() *IndexMeta { - if m != nil { - return m.Meta - } - return nil -} - -func (m *Index) GetMaxSlice() uint64 { - if m != nil { - return m.MaxSlice - } - return 0 -} - func (m *Index) GetFrames() []*Frame { if m != nil { return m.Frames @@ -458,13 +477,6 @@ func (m *Index) GetFrames() []*Frame { return nil } -func (m *Index) GetSlices() []uint64 { - if m != nil { - return m.Slices - } - return nil -} - func (m *Index) GetInputDefinitions() []*InputDefinition { if m != nil { return m.InputDefinitions @@ -481,7 +493,7 @@ type InputDefinition struct { func (m *InputDefinition) Reset() { *m = InputDefinition{} } func (m *InputDefinition) String() string { return proto.CompactTextString(m) } func (*InputDefinition) ProtoMessage() {} -func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *InputDefinition) GetName() string { if m != nil { @@ -513,7 +525,7 @@ type InputDefinitionField struct { func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } func (m *InputDefinitionField) String() string { return proto.CompactTextString(m) } func (*InputDefinitionField) ProtoMessage() {} -func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *InputDefinitionField) GetName() string { if m != nil { @@ -546,7 +558,7 @@ type InputDefinitionAction struct { func (m *InputDefinitionAction) Reset() { *m = InputDefinitionAction{} } func (m *InputDefinitionAction) String() string { return proto.CompactTextString(m) } func (*InputDefinitionAction) ProtoMessage() {} -func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *InputDefinitionAction) GetFrame() string { if m != nil { @@ -585,7 +597,7 @@ func (m *CreateInputDefinitionMessage) Reset() { *m = CreateInputDefinit func (m *CreateInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*CreateInputDefinitionMessage) ProtoMessage() {} func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{17} + return fileDescriptorPrivate, []int{18} } func (m *CreateInputDefinitionMessage) GetIndex() string { @@ -611,7 +623,7 @@ func (m *DeleteInputDefinitionMessage) Reset() { *m = DeleteInputDefinit func (m *DeleteInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*DeleteInputDefinitionMessage) ProtoMessage() {} func (*DeleteInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{18} + return fileDescriptorPrivate, []int{19} } func (m *DeleteInputDefinitionMessage) GetIndex() string { @@ -637,7 +649,7 @@ type URI struct { func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} -func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *URI) GetScheme() string { if m != nil { @@ -661,16 +673,15 @@ func (m *URI) GetPort() uint32 { } type NodeStatus struct { - URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` - URISet []*URI `protobuf:"bytes,4,rep,name=URISet" json:"URISet,omitempty"` + URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` + MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` + Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` } func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } func (m *NodeStatus) GetURI() *URI { if m != nil { @@ -679,23 +690,16 @@ func (m *NodeStatus) GetURI() *URI { return nil } -func (m *NodeStatus) GetState() string { +func (m *NodeStatus) GetMaxSlices() *MaxSlices { if m != nil { - return m.State - } - return "" -} - -func (m *NodeStatus) GetIndexes() []*Index { - if m != nil { - return m.Indexes + return m.MaxSlices } return nil } -func (m *NodeStatus) GetURISet() []*URI { +func (m *NodeStatus) GetSchema() *Schema { if m != nil { - return m.URISet + return m.Schema } return nil } @@ -708,7 +712,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *ClusterStatus) GetState() string { if m != nil { @@ -734,7 +738,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *Field) GetName() string { if m != nil { @@ -773,7 +777,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -806,7 +810,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -847,7 +851,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *ResizeSource) GetURI() *URI { if m != nil { @@ -893,7 +897,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{26} + return fileDescriptorPrivate, []int{27} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -917,7 +921,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *Topology) GetURISet() []*URI { if m != nil { @@ -933,13 +937,14 @@ func init() { proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") - proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") + proto.RegisterType((*MaxSlices)(nil), "internal.MaxSlices") proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") @@ -1216,7 +1221,7 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *MaxSlicesResponse) Marshal() (dAtA []byte, err error) { +func (m *MaxSlices) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1226,16 +1231,32 @@ func (m *MaxSlicesResponse) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *MaxSlicesResponse) MarshalTo(dAtA []byte) (int, error) { +func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.MaxSlices) > 0 { - for k, _ := range m.MaxSlices { + if len(m.Standard) > 0 { + for k, _ := range m.Standard { dAtA[i] = 0xa i++ - v := m.MaxSlices[k] + v := m.Standard[k] + mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(v)) + } + } + if len(m.Inverse) > 0 { + for k, _ := range m.Inverse { + dAtA[i] = 0x12 + i++ + v := m.Inverse[k] mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) dAtA[i] = 0xa @@ -1448,6 +1469,51 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { } i += n9 } + if len(m.Views) > 0 { + for _, s := range m.Views { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func (m *Schema) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Schema) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Indexes) > 0 { + for _, msg := range m.Indexes { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -1472,21 +1538,6 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } - if m.Meta != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n10, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n10 - } - if m.MaxSlice != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlice)) - } if len(m.Frames) > 0 { for _, msg := range m.Frames { dAtA[i] = 0x22 @@ -1499,23 +1550,6 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.Slices) > 0 { - dAtA12 := make([]byte, len(m.Slices)*10) - var j11 int - for _, num := range m.Slices { - for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j11++ - } - dAtA12[j11] = uint8(num) - j11++ - } - dAtA[i] = 0x2a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) - } if len(m.InputDefinitions) > 0 { for _, msg := range m.InputDefinitions { dAtA[i] = 0x32 @@ -1701,11 +1735,11 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Definition.Size())) - n13, err := m.Definition.MarshalTo(dAtA[i:]) + n10, err := m.Definition.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n10 } return i, nil } @@ -1794,41 +1828,31 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n14, err := m.URI.MarshalTo(dAtA[i:]) + n11, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n11 } - if len(m.State) > 0 { + if m.MaxSlices != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i += copy(dAtA[i:], m.State) - } - if len(m.Indexes) > 0 { - for _, msg := range m.Indexes { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) + n12, err := m.MaxSlices.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n12 } - if len(m.URISet) > 0 { - for _, msg := range m.URISet { - dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n + if m.Schema != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n13, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n13 } return i, nil } @@ -1969,21 +1993,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n15, err := m.URI.MarshalTo(dAtA[i:]) + n14, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n14 } if m.Coordinator != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n16, err := m.Coordinator.MarshalTo(dAtA[i:]) + n15, err := m.Coordinator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n15 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -2019,11 +2043,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n17, err := m.URI.MarshalTo(dAtA[i:]) + n16, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n16 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2075,11 +2099,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n18, err := m.URI.MarshalTo(dAtA[i:]) + n17, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n17 } return i, nil } @@ -2237,11 +2261,19 @@ func (m *Cache) Size() (n int) { return n } -func (m *MaxSlicesResponse) Size() (n int) { +func (m *MaxSlices) Size() (n int) { var l int _ = l - if len(m.MaxSlices) > 0 { - for k, v := range m.MaxSlices { + if len(m.Standard) > 0 { + for k, v := range m.Standard { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if len(m.Inverse) > 0 { + for k, v := range m.Inverse { _ = k _ = v mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) @@ -2334,6 +2366,24 @@ func (m *Frame) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if len(m.Views) > 0 { + for _, s := range m.Views { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *Schema) Size() (n int) { + var l int + _ = l + if len(m.Indexes) > 0 { + for _, e := range m.Indexes { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } return n } @@ -2344,26 +2394,12 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - if m.MaxSlice != 0 { - n += 1 + sovPrivate(uint64(m.MaxSlice)) - } if len(m.Frames) > 0 { for _, e := range m.Frames { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } } - if len(m.Slices) > 0 { - l = 0 - for _, e := range m.Slices { - l += sovPrivate(uint64(e)) - } - n += 1 + sovPrivate(uint64(l)) + l - } if len(m.InputDefinitions) > 0 { for _, e := range m.InputDefinitions { l = e.Size() @@ -2491,21 +2527,13 @@ func (m *NodeStatus) Size() (n int) { l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.State) - if l > 0 { + if m.MaxSlices != nil { + l = m.MaxSlices.Size() n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Indexes) > 0 { - for _, e := range m.Indexes { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - } - if len(m.URISet) > 0 { - for _, e := range m.URISet { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) } return n } @@ -3525,7 +3553,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } return nil } -func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { +func (m *MaxSlices) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3548,15 +3576,15 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: MaxSlicesResponse: wiretype end group for non-group") + return fmt.Errorf("proto: MaxSlices: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: MaxSlicesResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: MaxSlices: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Standard", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3580,8 +3608,8 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.MaxSlices == nil { - m.MaxSlices = make(map[string]uint64) + if m.Standard == nil { + m.Standard = make(map[string]uint64) } var mapkey string var mapvalue uint64 @@ -3659,7 +3687,114 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.MaxSlices[mapkey] = mapvalue + m.Standard[mapkey] = mapvalue + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Inverse == nil { + m.Inverse = make(map[string]uint64) + } + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Inverse[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -4331,6 +4466,116 @@ func (m *Frame) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Views", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Views = append(m.Views, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Schema) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Schema: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Schema: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Indexes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Indexes = append(m.Indexes, &Index{}) + if err := m.Indexes[len(m.Indexes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -4410,58 +4655,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Meta == nil { - m.Meta = &IndexMeta{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlice", wireType) - } - m.MaxSlice = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.MaxSlice |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) @@ -4493,68 +4686,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 5: - if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) - } else if wireType == 2 { - var packedLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - packedLen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if packedLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + packedLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - for iNdEx < postIndex { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) - } - } else { - return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) - } case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitions", wireType) @@ -5523,36 +5654,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.State = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Indexes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5576,14 +5678,16 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Indexes = append(m.Indexes, &Index{}) - if err := m.Indexes[len(m.Indexes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if m.MaxSlices == nil { + m.MaxSlices = &MaxSlices{} + } + if err := m.MaxSlices.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 4: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URISet", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5607,8 +5711,10 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.URISet = append(m.URISet, &URI{}) - if err := m.URISet[len(m.URISet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6672,74 +6778,77 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1096 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0x66, 0x3c, 0xb6, 0x63, 0x57, 0xd6, 0x89, 0xd3, 0x2c, 0x91, 0x13, 0x45, 0xde, 0xa8, 0x25, - 0xd8, 0x10, 0x89, 0x00, 0x41, 0x42, 0xfc, 0x1d, 0x60, 0xe3, 0xac, 0x32, 0xb0, 0x59, 0x96, 0x76, - 0xb2, 0xdc, 0x90, 0x3a, 0x4e, 0x93, 0x1d, 0x65, 0x3c, 0x6d, 0x66, 0xda, 0x49, 0xbc, 0x07, 0x6e, - 0xf0, 0x06, 0x48, 0x48, 0x1c, 0x39, 0xf3, 0x1e, 0x1c, 0x79, 0x04, 0x14, 0x2e, 0xbc, 0x01, 0x12, - 0x27, 0xd4, 0xd5, 0x3d, 0x3f, 0x1e, 0xff, 0x84, 0xe4, 0x36, 0xf5, 0x75, 0x75, 0xd5, 0xd7, 0x5f, - 0x57, 0x95, 0xdb, 0xd0, 0x18, 0x44, 0xfe, 0x05, 0x57, 0x62, 0x67, 0x10, 0x49, 0x25, 0x49, 0xcd, - 0x0f, 0x95, 0x88, 0x42, 0x1e, 0xd0, 0x2f, 0xa1, 0xee, 0x85, 0xa7, 0xe2, 0xea, 0x50, 0x28, 0x4e, - 0x36, 0x61, 0x71, 0x4f, 0x06, 0xc3, 0x7e, 0xf8, 0x84, 0x9f, 0x88, 0xa0, 0xe5, 0x6c, 0x3a, 0x5b, - 0x75, 0x96, 0x87, 0xb4, 0xc7, 0x91, 0xdf, 0x17, 0x5f, 0x0d, 0x79, 0xa8, 0x86, 0xfd, 0x56, 0xc9, - 0x78, 0xe4, 0x20, 0xfa, 0xaf, 0x03, 0xf5, 0xc7, 0x11, 0xef, 0x0b, 0x8c, 0xb8, 0x0e, 0x35, 0x26, - 0x2f, 0xf3, 0xe1, 0x52, 0x9b, 0xbc, 0x01, 0x4b, 0x5e, 0x78, 0x21, 0xa2, 0x58, 0xec, 0x87, 0xfc, - 0x24, 0x10, 0xa7, 0x18, 0xae, 0xc6, 0x0a, 0x28, 0xd9, 0x80, 0xfa, 0x1e, 0xef, 0xbd, 0x10, 0x47, - 0xa3, 0x81, 0x68, 0xb9, 0x18, 0x24, 0x03, 0xd2, 0xd5, 0xae, 0xff, 0x52, 0xb4, 0xca, 0x9b, 0xce, - 0x56, 0x83, 0x65, 0x40, 0x91, 0x6f, 0x65, 0x82, 0x2f, 0xa1, 0x70, 0x8f, 0xf1, 0xf0, 0x2c, 0xe5, - 0x50, 0x45, 0x0e, 0x63, 0x18, 0x79, 0x08, 0xd5, 0xc7, 0xbe, 0x08, 0x4e, 0xe3, 0xd6, 0xc2, 0xa6, - 0xbb, 0xb5, 0xb8, 0xbb, 0xbc, 0x93, 0xe8, 0xb7, 0x83, 0x38, 0xb3, 0xcb, 0x94, 0xc2, 0x92, 0xd7, - 0x1f, 0xc8, 0x48, 0x31, 0x11, 0x0f, 0x64, 0x18, 0x0b, 0xd2, 0x04, 0x77, 0x3f, 0x8a, 0xec, 0xd9, - 0xf5, 0x27, 0xfd, 0x1e, 0x9a, 0x8f, 0x02, 0xd9, 0x3b, 0xef, 0x70, 0xc5, 0x99, 0xf8, 0x6e, 0x28, - 0x62, 0x45, 0xee, 0x43, 0x05, 0x6f, 0xc1, 0xfa, 0x19, 0x43, 0xa3, 0xa8, 0xa4, 0x95, 0xd9, 0x18, - 0x1a, 0xc5, 0xfd, 0x28, 0x45, 0x99, 0x19, 0x43, 0xa3, 0xdd, 0xc0, 0xef, 0x19, 0x09, 0xca, 0xcc, - 0x18, 0x84, 0x40, 0xf9, 0xb9, 0x2f, 0x2e, 0xed, 0xb9, 0xf1, 0x9b, 0x7a, 0xb0, 0x92, 0xcb, 0x6f, - 0x69, 0xae, 0x42, 0x95, 0xc9, 0x4b, 0xaf, 0x13, 0xb7, 0x9c, 0x4d, 0x77, 0xab, 0xcc, 0xac, 0x85, - 0xea, 0xe2, 0xf5, 0xeb, 0xa5, 0x12, 0x2e, 0x65, 0x00, 0x5d, 0x83, 0x0a, 0x4a, 0xad, 0x4f, 0x99, - 0xed, 0xd5, 0x9f, 0xf4, 0x17, 0x07, 0x56, 0x0e, 0xf9, 0x15, 0xd2, 0x88, 0xd3, 0x34, 0x07, 0x50, - 0x4f, 0x41, 0xf4, 0x5e, 0xdc, 0xdd, 0xce, 0xb4, 0x9c, 0xf0, 0xcf, 0x90, 0xfd, 0x50, 0x45, 0x23, - 0x96, 0x6d, 0x5e, 0xff, 0x04, 0x96, 0xc6, 0x17, 0x35, 0x87, 0x73, 0x31, 0x4a, 0x94, 0x3e, 0x17, - 0x23, 0xad, 0xc9, 0x05, 0x0f, 0x86, 0x46, 0xbf, 0x32, 0x33, 0xc6, 0x47, 0xa5, 0x0f, 0x1c, 0xfa, - 0x0d, 0x90, 0xbd, 0x48, 0x70, 0x25, 0x30, 0xc0, 0xa1, 0x88, 0x63, 0x7e, 0x26, 0x66, 0xdf, 0x82, - 0x51, 0xb6, 0x94, 0x57, 0x76, 0x03, 0xea, 0x5e, 0x6c, 0x0b, 0x15, 0x6f, 0xa2, 0xc6, 0x32, 0x80, - 0x6e, 0x03, 0xe9, 0x88, 0x40, 0x28, 0x61, 0x7b, 0x6b, 0x4e, 0x7c, 0xda, 0x4d, 0xb8, 0xdc, 0xec, - 0x4b, 0x1e, 0x42, 0x59, 0xb7, 0x15, 0x52, 0x59, 0xdc, 0x7d, 0x35, 0x93, 0x2e, 0xed, 0x61, 0x86, - 0x0e, 0xd4, 0x4f, 0x82, 0xda, 0x56, 0xbc, 0xe1, 0x80, 0x53, 0xca, 0x2c, 0x49, 0xe5, 0x16, 0x53, - 0xa5, 0xcd, 0x6d, 0x53, 0x7d, 0x9a, 0x9c, 0xf5, 0xae, 0xa9, 0x68, 0xc7, 0xa2, 0xba, 0x5c, 0x9f, - 0xea, 0x55, 0xb3, 0x07, 0xbf, 0x67, 0x1f, 0xb9, 0xc8, 0xe3, 0x6f, 0xc7, 0xa6, 0xbc, 0x5d, 0x98, - 0x82, 0x72, 0x7a, 0x62, 0x25, 0x85, 0x65, 0x3b, 0x2c, 0xb5, 0x71, 0x0e, 0xe8, 0xac, 0x71, 0xab, - 0x3c, 0x31, 0x07, 0x34, 0xce, 0xec, 0xb2, 0x6e, 0x27, 0x5b, 0xe4, 0x15, 0xd3, 0x4e, 0xc6, 0x22, - 0xfb, 0xd0, 0xf4, 0xc2, 0xc1, 0x50, 0x75, 0xc4, 0xb7, 0x7e, 0xe8, 0x2b, 0x5f, 0x86, 0x71, 0xab, - 0x8a, 0xa1, 0xd6, 0xf2, 0x8c, 0xc6, 0x3c, 0xd8, 0xc4, 0x16, 0xfa, 0xa3, 0x03, 0xcb, 0x05, 0x70, - 0xc6, 0xa1, 0x13, 0xbe, 0xa5, 0xf9, 0x7c, 0xdf, 0x4f, 0x07, 0x9c, 0x8b, 0x8e, 0xed, 0x99, 0x6c, - 0xc6, 0xe7, 0xdd, 0xaf, 0x0e, 0xdc, 0x9f, 0xe6, 0x30, 0x95, 0x4d, 0x1b, 0xe0, 0x59, 0xe4, 0xf7, - 0x79, 0x34, 0xfa, 0x42, 0x8c, 0xec, 0xac, 0xcf, 0x21, 0xe4, 0x6b, 0x58, 0x2d, 0xc4, 0xfa, 0xac, - 0x67, 0x24, 0x32, 0xa4, 0x1e, 0xcc, 0x24, 0x65, 0xfc, 0xd8, 0x8c, 0xed, 0xf4, 0x1f, 0x07, 0x5e, - 0x9b, 0xba, 0x94, 0xd5, 0xa3, 0x93, 0x2f, 0xfd, 0x6d, 0x68, 0x3e, 0xd7, 0xa3, 0xa2, 0x23, 0x62, - 0xe5, 0x87, 0x5c, 0x7b, 0xda, 0x82, 0x9d, 0xc0, 0x89, 0x07, 0x35, 0xc4, 0x0e, 0xf9, 0xc0, 0xd2, - 0x7c, 0xeb, 0x06, 0x9a, 0x3b, 0x89, 0xbf, 0x99, 0x69, 0xe9, 0x76, 0x4d, 0x06, 0xa7, 0x6e, 0x32, - 0xc2, 0xd1, 0x58, 0xff, 0x18, 0x1a, 0x63, 0x1b, 0x6e, 0x35, 0xe7, 0x24, 0x6c, 0x24, 0xb3, 0x65, - 0x8c, 0xc9, 0xfc, 0x2e, 0xfd, 0x10, 0x20, 0x73, 0xb5, 0x03, 0x60, 0x4e, 0x7d, 0xe6, 0x9c, 0xe9, - 0x01, 0x6c, 0x24, 0x83, 0xef, 0x16, 0x09, 0x93, 0x6a, 0x29, 0x65, 0xd5, 0x42, 0xf7, 0xc1, 0x3d, - 0x66, 0x1e, 0x76, 0x52, 0xef, 0x85, 0x48, 0xaf, 0xc8, 0x5a, 0x7a, 0xcb, 0x81, 0x8c, 0x55, 0xb2, - 0x45, 0x7f, 0x6b, 0xec, 0x99, 0x8c, 0x14, 0x32, 0x6e, 0x30, 0xfc, 0xa6, 0x3f, 0x39, 0x00, 0x4f, - 0xe5, 0xa9, 0xe8, 0x2a, 0xae, 0x86, 0x31, 0x79, 0x80, 0x51, 0x31, 0xd6, 0xe2, 0x6e, 0x23, 0x3b, - 0xd3, 0x31, 0xf3, 0x18, 0xe6, 0xd3, 0xd3, 0x5e, 0x71, 0x95, 0x4e, 0x28, 0x34, 0xc8, 0x9b, 0xb0, - 0x80, 0x4c, 0x45, 0x52, 0x8b, 0xcb, 0x85, 0x01, 0xc2, 0x92, 0x75, 0xf2, 0x3a, 0x54, 0x8f, 0x99, - 0xd7, 0x15, 0xca, 0xce, 0x88, 0x42, 0x12, 0xbb, 0x48, 0x9f, 0x40, 0x63, 0x2f, 0x18, 0xc6, 0x4a, - 0x44, 0x96, 0x59, 0x9a, 0xd8, 0xc9, 0x27, 0xce, 0xa2, 0x95, 0xe6, 0x45, 0xeb, 0x42, 0x65, 0x76, - 0xdf, 0x11, 0x28, 0xe3, 0xd3, 0xc9, 0x4a, 0x85, 0xaf, 0xa6, 0x26, 0xb8, 0x87, 0xbe, 0xb9, 0x5b, - 0x97, 0xe9, 0x4f, 0x44, 0xf8, 0x15, 0xd6, 0x9e, 0x46, 0xb8, 0xfe, 0x61, 0x5a, 0x31, 0x77, 0xa9, - 0x9f, 0x0d, 0x77, 0xf9, 0x09, 0x49, 0x5e, 0x1f, 0x6e, 0xee, 0xf5, 0xf1, 0x9b, 0x03, 0x2b, 0x4c, - 0xc4, 0xfe, 0x4b, 0xe1, 0x85, 0xb1, 0x8a, 0x86, 0x69, 0x1f, 0x7e, 0x2e, 0x4f, 0xbc, 0x0e, 0x46, - 0x75, 0x99, 0x31, 0x92, 0xcb, 0x2a, 0xcd, 0xbc, 0xac, 0xb7, 0xf5, 0x7b, 0x55, 0x46, 0xa7, 0xba, - 0x19, 0x65, 0x64, 0x2b, 0xb5, 0xe0, 0x98, 0xf7, 0x20, 0xef, 0xc0, 0x42, 0x57, 0x0e, 0xa3, 0x5e, - 0x3a, 0xc1, 0x57, 0x33, 0x67, 0xc3, 0xca, 0x2c, 0xb3, 0xc4, 0x8d, 0xfe, 0xe0, 0xc0, 0xbd, 0xfc, - 0xca, 0xff, 0xaa, 0x20, 0xa3, 0x50, 0x69, 0xaa, 0x42, 0xee, 0x34, 0x85, 0xca, 0x99, 0x42, 0xd9, - 0x7b, 0xa3, 0x92, 0x7b, 0x6f, 0x50, 0x06, 0x6b, 0x13, 0xb2, 0xed, 0xc9, 0xfe, 0x40, 0xdf, 0xcf, - 0x1d, 0xe5, 0xa3, 0xef, 0x42, 0xed, 0x48, 0x0e, 0x64, 0x20, 0xcf, 0x46, 0xb9, 0x42, 0x73, 0xe6, - 0x14, 0xda, 0xa3, 0xe6, 0xef, 0xd7, 0x6d, 0xe7, 0x8f, 0xeb, 0xb6, 0xf3, 0xe7, 0x75, 0xdb, 0xf9, - 0xf9, 0xaf, 0xf6, 0x2b, 0x27, 0x55, 0xfc, 0x47, 0xf1, 0xde, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, - 0xd7, 0x59, 0xea, 0x61, 0x62, 0x0c, 0x00, 0x00, + // 1137 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0x8e, 0x6b, 0x3f, 0xc7, 0x8d, 0x33, 0x94, 0xc8, 0x89, 0x22, 0xd7, 0x8c, 0x04, + 0x0d, 0x95, 0x08, 0x34, 0x95, 0x10, 0x04, 0x21, 0x41, 0xe3, 0x54, 0x5d, 0x68, 0x4a, 0x19, 0x27, + 0x45, 0xe2, 0x80, 0x34, 0xb1, 0x87, 0x74, 0x95, 0xf5, 0x8e, 0xd9, 0x1d, 0x27, 0x71, 0x0f, 0xdc, + 0xe0, 0x00, 0x5f, 0x80, 0x3b, 0x67, 0xbe, 0x07, 0x47, 0x3e, 0x02, 0x0a, 0x1f, 0x02, 0x89, 0x0b, + 0x68, 0xde, 0xce, 0xec, 0xae, 0xff, 0xc5, 0x4a, 0x6e, 0xfb, 0xde, 0xbc, 0xf7, 0xe6, 0x37, 0xbf, + 0xf7, 0x67, 0x66, 0xa1, 0x36, 0x88, 0xfc, 0x33, 0xae, 0xc4, 0xf6, 0x20, 0x92, 0x4a, 0x92, 0xb2, + 0x1f, 0x2a, 0x11, 0x85, 0x3c, 0xa0, 0x5f, 0x42, 0xc5, 0x0b, 0x7b, 0xe2, 0xe2, 0x40, 0x28, 0x4e, + 0x5a, 0x50, 0xdd, 0x93, 0xc1, 0xb0, 0x1f, 0x3e, 0xe5, 0xc7, 0x22, 0x68, 0x38, 0x2d, 0x67, 0xab, + 0xc2, 0xf2, 0x2a, 0x6d, 0x71, 0xe8, 0xf7, 0xc5, 0x57, 0x43, 0x1e, 0xaa, 0x61, 0xbf, 0x51, 0x48, + 0x2c, 0x72, 0x2a, 0xfa, 0xaf, 0x03, 0x95, 0xc7, 0x11, 0xef, 0x0b, 0x8c, 0xb8, 0x01, 0x65, 0x26, + 0xcf, 0xf3, 0xe1, 0x52, 0x99, 0xbc, 0x0d, 0xb7, 0xbd, 0xf0, 0x4c, 0x44, 0xb1, 0xd8, 0x0f, 0xf9, + 0x71, 0x20, 0x7a, 0x18, 0xae, 0xcc, 0x26, 0xb4, 0x64, 0x13, 0x2a, 0x7b, 0xbc, 0xfb, 0x52, 0x1c, + 0x8e, 0x06, 0xa2, 0xe1, 0x62, 0x90, 0x4c, 0x91, 0xae, 0x76, 0xfc, 0x57, 0xa2, 0x51, 0x6c, 0x39, + 0x5b, 0x35, 0x96, 0x29, 0x26, 0xf1, 0x2e, 0x4d, 0xe1, 0x25, 0x14, 0x96, 0x19, 0x0f, 0x4f, 0x52, + 0x0c, 0x25, 0xc4, 0x30, 0xa6, 0x23, 0xf7, 0xa0, 0xf4, 0xd8, 0x17, 0x41, 0x2f, 0x6e, 0xdc, 0x6a, + 0xb9, 0x5b, 0xd5, 0x9d, 0x95, 0x6d, 0xcb, 0xdf, 0x36, 0xea, 0x99, 0x59, 0xa6, 0x14, 0x6e, 0x7b, + 0xfd, 0x81, 0x8c, 0x14, 0x13, 0xf1, 0x40, 0x86, 0xb1, 0x20, 0x75, 0x70, 0xf7, 0xa3, 0xc8, 0x9c, + 0x5d, 0x7f, 0xd2, 0x1f, 0xa0, 0xfe, 0x28, 0x90, 0xdd, 0xd3, 0x36, 0x57, 0x9c, 0x89, 0xef, 0x87, + 0x22, 0x56, 0xe4, 0x0e, 0x2c, 0x61, 0x16, 0x8c, 0x5d, 0x22, 0x68, 0x2d, 0x32, 0x69, 0x68, 0x4e, + 0x04, 0xad, 0x45, 0x7f, 0xa4, 0xa2, 0xc8, 0x12, 0x41, 0x6b, 0x3b, 0x81, 0xdf, 0x4d, 0x28, 0x28, + 0xb2, 0x44, 0x20, 0x04, 0x8a, 0x2f, 0x7c, 0x71, 0x6e, 0xce, 0x8d, 0xdf, 0xd4, 0x83, 0xd5, 0xdc, + 0xfe, 0x06, 0xe6, 0x1a, 0x94, 0x98, 0x3c, 0xf7, 0xda, 0x71, 0xc3, 0x69, 0xb9, 0x5b, 0x45, 0x66, + 0x24, 0x64, 0x17, 0xd3, 0xaf, 0x97, 0x0a, 0xb8, 0x94, 0x29, 0xe8, 0x3a, 0x2c, 0x21, 0xd5, 0xfa, + 0x94, 0x99, 0xaf, 0xfe, 0xa4, 0xff, 0x39, 0x50, 0x39, 0xe0, 0x17, 0x08, 0x23, 0x26, 0x9f, 0x40, + 0xb9, 0xa3, 0x78, 0xd8, 0xe3, 0x51, 0x0f, 0x8d, 0xaa, 0x3b, 0x6f, 0x66, 0x14, 0xa6, 0x66, 0xdb, + 0xd6, 0x66, 0x3f, 0x54, 0xd1, 0x88, 0xa5, 0x2e, 0x64, 0x17, 0x6e, 0x99, 0x9a, 0x40, 0x0c, 0xd5, + 0x9d, 0xd6, 0x2c, 0xef, 0xb4, 0x6c, 0xb4, 0xb3, 0x75, 0xd8, 0xf8, 0x18, 0x6a, 0x63, 0x61, 0x35, + 0xd6, 0x53, 0x31, 0xb2, 0x19, 0x39, 0x15, 0x23, 0xcd, 0xdd, 0x19, 0x0f, 0x86, 0x09, 0xcf, 0x45, + 0x96, 0x08, 0xbb, 0x85, 0x0f, 0x9d, 0x8d, 0x5d, 0x58, 0xce, 0x47, 0xbd, 0x8e, 0x2f, 0xfd, 0x16, + 0xc8, 0x5e, 0x24, 0xb8, 0x12, 0x08, 0xef, 0x40, 0xc4, 0x31, 0x3f, 0x11, 0xf3, 0x33, 0x9d, 0x64, + 0xaf, 0x90, 0xcf, 0xde, 0x26, 0x54, 0xbc, 0xd8, 0x1e, 0xdc, 0xc5, 0xba, 0xcc, 0x14, 0xf4, 0x3e, + 0x90, 0xb6, 0x08, 0x84, 0x12, 0xa6, 0x7f, 0xaf, 0x88, 0x4f, 0x3b, 0x16, 0xcb, 0x62, 0x5b, 0x72, + 0x0f, 0x8a, 0xba, 0x75, 0x11, 0x4a, 0x75, 0xe7, 0xf5, 0x8c, 0xe9, 0x74, 0x4e, 0x30, 0x34, 0xa0, + 0xbe, 0x0d, 0x6a, 0xda, 0x7d, 0xc1, 0x01, 0x67, 0x94, 0xb2, 0xdd, 0xca, 0x9d, 0xdc, 0x2a, 0x1d, + 0x20, 0x66, 0xab, 0x4f, 0xed, 0x59, 0x6f, 0xba, 0x15, 0xfd, 0xc6, 0x68, 0x75, 0x4b, 0x3c, 0xd3, + 0xab, 0x89, 0x0f, 0x7e, 0xcf, 0x3f, 0xf2, 0x04, 0x0e, 0x1d, 0x5b, 0xf7, 0x50, 0xdc, 0x70, 0x5b, + 0xae, 0x8e, 0x8d, 0x02, 0x7d, 0x08, 0xa5, 0x4e, 0xf7, 0xa5, 0xe8, 0x73, 0xf2, 0x8e, 0x2e, 0xd4, + 0x9e, 0xb8, 0x10, 0xb1, 0x29, 0xf3, 0x95, 0x09, 0xfa, 0x98, 0x5d, 0xa7, 0xbf, 0x38, 0x06, 0xfd, + 0x1c, 0x44, 0x25, 0xdc, 0x3b, 0x6e, 0x14, 0xa7, 0x26, 0x8e, 0xd6, 0x33, 0xb3, 0x4c, 0xf6, 0xa1, + 0xee, 0x85, 0x83, 0xa1, 0x6a, 0x8b, 0xef, 0xfc, 0xd0, 0x57, 0xbe, 0x0c, 0xe3, 0x46, 0x09, 0x5d, + 0xd6, 0xf3, 0x5b, 0x8f, 0x59, 0xb0, 0x29, 0x17, 0xfa, 0x93, 0x03, 0x2b, 0x13, 0xca, 0x05, 0xb8, + 0x0a, 0x57, 0xe3, 0xfa, 0x20, 0x1d, 0x99, 0x2e, 0x1a, 0x36, 0xe7, 0xa2, 0x19, 0x9f, 0xa0, 0xbf, + 0x39, 0x70, 0x67, 0x96, 0xc1, 0x4c, 0x34, 0x4d, 0x80, 0xe7, 0x91, 0xdf, 0xe7, 0xd1, 0xe8, 0x0b, + 0x31, 0x32, 0xb7, 0x47, 0x4e, 0x43, 0xbe, 0x86, 0xb5, 0x89, 0x58, 0x9f, 0x75, 0x13, 0x8a, 0x12, + 0x50, 0x77, 0xe7, 0x82, 0x4a, 0xec, 0xd8, 0x1c, 0x77, 0xfa, 0x8f, 0x03, 0x6f, 0xcc, 0x5c, 0xca, + 0xaa, 0xcf, 0xc9, 0x17, 0xfa, 0x7d, 0xa8, 0xbf, 0xd0, 0x83, 0xa1, 0x2d, 0x62, 0xe5, 0x87, 0x5c, + 0x5b, 0x9a, 0xf2, 0x9c, 0xd2, 0x13, 0x0f, 0xca, 0xa8, 0x3b, 0xe0, 0x03, 0x03, 0xf3, 0xdd, 0x05, + 0x30, 0xb7, 0xad, 0xbd, 0x99, 0x9b, 0x56, 0xd4, 0x60, 0x70, 0x8e, 0xdb, 0x4b, 0x01, 0x05, 0x3d, + 0x11, 0xc7, 0x1c, 0xae, 0x35, 0xd5, 0x24, 0x6c, 0xda, 0x49, 0x32, 0x86, 0xe4, 0xea, 0x9e, 0xfc, + 0x08, 0x20, 0x33, 0x35, 0xed, 0x7e, 0x45, 0x7d, 0xe6, 0x8c, 0xe9, 0x13, 0xd8, 0xb4, 0x63, 0xee, + 0x1a, 0x1b, 0xda, 0x6a, 0x29, 0x64, 0xd5, 0x42, 0xf7, 0xc1, 0x3d, 0x62, 0x9e, 0xbe, 0xea, 0xb0, + 0x5b, 0x6d, 0x8a, 0x8c, 0xa4, 0x5d, 0x9e, 0xc8, 0x58, 0x59, 0x17, 0xfd, 0xad, 0x75, 0xcf, 0x65, + 0xa4, 0x10, 0x71, 0x8d, 0xe1, 0x37, 0xfd, 0xd9, 0x01, 0x78, 0x26, 0x7b, 0xa2, 0xa3, 0xb8, 0x1a, + 0xc6, 0xe4, 0x2e, 0x46, 0xc5, 0x58, 0xd5, 0x9d, 0x5a, 0x76, 0xa6, 0x23, 0xe6, 0x31, 0xdc, 0xef, + 0x41, 0xee, 0x22, 0x9c, 0x9e, 0x30, 0xe9, 0x12, 0xcb, 0x5d, 0x97, 0x5b, 0x76, 0xa0, 0x18, 0xaa, + 0xea, 0x99, 0x7d, 0xa2, 0x37, 0xa0, 0x39, 0x7d, 0x0a, 0xb5, 0xbd, 0x60, 0x18, 0x2b, 0x11, 0x19, + 0x38, 0xfa, 0x26, 0x51, 0x5c, 0xa5, 0xf5, 0x87, 0x02, 0x79, 0x0b, 0x4a, 0x47, 0xcc, 0xeb, 0x08, + 0x65, 0xda, 0x76, 0x02, 0xa7, 0x59, 0xa4, 0x1d, 0x58, 0x9a, 0xdf, 0x6c, 0x04, 0x8a, 0xf8, 0x02, + 0x33, 0xfc, 0xe0, 0xe3, 0xab, 0x0e, 0xee, 0x81, 0x9f, 0x24, 0xd4, 0x65, 0xfa, 0x13, 0x35, 0xfc, + 0x02, 0x0b, 0x4e, 0x6b, 0xb8, 0xbe, 0x7b, 0x56, 0x93, 0x04, 0xea, 0x61, 0x79, 0x93, 0x5b, 0xc2, + 0x3e, 0x62, 0xdc, 0xdc, 0x23, 0xe6, 0x77, 0x07, 0x56, 0x99, 0x88, 0xfd, 0x57, 0xc2, 0x0b, 0x63, + 0x15, 0x0d, 0xd3, 0xe6, 0xfb, 0x5c, 0x1e, 0x7b, 0x6d, 0x8c, 0xea, 0xb2, 0x44, 0xb0, 0x19, 0x2a, + 0xcc, 0xcd, 0xd0, 0x7b, 0xfa, 0xd9, 0x2b, 0xa3, 0x9e, 0xee, 0x40, 0x19, 0x19, 0xce, 0x27, 0x0c, + 0xf3, 0x16, 0xe4, 0x7d, 0xb8, 0xd5, 0x91, 0xc3, 0xa8, 0x9b, 0x8e, 0xe7, 0xb5, 0xcc, 0x38, 0x41, + 0x95, 0x2c, 0x33, 0x6b, 0x46, 0x7f, 0x74, 0x60, 0x39, 0xbf, 0xb2, 0xb8, 0x6c, 0x52, 0x86, 0x0a, + 0x33, 0x19, 0x72, 0x67, 0x31, 0x54, 0xcc, 0x18, 0xca, 0x9e, 0x14, 0x4b, 0xb9, 0x27, 0x05, 0x65, + 0xb0, 0x3e, 0x45, 0xdb, 0x9e, 0xec, 0x0f, 0x74, 0x7e, 0x6e, 0x48, 0x1f, 0x7d, 0x00, 0xe5, 0x43, + 0x39, 0x90, 0x81, 0x3c, 0x19, 0xe5, 0x0a, 0xcd, 0xb9, 0xa2, 0xd0, 0x1e, 0xd5, 0xff, 0xb8, 0x6c, + 0x3a, 0x7f, 0x5e, 0x36, 0x9d, 0xbf, 0x2e, 0x9b, 0xce, 0xaf, 0x7f, 0x37, 0x5f, 0x3b, 0x2e, 0xe1, + 0x8f, 0xc9, 0xc3, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x75, 0xc6, 0x9e, 0xa9, 0x0c, 0x00, + 0x00, } diff --git a/internal/private.proto b/internal/private.proto index fdb5fe082..5101a6c35 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -38,8 +38,13 @@ message Cache { repeated uint64 IDs = 1; } -message MaxSlicesResponse { - map MaxSlices = 1; +//message MaxSlicesResponse { +// map MaxSlices = 1; +//} + +message MaxSlices { + map Standard = 1; + map Inverse = 2; } message CreateSliceMessage { @@ -71,14 +76,16 @@ message DeleteFrameMessage { message Frame { string Name = 1; FrameMeta Meta = 2; + repeated string Views = 3; +} + +message Schema { + repeated Index Indexes = 1; } message Index { string Name = 1; - IndexMeta Meta = 2; - uint64 MaxSlice = 3; repeated Frame Frames = 4; - repeated uint64 Slices = 5; repeated InputDefinition InputDefinitions = 6; } @@ -120,9 +127,8 @@ message URI { message NodeStatus { URI URI = 1; - string State = 2; - repeated Index Indexes = 3; - repeated URI URISet = 4; + MaxSlices MaxSlices = 2; + Schema Schema = 3; } message ClusterStatus { diff --git a/server.go b/server.go index 4e4a507fd..3139300c6 100644 --- a/server.go +++ b/server.go @@ -19,11 +19,9 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net" "net/http" - "net/url" "os" "os/exec" "runtime" @@ -40,7 +38,6 @@ import ( // Default server settings. const ( DefaultAntiEntropyInterval = 10 * time.Minute - DefaultPollingInterval = 60 * time.Second ) // Server represents a holder wrapped by a running HTTP server. @@ -65,7 +62,6 @@ type Server struct { // Background monitoring intervals. AntiEntropyInterval time.Duration - PollingInterval time.Duration MetricInterval time.Duration // TLS configuration @@ -92,7 +88,6 @@ func NewServer() *Server { Network: "tcp", AntiEntropyInterval: DefaultAntiEntropyInterval, - PollingInterval: DefaultPollingInterval, MetricInterval: 0, LogOutput: os.Stderr, @@ -198,9 +193,8 @@ func (s *Server) Open() error { /* // Start background monitoring. - s.wg.Add(3) + s.wg.Add(2) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() */ @@ -275,39 +269,6 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } -// monitorMaxSlices periodically pulls the highest slice from each node in the cluster. -func (s *Server) monitorMaxSlices() { - ticker := time.NewTicker(s.PollingInterval) - defer ticker.Stop() - - for { - select { - case <-s.closing: - return - case <-ticker.C: - } - - oldmaxslices := s.Holder.MaxSlices() - for _, node := range s.Cluster.Nodes { - if s.URI != node.URI { - maxSlices, _ := s.checkMaxSlices(node.URI) - for index, newmax := range maxSlices { - // if we don't know about an index locally, log an error because - // indexes should be created and synced prior to slice creation - if localIndex := s.Holder.Index(index); localIndex != nil { - if newmax > oldmaxslices[index] { - oldmaxslices[index] = newmax - localIndex.SetRemoteMaxSlice(newmax) - } - } else { - s.Logger().Printf("Local Index not found: %s", index) - } - } - } - } - } -} - // ReceiveMessage represents an implementation of BroadcastHandler. func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { @@ -392,10 +353,16 @@ func (s *Server) State() string { return s.Cluster.State } -// LocalStatus returns the state of the local node as well as the -// holder (indexes/frames) according to the local node. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. // Server implements StatusHandler. +// LocalStatus is used to periodically sync information +// between nodes. Under normal conditions, nodes should +// remain in sync through Broadcast messages. For cases +// where a node fails to receive a Broadcast message, or +// when a new (empty) node needs to get in sync with the +// rest of the cluster, two things are shared via gossip: +// - MaxSlice/MaxInverseSlice by Index +// - Schema +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalStatus() (proto.Message, error) { if s.Cluster == nil { return nil, errors.New("Server.Cluster is nil") @@ -405,61 +372,66 @@ func (s *Server) LocalStatus() (proto.Message, error) { } ns := internal.NodeStatus{ - URI: encodeURI(s.URI), - State: s.State(), - Indexes: EncodeIndexes(s.Holder.Indexes()), - URISet: encodeURIs(s.Cluster.URISet()), - } - - // TODO: get rid of this - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI) + URI: encodeURI(s.URI), + MaxSlices: s.Holder.EncodeMaxSlices(), + Schema: s.Holder.EncodeSchema(), } return &ns, nil } -// ClusterStatus returns the NodeState for all nodes in the cluster. +// ClusterStatus returns the ClusterState and URISet for the cluster. func (s *Server) ClusterStatus() (proto.Message, error) { - // Update local Node.state. - ns, err := s.LocalStatus() - if err != nil { - return nil, err - } - localNode := s.Cluster.localNode() - localNode.SetStatus(ns.(*internal.NodeStatus)) - return s.Cluster.Status(), nil } -// HandleRemoteStatus receives incoming NodeState from remote nodes. +// HandleRemoteStatus receives incoming NodeStatus from remote nodes. func (s *Server) HandleRemoteStatus(pb proto.Message) error { return s.mergeRemoteStatus(pb.(*internal.NodeStatus)) } func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { - // Ignore status updates from self. if s.URI == decodeURI(ns.URI) { return nil } - fmt.Printf("mergeRemoteStatus on (%s) from (%s)\n", s.URI, ns.URI) - // Update Node.state. - // Node can be nil if a merge occurs (via gossip) before the coordinator has - // a chance to broadcast the existence of the node. - uri := decodeURI(ns.URI) - if node := s.Cluster.NodeByURI(uri); node != nil { - node.SetStatus(ns) + // Sync maxSlices (standard). + oldmaxslices := s.Holder.MaxSlices() + for index, newMax := range ns.MaxSlices.Standard { + localIndex := s.Holder.Index(index) + // if we don't know about an index locally, log an error because + // indexes should be created and synced prior to slice creation + if localIndex == nil { + s.Logger().Printf("Local Index not found: %s", index) + continue + } + if newMax > oldmaxslices[index] { + oldmaxslices[index] = newMax + localIndex.SetRemoteMaxSlice(newMax) + } } - // Create indexes that don't exist. - for _, index := range ns.Indexes { - opt := IndexOptions{ - ColumnLabel: index.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(index.Meta.TimeQuantum), + // Sync maxSlices (inverse). + oldMaxInverseSlices := s.Holder.MaxInverseSlices() + for index, newMaxInverse := range ns.MaxSlices.Inverse { + localIndex := s.Holder.Index(index) + // if we don't know about an index locally, log an error because + // indexes should be created and synced prior to slice creation + if localIndex == nil { + s.Logger().Printf("Local Index not found: %s", index) + continue } + if newMaxInverse > oldMaxInverseSlices[index] { + oldMaxInverseSlices[index] = newMaxInverse + localIndex.SetRemoteMaxSlice(newMaxInverse) + } + } + + // Sync schema. + // Create indexes that don't exist. + for _, index := range ns.Schema.Indexes { + opt := IndexOptions{} idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) if err != nil { return err @@ -472,55 +444,12 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return err } } + // TODO: Create inputDefinitions that don't exist. } return nil } -func (s *Server) checkMaxSlices(uri URI) (map[string]uint64, error) { - // Create HTTP request. - req, err := http.NewRequest("GET", (&url.URL{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - Path: "/slices/max", - }).String(), nil) - - if err != nil { - return nil, err - } - - // Require protobuf encoding. - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := s.defaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // Read response into buffer. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - // Check status code. - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("invalid status checkMaxSlices: code=%d, err=%s, req=%v", resp.StatusCode, body, req) - } - - // Decode response object. - pb := internal.MaxSlicesResponse{} - - if err = proto.Unmarshal(body, &pb); err != nil { - return nil, err - } - - return pb.MaxSlices, nil -} - // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { // Disable metrics when poll interval is zero diff --git a/server/server.go b/server/server.go index 10220c1d4..34d397fcf 100644 --- a/server/server.go +++ b/server/server.go @@ -124,19 +124,6 @@ func (m *Command) SetupServer() error { cluster.ReplicaN = m.Config.Cluster.ReplicaN cluster.IndexReporter = m.Server.Holder - /* - // TODO travis: get rid of this URI code - for _, address := range m.Config.Cluster.Hosts { - uri, err := pilosa.NewURIFromAddress(address) - if err != nil { - return err - } - cluster.Nodes = append(cluster.Nodes, &pilosa.Node{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - }) - } - */ m.Server.Cluster = cluster // Setup logging output. From f49b2f9b99df6cc8ad7cbbe87d98e89fbab744c3 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 09:02:46 -0500 Subject: [PATCH 006/311] Adjust /status and /schema endpoints. Tried/failed to deprecate /slices/max (client is using it for backups). --- client.go | 10 ++++----- handler.go | 46 +++++++++++++++++------------------------- handler_test.go | 6 +++--- internal/private.proto | 4 ---- 4 files changed, 26 insertions(+), 40 deletions(-) diff --git a/client.go b/client.go index a579e527f..3b63895ec 100644 --- a/client.go +++ b/client.go @@ -100,9 +100,6 @@ func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. u := uriPathToURL(c.host, "/slices/max") - u.RawQuery = (&url.Values{ - "inverse": {strconv.FormatBool(inverse)}, - }).Encode() // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -119,14 +116,17 @@ func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string] } defer resp.Body.Close() - var rsp sliceMaxResponse + var rsp getSlicesMaxResponse if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("http: status=%d", resp.StatusCode) } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { return nil, fmt.Errorf("json decode: %s", err) } - return rsp.MaxSlices, nil + if inverse { + return rsp.Inverse, nil + } + return rsp.Standard, nil } // Schema returns all index and frame schema information. diff --git a/handler.go b/handler.go index ffe189e35..5c7aee509 100644 --- a/handler.go +++ b/handler.go @@ -132,7 +132,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - //router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") // TODO: this is being used by the client (for backups) + router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -216,13 +216,16 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status, err := h.StatusHandler.ClusterStatus() + pb, err := h.StatusHandler.ClusterStatus() if err != nil { h.logger().Printf("cluster status error: %s", err) return } + + cs := pb.(*internal.ClusterStatus) if err := json.NewEncoder(w).Encode(getStatusResponse{ - Status: status, + State: cs.State, + URISet: decodeURIs(cs.URISet), }); err != nil { h.logger().Printf("write status response error: %s", err) } @@ -233,7 +236,8 @@ type getSchemaResponse struct { } type getStatusResponse struct { - Status proto.Message `json:"status"` + State string `json:"state"` + URISet []URI `json:"uri-set"` } // handlePostQuery handles /query requests. @@ -305,33 +309,19 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -/* -func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - var ms map[string]uint64 - if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { - ms = h.Holder.MaxInverseSlices() - } else { - ms = h.Holder.MaxSlices() +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.Holder.MaxSlices(), + Inverse: h.Holder.MaxInverseSlices(), + }); err != nil { + h.logger().Printf("write slices-max response error: %s", err) } - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - pb := &internal.MaxSlicesResponse{ - MaxSlices: ms, - } - if buf, err := proto.Marshal(pb); err != nil { - h.logger().Printf("protobuf marshal error: %s", err) - } else if _, err := w.Write(buf); err != nil { - h.logger().Printf("stream write error: %s", err) - } - return - } - json.NewEncoder(w).Encode(sliceMaxResponse{ - MaxSlices: ms, - }) } -*/ -type sliceMaxResponse struct { - MaxSlices map[string]uint64 `json:"maxSlices"` +type getSlicesMaxResponse struct { + Standard map[string]uint64 `json:"standard"` + Inverse map[string]uint64 `json:"inverse"` } // handleGetIndexes handles GET /index request. diff --git a/handler_test.go b/handler_test.go index 0a540d059..05f93c6ab 100644 --- a/handler_test.go +++ b/handler_test.go @@ -147,7 +147,7 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"status":{"State":"NORMAL","URISet":[{"Scheme":"http","Host":"localhost","Port":10101}]}}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","uri-set":[{"scheme":"http","host":"localhost","port":10101}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -172,7 +172,7 @@ func TestHandler_MaxSlices(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -213,7 +213,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } diff --git a/internal/private.proto b/internal/private.proto index 5101a6c35..4658f21be 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -38,10 +38,6 @@ message Cache { repeated uint64 IDs = 1; } -//message MaxSlicesResponse { -// map MaxSlices = 1; -//} - message MaxSlices { map Standard = 1; map Inverse = 2; From 2f6c4509c287952fc06dc49aa99c670bc180b649 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 09:20:09 -0500 Subject: [PATCH 007/311] remove references to cluster.poll-interval --- cmd/root_test.go | 1 - ctl/generate_config.go | 1 - docs/configuration.md | 15 +-------------- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/cmd/root_test.go b/cmd/root_test.go index 93711a5a1..431d3a66f 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -181,7 +181,6 @@ func TestRootCommand_Config(t *testing.T) { bind = "127.0.0.1:10101" [cluster] - poll-interval = "2m0s" replicas = 2 partitions = 128 hosts = [ diff --git a/ctl/generate_config.go b/ctl/generate_config.go index 87f866bb8..151076ca8 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -43,7 +43,6 @@ bind = "localhost:10101" max-writes-per-request = 5000 [cluster] - poll-interval = "2m0s" replicas = 1 hosts = [ "localhost:10101", diff --git a/docs/configuration.md b/docs/configuration.md index 10325b09f..8bc241070 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,10 +27,9 @@ Every command line flag has a corresponding environment variable. The environmen ### Config file -The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file: +The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flag `--cluster.replicas=1` looks like this in the config file: ```toml [cluster] - poll-interval = "2m0s" replicas = 1 ``` @@ -123,18 +122,6 @@ Any flag that has a value that is a comma separated list on the command line bec hosts = ["localhost:10101"] ``` -#### Cluster Poll Interval - -* Description: Polling interval for cluster. -* Flag: `cluster.poll-interval="1m0s"` -* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"` -* Config: - - ```toml - [cluster] - poll-interval = "1m0s" - ``` - #### Cluster Replicas * Description: Number of hosts each piece of data should be stored on. From ed1b4fcc2e1dd5c49e0c792409999a778dec4e59 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 12:31:05 -0500 Subject: [PATCH 008/311] Copy fragment data from source nodes in resize instruction. --- client.go | 7 ++ cluster.go | 94 ++++++++++++++----- cluster_test.go | 1 - gossip/gossip.go | 2 +- handler.go | 6 +- holder.go | 5 - internal/private.pb.go | 204 +++++++++++++++++++++++++---------------- internal/private.proto | 1 + server/server.go | 2 +- 9 files changed, 210 insertions(+), 112 deletions(-) diff --git a/client.go b/client.go index 3b63895ec..6cbee6e26 100644 --- a/client.go +++ b/client.go @@ -682,6 +682,13 @@ func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, sli return nil, fmt.Errorf("unable to connect to any owner") } +func (c *Client) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) { + node := &Node{ + URI: uri, + } + return c.backupSliceNode(ctx, index, frame, view, slice, node) +} + func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ diff --git a/cluster.go b/cluster.go index 8bb9b5ff2..608d06079 100644 --- a/cluster.go +++ b/cluster.go @@ -15,7 +15,9 @@ package pilosa import ( + "context" "encoding/binary" + "errors" "fmt" "hash/fnv" "io" @@ -151,10 +153,10 @@ type Cluster struct { Topology *Topology // Required for cluster Resize. - State string - Coordinator URI - IndexReporter IndexReporter - Broadcaster Broadcaster + State string + Coordinator URI + Holder *Holder + Broadcaster Broadcaster joiningURIs chan URI @@ -511,12 +513,10 @@ func (c *Cluster) Open() error { return fmt.Errorf("considerTopology: %v", err) } // Add the local node to the cluster and update state. - fmt.Println("IS Coord") c.AddHost(c.URI) c.setState(state) } else { // Add the local node to the cluster. - fmt.Println("NOT Coord") c.AddHost(c.URI) } @@ -665,7 +665,7 @@ func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { toCluster.AddNode(addURI) // Add to the ResizeJob the instructions for each index. - for _, idx := range c.IndexReporter.Indexes() { + for _, idx := range c.Holder.Indexes() { // dataDiff is map[string][]*internal.ResizeSource, where string is // a host in toCluster. dataDiff := c.DataDiff(toCluster, idx) @@ -704,22 +704,67 @@ func (c *Cluster) CompleteCurrentJob(state string) { // followResizeInstruction is run by any node that receives a ResizeInstruction. func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { go func() { - // Request each source file in ResizeSources. - for _, src := range instr.Sources { - /************************************************************/ - // TODO travis: get the data files from other nodes. - fmt.Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.URI) - for i := 0; i <= 4; i++ { - fmt.Printf(" %d", i) - time.Sleep(1 * time.Second) - } - fmt.Println("") - /************************************************************/ - } - + // Prepare the return message. complete := &internal.ResizeInstructionComplete{ JobID: instr.JobID, URI: instr.URI, + Error: "", + } + + // Stop processing on any error. + if err := func() error { + // Create a client for calling remote nodes. + client, err := NewClientFromURI(&c.URI, nil) // TODO: ClientOptions + if err != nil { + return err + } + + // Request each source file in ResizeSources. + for _, src := range instr.Sources { + fmt.Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.URI) + + srcURI := decodeURI(src.URI) + + // Retrieve frame. + f := c.Holder.Frame(src.Index, src.Frame) + if f == nil { + return ErrFrameNotFound + } + + // Create view. + v, err := f.CreateViewIfNotExists(src.View) + if err != nil { + return err + } + + // Create the local fragment. + frag, err := v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + + // Stream slice from remote node. + rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + if err != nil { + return err + } else if rd == nil { + return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.URI) + } + + // Write to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + return err + } + } + return nil + }(); err != nil { + complete.Error = err.Error() } node := &Node{ @@ -732,8 +777,15 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { } func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { + j := c.Job(complete.JobID) + // Abort the job if an error exists in the complete object. + if complete.Error != "" { + j.result <- ResizeJobStateAborted + return errors.New(complete.Error) + } + j.mu.Lock() defer j.mu.Unlock() @@ -1031,7 +1083,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { } // If the index does not yet have data, go ahead and add the node. - if !c.IndexReporter.HasData() { + if !c.Holder.HasData() { uri := e.URI if err := c.AddHost(uri); err != nil { return err diff --git a/cluster_test.go b/cluster_test.go index 674be6919..32f48895a 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -261,7 +261,6 @@ func TestCluster_Resize(t *testing.T) { // Cluster 1 c1 := test.NewCluster(3) - c1.IndexReporter = h1 c1.ReplicaN = 2 // Cluster 2 diff --git a/gossip/gossip.go b/gossip/gossip.go index 2cd2e09e0..f4bf3d306 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -148,7 +148,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) g.config.memberlistConfig.AdvertisePort = gossipPort - //g.config.memberlistConfig.PushPullInterval = 15 * time.Second // Default is 15s in DefaultLocalConfig. + g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. // TODO travis: change this from 0 g.config.memberlistConfig.Delegate = g g.config.memberlistConfig.SecretKey = secretKey g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) diff --git a/handler.go b/handler.go index 5c7aee509..b34a616a5 100644 --- a/handler.go +++ b/handler.go @@ -1347,7 +1347,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } } -// handleGetFragmentBackup handles GET /fragment/data requests. +// handleGetFragmentData handles GET /fragment/data requests. func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { // Read slice parameter. q := r.URL.Query() @@ -1370,7 +1370,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } } -// handlePostFragmentRestore handles POST /fragment/data requests. +// handlePostFragmentData handles POST /fragment/data requests. func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) { // Read slice parameter. q := r.URL.Query() @@ -1408,7 +1408,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) } } -// handleGetFragmentData handles GET /fragment/block/data requests. +// handleGetFragmentBlockData handles GET /fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { // Read request object. var req internal.BlockDataRequest diff --git a/holder.go b/holder.go index ae314a274..8da92f498 100644 --- a/holder.go +++ b/holder.go @@ -649,8 +649,3 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err return nil } - -type IndexReporter interface { - HasData() bool - Indexes() []*Index -} diff --git a/internal/private.pb.go b/internal/private.pb.go index 6993a7070..1b96a50a6 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -450,11 +450,8 @@ func (m *Schema) GetIndexes() []*Index { } type Index struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - // IndexMeta Meta = 2; - // uint64 MaxSlice = 3; - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` - // repeated uint64 Slices = 5; + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` InputDefinitions []*InputDefinition `protobuf:"bytes,6,rep,name=InputDefinitions" json:"InputDefinitions,omitempty"` } @@ -889,8 +886,9 @@ func (m *ResizeSource) GetSlice() uint64 { } type ResizeInstructionComplete struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` } func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComplete{} } @@ -914,6 +912,13 @@ func (m *ResizeInstructionComplete) GetURI() *URI { return nil } +func (m *ResizeInstructionComplete) GetError() string { + if m != nil { + return m.Error + } + return "" +} + type Topology struct { URISet []*URI `protobuf:"bytes,1,rep,name=URISet" json:"URISet,omitempty"` } @@ -2105,6 +2110,12 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { } i += n17 } + if len(m.Error) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) + i += copy(dAtA[i:], m.Error) + } return i, nil } @@ -2650,6 +2661,10 @@ func (m *ResizeInstructionComplete) Size() (n int) { l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } + l = len(m.Error) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -6568,6 +6583,35 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -6778,77 +6822,77 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1137 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0x8e, 0x6b, 0x3f, 0xc7, 0x8d, 0x33, 0x94, 0xc8, 0x89, 0x22, 0xd7, 0x8c, 0x04, - 0x0d, 0x95, 0x08, 0x34, 0x95, 0x10, 0x04, 0x21, 0x41, 0xe3, 0x54, 0x5d, 0x68, 0x4a, 0x19, 0x27, - 0x45, 0xe2, 0x80, 0x34, 0xb1, 0x87, 0x74, 0x95, 0xf5, 0x8e, 0xd9, 0x1d, 0x27, 0x71, 0x0f, 0xdc, - 0xe0, 0x00, 0x5f, 0x80, 0x3b, 0x67, 0xbe, 0x07, 0x47, 0x3e, 0x02, 0x0a, 0x1f, 0x02, 0x89, 0x0b, - 0x68, 0xde, 0xce, 0xec, 0xae, 0xff, 0xc5, 0x4a, 0x6e, 0xfb, 0xde, 0xbc, 0xf7, 0xe6, 0x37, 0xbf, - 0xf7, 0x67, 0x66, 0xa1, 0x36, 0x88, 0xfc, 0x33, 0xae, 0xc4, 0xf6, 0x20, 0x92, 0x4a, 0x92, 0xb2, - 0x1f, 0x2a, 0x11, 0x85, 0x3c, 0xa0, 0x5f, 0x42, 0xc5, 0x0b, 0x7b, 0xe2, 0xe2, 0x40, 0x28, 0x4e, - 0x5a, 0x50, 0xdd, 0x93, 0xc1, 0xb0, 0x1f, 0x3e, 0xe5, 0xc7, 0x22, 0x68, 0x38, 0x2d, 0x67, 0xab, - 0xc2, 0xf2, 0x2a, 0x6d, 0x71, 0xe8, 0xf7, 0xc5, 0x57, 0x43, 0x1e, 0xaa, 0x61, 0xbf, 0x51, 0x48, - 0x2c, 0x72, 0x2a, 0xfa, 0xaf, 0x03, 0x95, 0xc7, 0x11, 0xef, 0x0b, 0x8c, 0xb8, 0x01, 0x65, 0x26, - 0xcf, 0xf3, 0xe1, 0x52, 0x99, 0xbc, 0x0d, 0xb7, 0xbd, 0xf0, 0x4c, 0x44, 0xb1, 0xd8, 0x0f, 0xf9, - 0x71, 0x20, 0x7a, 0x18, 0xae, 0xcc, 0x26, 0xb4, 0x64, 0x13, 0x2a, 0x7b, 0xbc, 0xfb, 0x52, 0x1c, - 0x8e, 0x06, 0xa2, 0xe1, 0x62, 0x90, 0x4c, 0x91, 0xae, 0x76, 0xfc, 0x57, 0xa2, 0x51, 0x6c, 0x39, - 0x5b, 0x35, 0x96, 0x29, 0x26, 0xf1, 0x2e, 0x4d, 0xe1, 0x25, 0x14, 0x96, 0x19, 0x0f, 0x4f, 0x52, - 0x0c, 0x25, 0xc4, 0x30, 0xa6, 0x23, 0xf7, 0xa0, 0xf4, 0xd8, 0x17, 0x41, 0x2f, 0x6e, 0xdc, 0x6a, - 0xb9, 0x5b, 0xd5, 0x9d, 0x95, 0x6d, 0xcb, 0xdf, 0x36, 0xea, 0x99, 0x59, 0xa6, 0x14, 0x6e, 0x7b, - 0xfd, 0x81, 0x8c, 0x14, 0x13, 0xf1, 0x40, 0x86, 0xb1, 0x20, 0x75, 0x70, 0xf7, 0xa3, 0xc8, 0x9c, - 0x5d, 0x7f, 0xd2, 0x1f, 0xa0, 0xfe, 0x28, 0x90, 0xdd, 0xd3, 0x36, 0x57, 0x9c, 0x89, 0xef, 0x87, - 0x22, 0x56, 0xe4, 0x0e, 0x2c, 0x61, 0x16, 0x8c, 0x5d, 0x22, 0x68, 0x2d, 0x32, 0x69, 0x68, 0x4e, - 0x04, 0xad, 0x45, 0x7f, 0xa4, 0xa2, 0xc8, 0x12, 0x41, 0x6b, 0x3b, 0x81, 0xdf, 0x4d, 0x28, 0x28, - 0xb2, 0x44, 0x20, 0x04, 0x8a, 0x2f, 0x7c, 0x71, 0x6e, 0xce, 0x8d, 0xdf, 0xd4, 0x83, 0xd5, 0xdc, - 0xfe, 0x06, 0xe6, 0x1a, 0x94, 0x98, 0x3c, 0xf7, 0xda, 0x71, 0xc3, 0x69, 0xb9, 0x5b, 0x45, 0x66, - 0x24, 0x64, 0x17, 0xd3, 0xaf, 0x97, 0x0a, 0xb8, 0x94, 0x29, 0xe8, 0x3a, 0x2c, 0x21, 0xd5, 0xfa, - 0x94, 0x99, 0xaf, 0xfe, 0xa4, 0xff, 0x39, 0x50, 0x39, 0xe0, 0x17, 0x08, 0x23, 0x26, 0x9f, 0x40, - 0xb9, 0xa3, 0x78, 0xd8, 0xe3, 0x51, 0x0f, 0x8d, 0xaa, 0x3b, 0x6f, 0x66, 0x14, 0xa6, 0x66, 0xdb, - 0xd6, 0x66, 0x3f, 0x54, 0xd1, 0x88, 0xa5, 0x2e, 0x64, 0x17, 0x6e, 0x99, 0x9a, 0x40, 0x0c, 0xd5, - 0x9d, 0xd6, 0x2c, 0xef, 0xb4, 0x6c, 0xb4, 0xb3, 0x75, 0xd8, 0xf8, 0x18, 0x6a, 0x63, 0x61, 0x35, - 0xd6, 0x53, 0x31, 0xb2, 0x19, 0x39, 0x15, 0x23, 0xcd, 0xdd, 0x19, 0x0f, 0x86, 0x09, 0xcf, 0x45, - 0x96, 0x08, 0xbb, 0x85, 0x0f, 0x9d, 0x8d, 0x5d, 0x58, 0xce, 0x47, 0xbd, 0x8e, 0x2f, 0xfd, 0x16, - 0xc8, 0x5e, 0x24, 0xb8, 0x12, 0x08, 0xef, 0x40, 0xc4, 0x31, 0x3f, 0x11, 0xf3, 0x33, 0x9d, 0x64, - 0xaf, 0x90, 0xcf, 0xde, 0x26, 0x54, 0xbc, 0xd8, 0x1e, 0xdc, 0xc5, 0xba, 0xcc, 0x14, 0xf4, 0x3e, - 0x90, 0xb6, 0x08, 0x84, 0x12, 0xa6, 0x7f, 0xaf, 0x88, 0x4f, 0x3b, 0x16, 0xcb, 0x62, 0x5b, 0x72, - 0x0f, 0x8a, 0xba, 0x75, 0x11, 0x4a, 0x75, 0xe7, 0xf5, 0x8c, 0xe9, 0x74, 0x4e, 0x30, 0x34, 0xa0, - 0xbe, 0x0d, 0x6a, 0xda, 0x7d, 0xc1, 0x01, 0x67, 0x94, 0xb2, 0xdd, 0xca, 0x9d, 0xdc, 0x2a, 0x1d, - 0x20, 0x66, 0xab, 0x4f, 0xed, 0x59, 0x6f, 0xba, 0x15, 0xfd, 0xc6, 0x68, 0x75, 0x4b, 0x3c, 0xd3, - 0xab, 0x89, 0x0f, 0x7e, 0xcf, 0x3f, 0xf2, 0x04, 0x0e, 0x1d, 0x5b, 0xf7, 0x50, 0xdc, 0x70, 0x5b, - 0xae, 0x8e, 0x8d, 0x02, 0x7d, 0x08, 0xa5, 0x4e, 0xf7, 0xa5, 0xe8, 0x73, 0xf2, 0x8e, 0x2e, 0xd4, - 0x9e, 0xb8, 0x10, 0xb1, 0x29, 0xf3, 0x95, 0x09, 0xfa, 0x98, 0x5d, 0xa7, 0xbf, 0x38, 0x06, 0xfd, - 0x1c, 0x44, 0x25, 0xdc, 0x3b, 0x6e, 0x14, 0xa7, 0x26, 0x8e, 0xd6, 0x33, 0xb3, 0x4c, 0xf6, 0xa1, - 0xee, 0x85, 0x83, 0xa1, 0x6a, 0x8b, 0xef, 0xfc, 0xd0, 0x57, 0xbe, 0x0c, 0xe3, 0x46, 0x09, 0x5d, - 0xd6, 0xf3, 0x5b, 0x8f, 0x59, 0xb0, 0x29, 0x17, 0xfa, 0x93, 0x03, 0x2b, 0x13, 0xca, 0x05, 0xb8, - 0x0a, 0x57, 0xe3, 0xfa, 0x20, 0x1d, 0x99, 0x2e, 0x1a, 0x36, 0xe7, 0xa2, 0x19, 0x9f, 0xa0, 0xbf, - 0x39, 0x70, 0x67, 0x96, 0xc1, 0x4c, 0x34, 0x4d, 0x80, 0xe7, 0x91, 0xdf, 0xe7, 0xd1, 0xe8, 0x0b, - 0x31, 0x32, 0xb7, 0x47, 0x4e, 0x43, 0xbe, 0x86, 0xb5, 0x89, 0x58, 0x9f, 0x75, 0x13, 0x8a, 0x12, - 0x50, 0x77, 0xe7, 0x82, 0x4a, 0xec, 0xd8, 0x1c, 0x77, 0xfa, 0x8f, 0x03, 0x6f, 0xcc, 0x5c, 0xca, - 0xaa, 0xcf, 0xc9, 0x17, 0xfa, 0x7d, 0xa8, 0xbf, 0xd0, 0x83, 0xa1, 0x2d, 0x62, 0xe5, 0x87, 0x5c, - 0x5b, 0x9a, 0xf2, 0x9c, 0xd2, 0x13, 0x0f, 0xca, 0xa8, 0x3b, 0xe0, 0x03, 0x03, 0xf3, 0xdd, 0x05, - 0x30, 0xb7, 0xad, 0xbd, 0x99, 0x9b, 0x56, 0xd4, 0x60, 0x70, 0x8e, 0xdb, 0x4b, 0x01, 0x05, 0x3d, - 0x11, 0xc7, 0x1c, 0xae, 0x35, 0xd5, 0x24, 0x6c, 0xda, 0x49, 0x32, 0x86, 0xe4, 0xea, 0x9e, 0xfc, - 0x08, 0x20, 0x33, 0x35, 0xed, 0x7e, 0x45, 0x7d, 0xe6, 0x8c, 0xe9, 0x13, 0xd8, 0xb4, 0x63, 0xee, - 0x1a, 0x1b, 0xda, 0x6a, 0x29, 0x64, 0xd5, 0x42, 0xf7, 0xc1, 0x3d, 0x62, 0x9e, 0xbe, 0xea, 0xb0, - 0x5b, 0x6d, 0x8a, 0x8c, 0xa4, 0x5d, 0x9e, 0xc8, 0x58, 0x59, 0x17, 0xfd, 0xad, 0x75, 0xcf, 0x65, - 0xa4, 0x10, 0x71, 0x8d, 0xe1, 0x37, 0xfd, 0xd9, 0x01, 0x78, 0x26, 0x7b, 0xa2, 0xa3, 0xb8, 0x1a, - 0xc6, 0xe4, 0x2e, 0x46, 0xc5, 0x58, 0xd5, 0x9d, 0x5a, 0x76, 0xa6, 0x23, 0xe6, 0x31, 0xdc, 0xef, - 0x41, 0xee, 0x22, 0x9c, 0x9e, 0x30, 0xe9, 0x12, 0xcb, 0x5d, 0x97, 0x5b, 0x76, 0xa0, 0x18, 0xaa, - 0xea, 0x99, 0x7d, 0xa2, 0x37, 0xa0, 0x39, 0x7d, 0x0a, 0xb5, 0xbd, 0x60, 0x18, 0x2b, 0x11, 0x19, - 0x38, 0xfa, 0x26, 0x51, 0x5c, 0xa5, 0xf5, 0x87, 0x02, 0x79, 0x0b, 0x4a, 0x47, 0xcc, 0xeb, 0x08, - 0x65, 0xda, 0x76, 0x02, 0xa7, 0x59, 0xa4, 0x1d, 0x58, 0x9a, 0xdf, 0x6c, 0x04, 0x8a, 0xf8, 0x02, - 0x33, 0xfc, 0xe0, 0xe3, 0xab, 0x0e, 0xee, 0x81, 0x9f, 0x24, 0xd4, 0x65, 0xfa, 0x13, 0x35, 0xfc, - 0x02, 0x0b, 0x4e, 0x6b, 0xb8, 0xbe, 0x7b, 0x56, 0x93, 0x04, 0xea, 0x61, 0x79, 0x93, 0x5b, 0xc2, - 0x3e, 0x62, 0xdc, 0xdc, 0x23, 0xe6, 0x77, 0x07, 0x56, 0x99, 0x88, 0xfd, 0x57, 0xc2, 0x0b, 0x63, - 0x15, 0x0d, 0xd3, 0xe6, 0xfb, 0x5c, 0x1e, 0x7b, 0x6d, 0x8c, 0xea, 0xb2, 0x44, 0xb0, 0x19, 0x2a, - 0xcc, 0xcd, 0xd0, 0x7b, 0xfa, 0xd9, 0x2b, 0xa3, 0x9e, 0xee, 0x40, 0x19, 0x19, 0xce, 0x27, 0x0c, - 0xf3, 0x16, 0xe4, 0x7d, 0xb8, 0xd5, 0x91, 0xc3, 0xa8, 0x9b, 0x8e, 0xe7, 0xb5, 0xcc, 0x38, 0x41, - 0x95, 0x2c, 0x33, 0x6b, 0x46, 0x7f, 0x74, 0x60, 0x39, 0xbf, 0xb2, 0xb8, 0x6c, 0x52, 0x86, 0x0a, - 0x33, 0x19, 0x72, 0x67, 0x31, 0x54, 0xcc, 0x18, 0xca, 0x9e, 0x14, 0x4b, 0xb9, 0x27, 0x05, 0x65, - 0xb0, 0x3e, 0x45, 0xdb, 0x9e, 0xec, 0x0f, 0x74, 0x7e, 0x6e, 0x48, 0x1f, 0x7d, 0x00, 0xe5, 0x43, - 0x39, 0x90, 0x81, 0x3c, 0x19, 0xe5, 0x0a, 0xcd, 0xb9, 0xa2, 0xd0, 0x1e, 0xd5, 0xff, 0xb8, 0x6c, - 0x3a, 0x7f, 0x5e, 0x36, 0x9d, 0xbf, 0x2e, 0x9b, 0xce, 0xaf, 0x7f, 0x37, 0x5f, 0x3b, 0x2e, 0xe1, - 0x8f, 0xc9, 0xc3, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x75, 0xc6, 0x9e, 0xa9, 0x0c, 0x00, - 0x00, + // 1147 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0x66, 0x3c, 0xb6, 0xd7, 0x2e, 0xc7, 0x1b, 0xa7, 0x09, 0x91, 0x13, 0x45, 0x5e, 0xd3, 0x12, + 0x6c, 0x58, 0x89, 0xc0, 0x66, 0x25, 0x04, 0x41, 0x48, 0xb0, 0xb1, 0x57, 0x3b, 0xb0, 0x59, 0x96, + 0x76, 0xb2, 0x48, 0x1c, 0x90, 0x3a, 0x76, 0x93, 0x8c, 0x62, 0xcf, 0x98, 0x9e, 0x76, 0x12, 0xef, + 0x81, 0x1b, 0x1c, 0xe0, 0x05, 0xb8, 0x73, 0xe6, 0x3d, 0x38, 0xf2, 0x08, 0x28, 0x3c, 0x04, 0x12, + 0x17, 0x50, 0xd7, 0x74, 0xcf, 0x8c, 0xff, 0x12, 0x25, 0xb7, 0xa9, 0xea, 0xaa, 0xea, 0xaf, 0xbf, + 0xfa, 0xe9, 0x1e, 0xa8, 0x0e, 0xa5, 0x7f, 0xc6, 0x95, 0xd8, 0x1e, 0xca, 0x50, 0x85, 0xa4, 0xe4, + 0x07, 0x4a, 0xc8, 0x80, 0xf7, 0xe9, 0x97, 0x50, 0xf6, 0x82, 0x9e, 0xb8, 0xd8, 0x17, 0x8a, 0x93, + 0x26, 0x54, 0xf6, 0xc2, 0xfe, 0x68, 0x10, 0x3c, 0xe3, 0x47, 0xa2, 0x5f, 0x77, 0x9a, 0xce, 0x56, + 0x99, 0x65, 0x55, 0xda, 0xe2, 0xc0, 0x1f, 0x88, 0xaf, 0x46, 0x3c, 0x50, 0xa3, 0x41, 0x3d, 0x17, + 0x5b, 0x64, 0x54, 0xf4, 0x5f, 0x07, 0xca, 0x4f, 0x24, 0x1f, 0x08, 0x8c, 0xb8, 0x01, 0x25, 0x16, + 0x9e, 0x67, 0xc3, 0x25, 0x32, 0x79, 0x1b, 0xee, 0x7a, 0xc1, 0x99, 0x90, 0x91, 0x68, 0x07, 0xfc, + 0xa8, 0x2f, 0x7a, 0x18, 0xae, 0xc4, 0xa6, 0xb4, 0x64, 0x13, 0xca, 0x7b, 0xbc, 0x7b, 0x22, 0x0e, + 0xc6, 0x43, 0x51, 0x77, 0x31, 0x48, 0xaa, 0x48, 0x56, 0x3b, 0xfe, 0x2b, 0x51, 0xcf, 0x37, 0x9d, + 0xad, 0x2a, 0x4b, 0x15, 0xd3, 0x78, 0x0b, 0x33, 0x78, 0x09, 0x85, 0x25, 0xc6, 0x83, 0xe3, 0x04, + 0x43, 0x11, 0x31, 0x4c, 0xe8, 0xc8, 0x7d, 0x28, 0x3e, 0xf1, 0x45, 0xbf, 0x17, 0xd5, 0xef, 0x34, + 0xdd, 0xad, 0xca, 0xce, 0xf2, 0xb6, 0xe5, 0x6f, 0x1b, 0xf5, 0xcc, 0x2c, 0x53, 0x0a, 0x77, 0xbd, + 0xc1, 0x30, 0x94, 0x8a, 0x89, 0x68, 0x18, 0x06, 0x91, 0x20, 0x35, 0x70, 0xdb, 0x52, 0x9a, 0xb3, + 0xeb, 0x4f, 0xfa, 0x03, 0xd4, 0x1e, 0xf7, 0xc3, 0xee, 0x69, 0x8b, 0x2b, 0xce, 0xc4, 0xf7, 0x23, + 0x11, 0x29, 0xb2, 0x0a, 0x05, 0xcc, 0x82, 0xb1, 0x8b, 0x05, 0xad, 0x45, 0x26, 0x0d, 0xcd, 0xb1, + 0xa0, 0xb5, 0xe8, 0x8f, 0x54, 0xe4, 0x59, 0x2c, 0x68, 0x6d, 0xa7, 0xef, 0x77, 0x63, 0x0a, 0xf2, + 0x2c, 0x16, 0x08, 0x81, 0xfc, 0x4b, 0x5f, 0x9c, 0x9b, 0x73, 0xe3, 0x37, 0xf5, 0x60, 0x25, 0xb3, + 0xbf, 0x81, 0xb9, 0x06, 0x45, 0x16, 0x9e, 0x7b, 0xad, 0xa8, 0xee, 0x34, 0xdd, 0xad, 0x3c, 0x33, + 0x12, 0xb2, 0x8b, 0xe9, 0xd7, 0x4b, 0x39, 0x5c, 0x4a, 0x15, 0x74, 0x1d, 0x0a, 0x48, 0xb5, 0x3e, + 0x65, 0xea, 0xab, 0x3f, 0xe9, 0x7f, 0x0e, 0x94, 0xf7, 0xf9, 0x05, 0xc2, 0x88, 0xc8, 0x27, 0x50, + 0xea, 0x28, 0x1e, 0xf4, 0xb8, 0xec, 0xa1, 0x51, 0x65, 0xe7, 0xcd, 0x94, 0xc2, 0xc4, 0x6c, 0xdb, + 0xda, 0xb4, 0x03, 0x25, 0xc7, 0x2c, 0x71, 0x21, 0xbb, 0x70, 0xc7, 0xd4, 0x04, 0x62, 0xa8, 0xec, + 0x34, 0xe7, 0x79, 0x27, 0x65, 0xa3, 0x9d, 0xad, 0xc3, 0xc6, 0xc7, 0x50, 0x9d, 0x08, 0xab, 0xb1, + 0x9e, 0x8a, 0xb1, 0xcd, 0xc8, 0xa9, 0x18, 0x6b, 0xee, 0xce, 0x78, 0x7f, 0x14, 0xf3, 0x9c, 0x67, + 0xb1, 0xb0, 0x9b, 0xfb, 0xd0, 0xd9, 0xd8, 0x85, 0xa5, 0x6c, 0xd4, 0x9b, 0xf8, 0xd2, 0x6f, 0x81, + 0xec, 0x49, 0xc1, 0x95, 0x40, 0x78, 0xfb, 0x22, 0x8a, 0xf8, 0xb1, 0x58, 0x9c, 0xe9, 0x38, 0x7b, + 0xb9, 0x6c, 0xf6, 0x36, 0xa1, 0xec, 0x45, 0xf6, 0xe0, 0x2e, 0xd6, 0x65, 0xaa, 0xa0, 0x0f, 0x80, + 0xb4, 0x44, 0x5f, 0x28, 0x61, 0xfa, 0xf7, 0x8a, 0xf8, 0xb4, 0x63, 0xb1, 0x5c, 0x6f, 0x4b, 0xee, + 0x43, 0x5e, 0xb7, 0x2e, 0x42, 0xa9, 0xec, 0xbc, 0x9e, 0x32, 0x9d, 0xcc, 0x09, 0x86, 0x06, 0xd4, + 0xb7, 0x41, 0x4d, 0xbb, 0x5f, 0x73, 0xc0, 0x39, 0xa5, 0x6c, 0xb7, 0x72, 0xa7, 0xb7, 0x4a, 0x06, + 0x88, 0xd9, 0xea, 0x53, 0x7b, 0xd6, 0xdb, 0x6e, 0x45, 0xbf, 0x31, 0x5a, 0xdd, 0x12, 0xcf, 0xf5, + 0x6a, 0xec, 0x83, 0xdf, 0x8b, 0x8f, 0x3c, 0x85, 0x43, 0xc7, 0xd6, 0x3d, 0x14, 0xd5, 0xdd, 0xa6, + 0xab, 0x63, 0xa3, 0x40, 0x1f, 0x41, 0xb1, 0xd3, 0x3d, 0x11, 0x03, 0x4e, 0xde, 0xd1, 0x85, 0xda, + 0x13, 0x17, 0x22, 0x32, 0x65, 0xbe, 0x3c, 0x45, 0x1f, 0xb3, 0xeb, 0xf4, 0x17, 0xc7, 0xa0, 0x5f, + 0x80, 0xa8, 0x88, 0x7b, 0x47, 0xf5, 0xfc, 0xcc, 0xc4, 0xd1, 0x7a, 0x66, 0x96, 0x49, 0x1b, 0x6a, + 0x5e, 0x30, 0x1c, 0xa9, 0x96, 0xf8, 0xce, 0x0f, 0x7c, 0xe5, 0x87, 0x41, 0x54, 0x2f, 0xa2, 0xcb, + 0x7a, 0x76, 0xeb, 0x09, 0x0b, 0x36, 0xe3, 0x42, 0x7f, 0x72, 0x60, 0x79, 0x4a, 0x79, 0x0d, 0xae, + 0xdc, 0xd5, 0xb8, 0x3e, 0x48, 0x46, 0xa6, 0x8b, 0x86, 0x8d, 0x85, 0x68, 0x26, 0x27, 0xe8, 0x6f, + 0x0e, 0xac, 0xce, 0x33, 0x98, 0x8b, 0xa6, 0x01, 0xf0, 0x42, 0xfa, 0x03, 0x2e, 0xc7, 0x5f, 0x88, + 0xb1, 0xb9, 0x3d, 0x32, 0x1a, 0xf2, 0x35, 0xac, 0x4d, 0xc5, 0xfa, 0xac, 0x1b, 0x53, 0x14, 0x83, + 0xba, 0xb7, 0x10, 0x54, 0x6c, 0xc7, 0x16, 0xb8, 0xd3, 0x7f, 0x1c, 0x78, 0x63, 0xee, 0x52, 0x5a, + 0x7d, 0x4e, 0xb6, 0xd0, 0x1f, 0x40, 0xed, 0xa5, 0x1e, 0x0c, 0x2d, 0x11, 0x29, 0x3f, 0xe0, 0xda, + 0xd2, 0x94, 0xe7, 0x8c, 0x9e, 0x78, 0x50, 0x42, 0xdd, 0x3e, 0x1f, 0x1a, 0x98, 0xef, 0x5e, 0x03, + 0x73, 0xdb, 0xda, 0x9b, 0xb9, 0x69, 0x45, 0x0d, 0x06, 0xe7, 0xb8, 0xbd, 0x14, 0x50, 0xd0, 0x13, + 0x71, 0xc2, 0xe1, 0x46, 0x53, 0x2d, 0x84, 0x4d, 0x3b, 0x49, 0x26, 0x90, 0x5c, 0xdd, 0x93, 0x1f, + 0x01, 0xa4, 0xa6, 0xa6, 0xdd, 0xaf, 0xa8, 0xcf, 0x8c, 0x31, 0x7d, 0x0a, 0x9b, 0x76, 0xcc, 0xdd, + 0x60, 0x43, 0x5b, 0x2d, 0xb9, 0xb4, 0x5a, 0x68, 0x1b, 0xdc, 0x43, 0xe6, 0xe9, 0xab, 0x0e, 0xbb, + 0xd5, 0xa6, 0xc8, 0x48, 0xda, 0xe5, 0x69, 0x18, 0x29, 0xeb, 0xa2, 0xbf, 0xb5, 0xee, 0x45, 0x28, + 0x15, 0x22, 0xae, 0x32, 0xfc, 0xa6, 0x3f, 0x3b, 0x00, 0xcf, 0xc3, 0x9e, 0xe8, 0x28, 0xae, 0x46, + 0x11, 0xb9, 0x87, 0x51, 0x31, 0x56, 0x65, 0xa7, 0x9a, 0x9e, 0xe9, 0x90, 0x79, 0x0c, 0xf7, 0x7b, + 0x98, 0xb9, 0x08, 0x67, 0x27, 0x4c, 0xb2, 0xc4, 0x32, 0xd7, 0xe5, 0x96, 0x1d, 0x28, 0x86, 0xaa, + 0x5a, 0x6a, 0x1f, 0xeb, 0x0d, 0x68, 0x4e, 0x9f, 0x41, 0x75, 0xaf, 0x3f, 0x8a, 0x94, 0x90, 0x06, + 0x8e, 0xbe, 0x49, 0x14, 0x57, 0x49, 0xfd, 0xa1, 0x40, 0xde, 0x82, 0xe2, 0x21, 0xf3, 0x3a, 0x42, + 0x99, 0xb6, 0x9d, 0xc2, 0x69, 0x16, 0x69, 0x07, 0x0a, 0x8b, 0x9b, 0x8d, 0x40, 0x1e, 0x5f, 0x60, + 0x86, 0x1f, 0x7c, 0x7c, 0xd5, 0xc0, 0xdd, 0xf7, 0xe3, 0x84, 0xba, 0x4c, 0x7f, 0xa2, 0x86, 0x5f, + 0x60, 0xc1, 0x69, 0x0d, 0xd7, 0x77, 0xcf, 0x4a, 0x9c, 0x40, 0x3d, 0x2c, 0x6f, 0x73, 0x4b, 0xd8, + 0x47, 0x8c, 0x9b, 0x79, 0xc4, 0xfc, 0xee, 0xc0, 0x0a, 0x13, 0x91, 0xff, 0x4a, 0x78, 0x41, 0xa4, + 0xe4, 0x28, 0x69, 0xbe, 0xcf, 0xc3, 0x23, 0xaf, 0x85, 0x51, 0x5d, 0x16, 0x0b, 0x36, 0x43, 0xb9, + 0x85, 0x19, 0x7a, 0x4f, 0x3f, 0x7b, 0x43, 0xd9, 0xd3, 0x1d, 0x18, 0x4a, 0xc3, 0xf9, 0x94, 0x61, + 0xd6, 0x82, 0xbc, 0x0f, 0x77, 0x3a, 0xe1, 0x48, 0x76, 0x93, 0xf1, 0xbc, 0x96, 0x1a, 0xc7, 0xa8, + 0xe2, 0x65, 0x66, 0xcd, 0xe8, 0x8f, 0x0e, 0x2c, 0x65, 0x57, 0xae, 0x2f, 0x9b, 0x84, 0xa1, 0xdc, + 0x5c, 0x86, 0xdc, 0x79, 0x0c, 0xe5, 0x53, 0x86, 0xd2, 0x27, 0x45, 0x21, 0xf3, 0xa4, 0xa0, 0x27, + 0xb0, 0x3e, 0x43, 0xdb, 0x5e, 0x38, 0x18, 0xea, 0xfc, 0xdc, 0x96, 0xbe, 0x55, 0x28, 0xb4, 0xa5, + 0x34, 0xc4, 0x95, 0x59, 0x2c, 0xd0, 0x87, 0x50, 0x3a, 0x08, 0x87, 0x61, 0x3f, 0x3c, 0x1e, 0x67, + 0xca, 0xcf, 0xb9, 0xa2, 0xfc, 0x1e, 0xd7, 0xfe, 0xb8, 0x6c, 0x38, 0x7f, 0x5e, 0x36, 0x9c, 0xbf, + 0x2e, 0x1b, 0xce, 0xaf, 0x7f, 0x37, 0x5e, 0x3b, 0x2a, 0xe2, 0xef, 0xca, 0xa3, 0xff, 0x03, 0x00, + 0x00, 0xff, 0xff, 0x64, 0x17, 0xa4, 0x6c, 0xbf, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 4658f21be..8676a18c6 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -163,6 +163,7 @@ message ResizeSource { message ResizeInstructionComplete { int64 JobID = 1; URI URI = 2; + string Error = 3; } message Topology { diff --git a/server/server.go b/server/server.go index 34d397fcf..4a8f0558c 100644 --- a/server/server.go +++ b/server/server.go @@ -122,7 +122,7 @@ func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN - cluster.IndexReporter = m.Server.Holder + cluster.Holder = m.Server.Holder m.Server.Cluster = cluster From f4b07294583c701ae334f7246832366c0e3fee39 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 13:03:55 -0500 Subject: [PATCH 009/311] Rename structs and funcs. NodeSet -> MemberSet URISet -> NodeSet AddNode -> AddNodeBasicSorted AddHost -> AddNode --- broadcast.go | 28 +++--- cluster.go | 76 ++++++++-------- cluster_internal_test.go | 4 +- cluster_test.go | 10 +-- gossip/gossip.go | 50 +++++------ handler.go | 8 +- handler_test.go | 2 +- internal/private.pb.go | 186 +++++++++++++++++++-------------------- internal/private.proto | 4 +- server.go | 4 +- server/server.go | 12 +-- server/server_test.go | 18 ++-- test/handler.go | 4 +- 13 files changed, 203 insertions(+), 203 deletions(-) diff --git a/broadcast.go b/broadcast.go index 3132844cf..e0da4756b 100644 --- a/broadcast.go +++ b/broadcast.go @@ -22,37 +22,37 @@ import ( "github.com/pilosa/pilosa/internal" ) -// NodeSet represents an interface for Node membership and inter-node communication. -type NodeSet interface { +// MemberSet represents an interface for Node membership and inter-node communication. +type MemberSet interface { // Returns a list of all Nodes in the cluster Nodes() []*Node - // Open starts any network activity implemented by the NodeSet + // Open starts any network activity implemented by the MemberSet Open() error } -// StaticNodeSet represents a basic NodeSet for testing. -type StaticNodeSet struct { +// StaticMemberSet represents a basic MemberSet for testing. +type StaticMemberSet struct { nodes []*Node } -// NewStaticNodeSet creates a statically defined NodeSet. -func NewStaticNodeSet() *StaticNodeSet { - return &StaticNodeSet{} +// NewStaticMemberSet creates a statically defined MemberSet. +func NewStaticMemberSet() *StaticMemberSet { + return &StaticMemberSet{} } -// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. -func (s *StaticNodeSet) Nodes() []*Node { +// Nodes implements the MemberSet interface and returns a list of nodes in the cluster. +func (s *StaticMemberSet) Nodes() []*Node { return s.nodes } -// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing. -func (s *StaticNodeSet) Open() error { +// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing. +func (s *StaticMemberSet) Open() error { return nil } -// Join sets the NodeSet nodes to the slice of Nodes passed in. -func (s *StaticNodeSet) Join(nodes []*Node) error { +// Join sets the MemberSet nodes to the slice of Nodes passed in. +func (s *StaticMemberSet) Join(nodes []*Node) error { s.nodes = nodes return nil } diff --git a/cluster.go b/cluster.go index 608d06079..f2ff60d9e 100644 --- a/cluster.go +++ b/cluster.go @@ -129,9 +129,9 @@ func (h ByHost) Less(i, j int) bool { return h[i].URI.String() < h[j].URI.String // Cluster represents a collection of nodes. type Cluster struct { - URI URI - Nodes []*Node // TODO phase this out? - NodeSet NodeSet + URI URI + Nodes []*Node // TODO phase this out? + MemberSet MemberSet // Hashing algorithm used to assign partitions to nodes. Hasher Hasher @@ -198,12 +198,12 @@ func (c *Cluster) IsCoordinator() bool { return c.Coordinator == c.URI } -// AddHost adds a node to the Cluster and updates and saves the +// AddNode adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddHost(uri URI) error { +func (c *Cluster) AddNode(uri URI) error { // add to cluster - _, added := c.AddNode(uri) + _, added := c.AddNodeBasicSorted(uri) if !added { return nil } @@ -220,8 +220,8 @@ func (c *Cluster) AddHost(uri URI) error { return c.saveTopology() } -// URISet returns the list of uris in the cluster. -func (c *Cluster) URISet() []URI { +// NodeSet returns the list of uris in the cluster. +func (c *Cluster) NodeSet() []URI { return Nodes(c.Nodes).URIs() } @@ -237,8 +237,8 @@ func (c *Cluster) setState(state string) { // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ - State: c.State, - URISet: encodeURIs(c.URISet()), + State: c.State, + NodeSet: encodeURIs(c.NodeSet()), } } @@ -252,9 +252,9 @@ func (c *Cluster) NodeByURI(uri URI) *Node { return nil } -// AddNode adds a node to the cluster, sorted by uri. +// AddNodeBasicSorted adds a node to the cluster, sorted by uri. // Returns a pointer to the node and true if the node was added. -func (c *Cluster) AddNode(uri URI) (*Node, bool) { +func (c *Cluster) AddNodeBasicSorted(uri URI) (*Node, bool) { n := c.NodeByURI(uri) if n != nil { return n, false @@ -513,11 +513,11 @@ func (c *Cluster) Open() error { return fmt.Errorf("considerTopology: %v", err) } // Add the local node to the cluster and update state. - c.AddHost(c.URI) + c.AddNode(c.URI) c.setState(state) } else { // Add the local node to the cluster. - c.AddHost(c.URI) + c.AddNode(c.URI) } // Start the EventReceiver. @@ -525,9 +525,9 @@ func (c *Cluster) Open() error { return fmt.Errorf("starting EventReceiver: %v", err) } - // Open NodeSet communication. - if err := c.NodeSet.Open(); err != nil { - return fmt.Errorf("opening NodeSet: %v", err) + // Open MemberSet communication. + if err := c.MemberSet.Open(); err != nil { + return fmt.Errorf("opening MemberSet: %v", err) } // Listen for cluster-resize events. @@ -546,11 +546,11 @@ func (c *Cluster) Close() error { } func (c *Cluster) needTopologyAgreement() bool { - return c.State == NodeStateStarting && !URISlicesAreEqual(c.Topology.URISet, c.URISet()) + return c.State == NodeStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } func (c *Cluster) haveTopologyAgreement() bool { - return URISlicesAreEqual(c.Topology.URISet, c.URISet()) + return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } func (c *Cluster) handleJoiningHost(uri URI) error { @@ -571,7 +571,7 @@ func (c *Cluster) handleJoiningHost(uri URI) error { case ResizeJobStateDone: c.CompleteCurrentJob(ResizeJobStateDone) // Add uri to the cluster. - return c.AddHost(uri) + return c.AddNode(uri) case ResizeJobStateAborted: c.CompleteCurrentJob(ResizeJobStateAborted) } @@ -662,7 +662,7 @@ func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { toCluster.Hasher = c.Hasher toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN - toCluster.AddNode(addURI) + toCluster.AddNodeBasicSorted(addURI) // Add to the ResizeJob the instructions for each index. for _, idx := range c.Holder.Indexes() { @@ -922,9 +922,9 @@ func (j *ResizeJob) distributeResizeInstructions() error { return nil } -type URISet []URI +type NodeSet []URI -func (u URISet) ToHostPortStrings() []string { +func (u NodeSet) ToHostPortStrings() []string { other := make([]string, 0, len(u)) for _, uri := range u { other = append(other, uri.HostPort()) @@ -934,8 +934,8 @@ func (u URISet) ToHostPortStrings() []string { // Topology represents the list of hosts in the cluster. type Topology struct { - mu sync.RWMutex - URISet []URI + mu sync.RWMutex + NodeSet []URI } func NewTopology() *Topology { @@ -950,7 +950,7 @@ func (t *Topology) ContainsURI(uri URI) bool { } func (t *Topology) containsURI(uri URI) bool { - for _, turi := range t.URISet { + for _, turi := range t.NodeSet { if turi == uri { return true } @@ -958,14 +958,14 @@ func (t *Topology) containsURI(uri URI) bool { return false } -// AddHost adds the uri to the topology and returns true if added. +// AddNode adds the uri to the topology and returns true if added. func (t *Topology) AddURI(uri URI) bool { t.mu.Lock() defer t.mu.Unlock() if t.containsURI(uri) { return false } - t.URISet = append(t.URISet, uri) + t.NodeSet = append(t.NodeSet, uri) return true } @@ -1007,7 +1007,7 @@ func encodeTopology(topology *Topology) *internal.Topology { return nil } return &internal.Topology{ - URISet: encodeURIs(topology.URISet), + NodeSet: encodeURIs(topology.NodeSet), } } @@ -1017,24 +1017,24 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { } t := &Topology{ - URISet: decodeURIs(topology.URISet), + NodeSet: decodeURIs(topology.NodeSet), } return t, nil } func (c *Cluster) considerTopology() (string, error) { // If there is no .topology file, it's safe to go to state NORMAL. - if len(c.Topology.URISet) == 0 { + if len(c.Topology.NodeSet) == 0 { return NodeStateNormal, nil } // The local node (coordinator) must be in the .topology. if !c.Topology.ContainsURI(c.Coordinator) { - return "", fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.URISet) + return "", fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.NodeSet) } // If local node is the only thing in .topology, continue to state NORMAL. - if len(c.Topology.URISet) == 1 { + if len(c.Topology.NodeSet) == 1 { return NodeStateNormal, nil } @@ -1064,11 +1064,11 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { } uri := e.URI - if err := c.AddHost(uri); err != nil { + if err := c.AddNode(uri); err != nil { return err } - // If the result of the previous AddHost completed the joining of nodes + // If the result of the previous AddNode completed the joining of nodes // in the topology, then change the state to NORMAL. if c.haveTopologyAgreement() { return c.setStateAndBroadcast(NodeStateNormal) @@ -1085,7 +1085,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { // If the index does not yet have data, go ahead and add the node. if !c.Holder.HasData() { uri := e.URI - if err := c.AddHost(uri); err != nil { + if err := c.AddNode(uri); err != nil { return err } return c.setStateAndBroadcast(NodeStateNormal) @@ -1113,8 +1113,8 @@ func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { return nil } - for _, uri := range decodeURIs(cs.URISet) { - c.AddHost(uri) + for _, uri := range decodeURIs(cs.NodeSet) { + c.AddNode(uri) } c.setState(cs.State) diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 648164ca7..4ef65c958 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -31,8 +31,8 @@ func TestFragCombos(t *testing.T) { if err != nil { t.Fatal(err) } - c.AddNode(*uri0) - c.AddNode(*uri1) + c.AddNodeBasicSorted(*uri0) + c.AddNodeBasicSorted(*uri1) tests := []struct { idx string diff --git a/cluster_test.go b/cluster_test.go index 32f48895a..6fa3ed6e5 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -114,7 +114,7 @@ func TestCluster_Nodes(t *testing.T) { {URI: uri2}, } - t.Run("URISet", func(t *testing.T) { + t.Run("NodeSet", func(t *testing.T) { actual := pilosa.Nodes(nodes).URIs() expected := []pilosa.URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { @@ -198,17 +198,17 @@ func TestCluster_Topology(t *testing.T) { base := test.NewURIFromHostPort("host0", 0) invalid := test.NewURIFromHostPort("invalid", 0) - t.Run("AddHost", func(t *testing.T) { - err := c1.AddHost(uri1) + t.Run("AddNode", func(t *testing.T) { + err := c1.AddNode(uri1) if err != nil { t.Fatal(err) } // add the same host. - err = c1.AddHost(uri1) + err = c1.AddNode(uri1) if err != nil { t.Fatal(err) } - err = c1.AddHost(uri2) + err = c1.AddNode(uri2) if err != nil { t.Fatal(err) } diff --git a/gossip/gossip.go b/gossip/gossip.go index f4bf3d306..f1599bf2d 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -28,10 +28,10 @@ import ( "github.com/pilosa/pilosa/internal" ) -// GossipNodeSet represents a gossip implementation of NodeSet using memberlist -// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster -// GossipNodeSet also represents an implementation of memberlist.Delegate -type GossipNodeSet struct { +// GossipMemberSet represents a gossip implementation of MemberSet using memberlist +// GossipMemberSet also represents a gossip implementation of pilosa.Broadcaster +// GossipMemberSet also represents an implementation of memberlist.Delegate +type GossipMemberSet struct { memberlist *memberlist.Memberlist handler pilosa.BroadcastHandler @@ -44,8 +44,8 @@ type GossipNodeSet struct { LogOutput io.Writer } -// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. -func (g *GossipNodeSet) Nodes() []*pilosa.Node { +// Nodes implements the MemberSet interface and returns a list of nodes in the cluster. +func (g *GossipMemberSet) Nodes() []*pilosa.Node { a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { uri, _ := pilosa.NewURIFromAddress(n.Name) @@ -56,15 +56,15 @@ func (g *GossipNodeSet) Nodes() []*pilosa.Node { } // Start implements the BroadcastReceiver interface and sets the BroadcastHandler. -func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error { +func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { g.handler = h return nil } -// Open implements the NodeSet interface to start network activity. -func (g *GossipNodeSet) Open() error { +// Open implements the MemberSet interface to start network activity. +func (g *GossipMemberSet) Open() error { if g.handler == nil { - return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") + return fmt.Errorf("opening GossipMemberSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") } err := error(nil) @@ -88,7 +88,7 @@ func (g *GossipNodeSet) Open() error { // attach to gossip seed node nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds - err = g.joinWithRetry(pilosa.URISet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) + err = g.joinWithRetry(pilosa.NodeSet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) if err != nil { return err } @@ -96,7 +96,7 @@ func (g *GossipNodeSet) Open() error { } // joinWithRetry wraps the standard memberlist Join function in a retry. -func (g *GossipNodeSet) joinWithRetry(hosts []string) error { +func (g *GossipMemberSet) joinWithRetry(hosts []string) error { err := retry(60, 2*time.Second, func() error { _, err := g.memberlist.Join(hosts) return err @@ -120,8 +120,8 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) { return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } -// logger returns a logger for the GossipNodeSet. -func (g *GossipNodeSet) logger() *log.Logger { +// logger returns a logger for the GossipMemberSet. +func (g *GossipMemberSet) logger() *log.Logger { return log.New(g.LogOutput, "", log.LstdFlags) } @@ -132,9 +132,9 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } -// NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) *GossipNodeSet { - g := &GossipNodeSet{ +// NewGossipMemberSet returns a new instance of GossipMemberSet. +func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) *GossipMemberSet { + g := &GossipMemberSet{ LogOutput: server.LogOutput, } @@ -159,7 +159,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed } // SendSync implementation of the Broadcaster interface. -func (g *GossipNodeSet) SendSync(pb proto.Message) error { +func (g *GossipMemberSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { return err @@ -187,7 +187,7 @@ func (g *GossipNodeSet) SendSync(pb proto.Message) error { } // SendAsync implementation of the Broadcaster interface. -func (g *GossipNodeSet) SendAsync(pb proto.Message) error { +func (g *GossipMemberSet) SendAsync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { return err @@ -202,7 +202,7 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error { } // SendTo implementation of the Broadcaster interface. -func (g *GossipNodeSet) SendTo(to *pilosa.Node, pb proto.Message) error { +func (g *GossipMemberSet) SendTo(to *pilosa.Node, pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { return err @@ -221,13 +221,13 @@ func (g *GossipNodeSet) SendTo(to *pilosa.Node, pb proto.Message) error { } // NodeMeta implementation of the memberlist.Delegate interface. -func (g *GossipNodeSet) NodeMeta(limit int) []byte { +func (g *GossipMemberSet) NodeMeta(limit int) []byte { return []byte{} } // NotifyMsg implementation of the memberlist.Delegate interface // called when a user-data message is received. -func (g *GossipNodeSet) NotifyMsg(b []byte) { +func (g *GossipMemberSet) NotifyMsg(b []byte) { m, err := pilosa.UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -241,13 +241,13 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) { // GetBroadcasts implementation of the memberlist.Delegate interface // called when user data messages can be broadcast. -func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { +func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. -func (g *GossipNodeSet) LocalState(join bool) []byte { +func (g *GossipMemberSet) LocalState(join bool) []byte { pb, err := g.statusHandler.LocalStatus() if err != nil { g.logger().Printf("error getting local state, err=%s", err) @@ -265,7 +265,7 @@ func (g *GossipNodeSet) LocalState(join bool) []byte { // MergeRemoteState implementation of the memberlist.Delegate interface // receive and process the remote side's LocalState. -func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { +func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeStatus if err := proto.Unmarshal(buf, &pb); err != nil { diff --git a/handler.go b/handler.go index b34a616a5..0aeb9a441 100644 --- a/handler.go +++ b/handler.go @@ -224,8 +224,8 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { cs := pb.(*internal.ClusterStatus) if err := json.NewEncoder(w).Encode(getStatusResponse{ - State: cs.State, - URISet: decodeURIs(cs.URISet), + State: cs.State, + NodeSet: decodeURIs(cs.NodeSet), }); err != nil { h.logger().Printf("write status response error: %s", err) } @@ -236,8 +236,8 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - URISet []URI `json:"uri-set"` + State string `json:"state"` + NodeSet []URI `json:"nodes"` } // handlePostQuery handles /query requests. diff --git a/handler_test.go b/handler_test.go index 05f93c6ab..4c4b4c5ca 100644 --- a/handler_test.go +++ b/handler_test.go @@ -147,7 +147,7 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","uri-set":[{"scheme":"http","host":"localhost","port":10101}]}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"scheme":"http","host":"localhost","port":10101}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } diff --git a/internal/private.pb.go b/internal/private.pb.go index 1b96a50a6..421179ca6 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -702,8 +702,8 @@ func (m *NodeStatus) GetSchema() *Schema { } type ClusterStatus struct { - State string `protobuf:"bytes,1,opt,name=State,proto3" json:"State,omitempty"` - URISet []*URI `protobuf:"bytes,2,rep,name=URISet" json:"URISet,omitempty"` + State string `protobuf:"bytes,1,opt,name=State,proto3" json:"State,omitempty"` + NodeSet []*URI `protobuf:"bytes,2,rep,name=NodeSet" json:"NodeSet,omitempty"` } func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } @@ -718,9 +718,9 @@ func (m *ClusterStatus) GetState() string { return "" } -func (m *ClusterStatus) GetURISet() []*URI { +func (m *ClusterStatus) GetNodeSet() []*URI { if m != nil { - return m.URISet + return m.NodeSet } return nil } @@ -920,7 +920,7 @@ func (m *ResizeInstructionComplete) GetError() string { } type Topology struct { - URISet []*URI `protobuf:"bytes,1,rep,name=URISet" json:"URISet,omitempty"` + NodeSet []*URI `protobuf:"bytes,1,rep,name=NodeSet" json:"NodeSet,omitempty"` } func (m *Topology) Reset() { *m = Topology{} } @@ -928,9 +928,9 @@ func (m *Topology) String() string { return proto.CompactTextString(m func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } -func (m *Topology) GetURISet() []*URI { +func (m *Topology) GetNodeSet() []*URI { if m != nil { - return m.URISet + return m.NodeSet } return nil } @@ -1883,8 +1883,8 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if len(m.URISet) > 0 { - for _, msg := range m.URISet { + if len(m.NodeSet) > 0 { + for _, msg := range m.NodeSet { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -2134,8 +2134,8 @@ func (m *Topology) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.URISet) > 0 { - for _, msg := range m.URISet { + if len(m.NodeSet) > 0 { + for _, msg := range m.NodeSet { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -2556,8 +2556,8 @@ func (m *ClusterStatus) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.URISet) > 0 { - for _, e := range m.URISet { + if len(m.NodeSet) > 0 { + for _, e := range m.NodeSet { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -2671,8 +2671,8 @@ func (m *ResizeInstructionComplete) Size() (n int) { func (m *Topology) Size() (n int) { var l int _ = l - if len(m.URISet) > 0 { - for _, e := range m.URISet { + if len(m.NodeSet) > 0 { + for _, e := range m.NodeSet { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -5814,7 +5814,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URISet", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NodeSet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5838,8 +5838,8 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.URISet = append(m.URISet, &URI{}) - if err := m.URISet[len(m.URISet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.NodeSet = append(m.NodeSet, &URI{}) + if err := m.NodeSet[len(m.NodeSet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6664,7 +6664,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URISet", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NodeSet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -6688,8 +6688,8 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.URISet = append(m.URISet, &URI{}) - if err := m.URISet[len(m.URISet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.NodeSet = append(m.NodeSet, &URI{}) + if err := m.NodeSet[len(m.NodeSet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6822,77 +6822,77 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1147 bytes of a gzipped FileDescriptorProto + // 1144 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0x66, 0x3c, 0xb6, 0xd7, 0x2e, 0xc7, 0x1b, 0xa7, 0x09, 0x91, 0x13, 0x45, 0x5e, 0xd3, 0x12, - 0x6c, 0x58, 0x89, 0xc0, 0x66, 0x25, 0x04, 0x41, 0x48, 0xb0, 0xb1, 0x57, 0x3b, 0xb0, 0x59, 0x96, - 0x76, 0xb2, 0x48, 0x1c, 0x90, 0x3a, 0x76, 0x93, 0x8c, 0x62, 0xcf, 0x98, 0x9e, 0x76, 0x12, 0xef, - 0x81, 0x1b, 0x1c, 0xe0, 0x05, 0xb8, 0x73, 0xe6, 0x3d, 0x38, 0xf2, 0x08, 0x28, 0x3c, 0x04, 0x12, - 0x17, 0x50, 0xd7, 0x74, 0xcf, 0x8c, 0xff, 0x12, 0x25, 0xb7, 0xa9, 0xea, 0xaa, 0xea, 0xaf, 0xbf, - 0xfa, 0xe9, 0x1e, 0xa8, 0x0e, 0xa5, 0x7f, 0xc6, 0x95, 0xd8, 0x1e, 0xca, 0x50, 0x85, 0xa4, 0xe4, - 0x07, 0x4a, 0xc8, 0x80, 0xf7, 0xe9, 0x97, 0x50, 0xf6, 0x82, 0x9e, 0xb8, 0xd8, 0x17, 0x8a, 0x93, - 0x26, 0x54, 0xf6, 0xc2, 0xfe, 0x68, 0x10, 0x3c, 0xe3, 0x47, 0xa2, 0x5f, 0x77, 0x9a, 0xce, 0x56, - 0x99, 0x65, 0x55, 0xda, 0xe2, 0xc0, 0x1f, 0x88, 0xaf, 0x46, 0x3c, 0x50, 0xa3, 0x41, 0x3d, 0x17, - 0x5b, 0x64, 0x54, 0xf4, 0x5f, 0x07, 0xca, 0x4f, 0x24, 0x1f, 0x08, 0x8c, 0xb8, 0x01, 0x25, 0x16, - 0x9e, 0x67, 0xc3, 0x25, 0x32, 0x79, 0x1b, 0xee, 0x7a, 0xc1, 0x99, 0x90, 0x91, 0x68, 0x07, 0xfc, - 0xa8, 0x2f, 0x7a, 0x18, 0xae, 0xc4, 0xa6, 0xb4, 0x64, 0x13, 0xca, 0x7b, 0xbc, 0x7b, 0x22, 0x0e, - 0xc6, 0x43, 0x51, 0x77, 0x31, 0x48, 0xaa, 0x48, 0x56, 0x3b, 0xfe, 0x2b, 0x51, 0xcf, 0x37, 0x9d, - 0xad, 0x2a, 0x4b, 0x15, 0xd3, 0x78, 0x0b, 0x33, 0x78, 0x09, 0x85, 0x25, 0xc6, 0x83, 0xe3, 0x04, - 0x43, 0x11, 0x31, 0x4c, 0xe8, 0xc8, 0x7d, 0x28, 0x3e, 0xf1, 0x45, 0xbf, 0x17, 0xd5, 0xef, 0x34, - 0xdd, 0xad, 0xca, 0xce, 0xf2, 0xb6, 0xe5, 0x6f, 0x1b, 0xf5, 0xcc, 0x2c, 0x53, 0x0a, 0x77, 0xbd, - 0xc1, 0x30, 0x94, 0x8a, 0x89, 0x68, 0x18, 0x06, 0x91, 0x20, 0x35, 0x70, 0xdb, 0x52, 0x9a, 0xb3, - 0xeb, 0x4f, 0xfa, 0x03, 0xd4, 0x1e, 0xf7, 0xc3, 0xee, 0x69, 0x8b, 0x2b, 0xce, 0xc4, 0xf7, 0x23, - 0x11, 0x29, 0xb2, 0x0a, 0x05, 0xcc, 0x82, 0xb1, 0x8b, 0x05, 0xad, 0x45, 0x26, 0x0d, 0xcd, 0xb1, - 0xa0, 0xb5, 0xe8, 0x8f, 0x54, 0xe4, 0x59, 0x2c, 0x68, 0x6d, 0xa7, 0xef, 0x77, 0x63, 0x0a, 0xf2, - 0x2c, 0x16, 0x08, 0x81, 0xfc, 0x4b, 0x5f, 0x9c, 0x9b, 0x73, 0xe3, 0x37, 0xf5, 0x60, 0x25, 0xb3, - 0xbf, 0x81, 0xb9, 0x06, 0x45, 0x16, 0x9e, 0x7b, 0xad, 0xa8, 0xee, 0x34, 0xdd, 0xad, 0x3c, 0x33, - 0x12, 0xb2, 0x8b, 0xe9, 0xd7, 0x4b, 0x39, 0x5c, 0x4a, 0x15, 0x74, 0x1d, 0x0a, 0x48, 0xb5, 0x3e, - 0x65, 0xea, 0xab, 0x3f, 0xe9, 0x7f, 0x0e, 0x94, 0xf7, 0xf9, 0x05, 0xc2, 0x88, 0xc8, 0x27, 0x50, - 0xea, 0x28, 0x1e, 0xf4, 0xb8, 0xec, 0xa1, 0x51, 0x65, 0xe7, 0xcd, 0x94, 0xc2, 0xc4, 0x6c, 0xdb, - 0xda, 0xb4, 0x03, 0x25, 0xc7, 0x2c, 0x71, 0x21, 0xbb, 0x70, 0xc7, 0xd4, 0x04, 0x62, 0xa8, 0xec, - 0x34, 0xe7, 0x79, 0x27, 0x65, 0xa3, 0x9d, 0xad, 0xc3, 0xc6, 0xc7, 0x50, 0x9d, 0x08, 0xab, 0xb1, - 0x9e, 0x8a, 0xb1, 0xcd, 0xc8, 0xa9, 0x18, 0x6b, 0xee, 0xce, 0x78, 0x7f, 0x14, 0xf3, 0x9c, 0x67, - 0xb1, 0xb0, 0x9b, 0xfb, 0xd0, 0xd9, 0xd8, 0x85, 0xa5, 0x6c, 0xd4, 0x9b, 0xf8, 0xd2, 0x6f, 0x81, - 0xec, 0x49, 0xc1, 0x95, 0x40, 0x78, 0xfb, 0x22, 0x8a, 0xf8, 0xb1, 0x58, 0x9c, 0xe9, 0x38, 0x7b, - 0xb9, 0x6c, 0xf6, 0x36, 0xa1, 0xec, 0x45, 0xf6, 0xe0, 0x2e, 0xd6, 0x65, 0xaa, 0xa0, 0x0f, 0x80, - 0xb4, 0x44, 0x5f, 0x28, 0x61, 0xfa, 0xf7, 0x8a, 0xf8, 0xb4, 0x63, 0xb1, 0x5c, 0x6f, 0x4b, 0xee, - 0x43, 0x5e, 0xb7, 0x2e, 0x42, 0xa9, 0xec, 0xbc, 0x9e, 0x32, 0x9d, 0xcc, 0x09, 0x86, 0x06, 0xd4, - 0xb7, 0x41, 0x4d, 0xbb, 0x5f, 0x73, 0xc0, 0x39, 0xa5, 0x6c, 0xb7, 0x72, 0xa7, 0xb7, 0x4a, 0x06, - 0x88, 0xd9, 0xea, 0x53, 0x7b, 0xd6, 0xdb, 0x6e, 0x45, 0xbf, 0x31, 0x5a, 0xdd, 0x12, 0xcf, 0xf5, - 0x6a, 0xec, 0x83, 0xdf, 0x8b, 0x8f, 0x3c, 0x85, 0x43, 0xc7, 0xd6, 0x3d, 0x14, 0xd5, 0xdd, 0xa6, - 0xab, 0x63, 0xa3, 0x40, 0x1f, 0x41, 0xb1, 0xd3, 0x3d, 0x11, 0x03, 0x4e, 0xde, 0xd1, 0x85, 0xda, - 0x13, 0x17, 0x22, 0x32, 0x65, 0xbe, 0x3c, 0x45, 0x1f, 0xb3, 0xeb, 0xf4, 0x17, 0xc7, 0xa0, 0x5f, - 0x80, 0xa8, 0x88, 0x7b, 0x47, 0xf5, 0xfc, 0xcc, 0xc4, 0xd1, 0x7a, 0x66, 0x96, 0x49, 0x1b, 0x6a, - 0x5e, 0x30, 0x1c, 0xa9, 0x96, 0xf8, 0xce, 0x0f, 0x7c, 0xe5, 0x87, 0x41, 0x54, 0x2f, 0xa2, 0xcb, - 0x7a, 0x76, 0xeb, 0x09, 0x0b, 0x36, 0xe3, 0x42, 0x7f, 0x72, 0x60, 0x79, 0x4a, 0x79, 0x0d, 0xae, - 0xdc, 0xd5, 0xb8, 0x3e, 0x48, 0x46, 0xa6, 0x8b, 0x86, 0x8d, 0x85, 0x68, 0x26, 0x27, 0xe8, 0x6f, - 0x0e, 0xac, 0xce, 0x33, 0x98, 0x8b, 0xa6, 0x01, 0xf0, 0x42, 0xfa, 0x03, 0x2e, 0xc7, 0x5f, 0x88, - 0xb1, 0xb9, 0x3d, 0x32, 0x1a, 0xf2, 0x35, 0xac, 0x4d, 0xc5, 0xfa, 0xac, 0x1b, 0x53, 0x14, 0x83, - 0xba, 0xb7, 0x10, 0x54, 0x6c, 0xc7, 0x16, 0xb8, 0xd3, 0x7f, 0x1c, 0x78, 0x63, 0xee, 0x52, 0x5a, - 0x7d, 0x4e, 0xb6, 0xd0, 0x1f, 0x40, 0xed, 0xa5, 0x1e, 0x0c, 0x2d, 0x11, 0x29, 0x3f, 0xe0, 0xda, - 0xd2, 0x94, 0xe7, 0x8c, 0x9e, 0x78, 0x50, 0x42, 0xdd, 0x3e, 0x1f, 0x1a, 0x98, 0xef, 0x5e, 0x03, - 0x73, 0xdb, 0xda, 0x9b, 0xb9, 0x69, 0x45, 0x0d, 0x06, 0xe7, 0xb8, 0xbd, 0x14, 0x50, 0xd0, 0x13, - 0x71, 0xc2, 0xe1, 0x46, 0x53, 0x2d, 0x84, 0x4d, 0x3b, 0x49, 0x26, 0x90, 0x5c, 0xdd, 0x93, 0x1f, - 0x01, 0xa4, 0xa6, 0xa6, 0xdd, 0xaf, 0xa8, 0xcf, 0x8c, 0x31, 0x7d, 0x0a, 0x9b, 0x76, 0xcc, 0xdd, - 0x60, 0x43, 0x5b, 0x2d, 0xb9, 0xb4, 0x5a, 0x68, 0x1b, 0xdc, 0x43, 0xe6, 0xe9, 0xab, 0x0e, 0xbb, - 0xd5, 0xa6, 0xc8, 0x48, 0xda, 0xe5, 0x69, 0x18, 0x29, 0xeb, 0xa2, 0xbf, 0xb5, 0xee, 0x45, 0x28, - 0x15, 0x22, 0xae, 0x32, 0xfc, 0xa6, 0x3f, 0x3b, 0x00, 0xcf, 0xc3, 0x9e, 0xe8, 0x28, 0xae, 0x46, - 0x11, 0xb9, 0x87, 0x51, 0x31, 0x56, 0x65, 0xa7, 0x9a, 0x9e, 0xe9, 0x90, 0x79, 0x0c, 0xf7, 0x7b, - 0x98, 0xb9, 0x08, 0x67, 0x27, 0x4c, 0xb2, 0xc4, 0x32, 0xd7, 0xe5, 0x96, 0x1d, 0x28, 0x86, 0xaa, - 0x5a, 0x6a, 0x1f, 0xeb, 0x0d, 0x68, 0x4e, 0x9f, 0x41, 0x75, 0xaf, 0x3f, 0x8a, 0x94, 0x90, 0x06, - 0x8e, 0xbe, 0x49, 0x14, 0x57, 0x49, 0xfd, 0xa1, 0x40, 0xde, 0x82, 0xe2, 0x21, 0xf3, 0x3a, 0x42, - 0x99, 0xb6, 0x9d, 0xc2, 0x69, 0x16, 0x69, 0x07, 0x0a, 0x8b, 0x9b, 0x8d, 0x40, 0x1e, 0x5f, 0x60, - 0x86, 0x1f, 0x7c, 0x7c, 0xd5, 0xc0, 0xdd, 0xf7, 0xe3, 0x84, 0xba, 0x4c, 0x7f, 0xa2, 0x86, 0x5f, - 0x60, 0xc1, 0x69, 0x0d, 0xd7, 0x77, 0xcf, 0x4a, 0x9c, 0x40, 0x3d, 0x2c, 0x6f, 0x73, 0x4b, 0xd8, - 0x47, 0x8c, 0x9b, 0x79, 0xc4, 0xfc, 0xee, 0xc0, 0x0a, 0x13, 0x91, 0xff, 0x4a, 0x78, 0x41, 0xa4, - 0xe4, 0x28, 0x69, 0xbe, 0xcf, 0xc3, 0x23, 0xaf, 0x85, 0x51, 0x5d, 0x16, 0x0b, 0x36, 0x43, 0xb9, - 0x85, 0x19, 0x7a, 0x4f, 0x3f, 0x7b, 0x43, 0xd9, 0xd3, 0x1d, 0x18, 0x4a, 0xc3, 0xf9, 0x94, 0x61, - 0xd6, 0x82, 0xbc, 0x0f, 0x77, 0x3a, 0xe1, 0x48, 0x76, 0x93, 0xf1, 0xbc, 0x96, 0x1a, 0xc7, 0xa8, - 0xe2, 0x65, 0x66, 0xcd, 0xe8, 0x8f, 0x0e, 0x2c, 0x65, 0x57, 0xae, 0x2f, 0x9b, 0x84, 0xa1, 0xdc, - 0x5c, 0x86, 0xdc, 0x79, 0x0c, 0xe5, 0x53, 0x86, 0xd2, 0x27, 0x45, 0x21, 0xf3, 0xa4, 0xa0, 0x27, - 0xb0, 0x3e, 0x43, 0xdb, 0x5e, 0x38, 0x18, 0xea, 0xfc, 0xdc, 0x96, 0xbe, 0x55, 0x28, 0xb4, 0xa5, - 0x34, 0xc4, 0x95, 0x59, 0x2c, 0xd0, 0x87, 0x50, 0x3a, 0x08, 0x87, 0x61, 0x3f, 0x3c, 0x1e, 0x67, - 0xca, 0xcf, 0xb9, 0xa2, 0xfc, 0x1e, 0xd7, 0xfe, 0xb8, 0x6c, 0x38, 0x7f, 0x5e, 0x36, 0x9c, 0xbf, - 0x2e, 0x1b, 0xce, 0xaf, 0x7f, 0x37, 0x5e, 0x3b, 0x2a, 0xe2, 0xef, 0xca, 0xa3, 0xff, 0x03, 0x00, - 0x00, 0xff, 0xff, 0x64, 0x17, 0xa4, 0x6c, 0xbf, 0x0c, 0x00, 0x00, + 0x10, 0x66, 0x3c, 0xb6, 0x63, 0x97, 0xe3, 0x8d, 0xd3, 0x84, 0xc8, 0x89, 0x22, 0xaf, 0xe9, 0x03, + 0x09, 0x2b, 0x11, 0x20, 0x91, 0x10, 0x04, 0x21, 0xc1, 0xc6, 0x5e, 0xed, 0x00, 0x09, 0x4b, 0x3b, + 0xbb, 0x48, 0x1c, 0x90, 0x3a, 0x76, 0x93, 0x8c, 0x32, 0x9e, 0x31, 0x33, 0xed, 0x24, 0xde, 0x03, + 0x37, 0x38, 0xc0, 0x0b, 0x70, 0xe7, 0xcc, 0x7b, 0x70, 0xe4, 0x11, 0x50, 0x78, 0x08, 0x24, 0x2e, + 0xac, 0xba, 0xa6, 0x7b, 0x66, 0xfc, 0x17, 0x2b, 0xb9, 0x4d, 0x55, 0x57, 0x55, 0x7f, 0xfd, 0xd5, + 0x4f, 0xf7, 0x40, 0x75, 0x10, 0xba, 0x97, 0x5c, 0x8a, 0xdd, 0x41, 0x18, 0xc8, 0x80, 0x94, 0x5c, + 0x5f, 0x8a, 0xd0, 0xe7, 0x1e, 0xfd, 0x0a, 0xca, 0x8e, 0xdf, 0x13, 0xd7, 0x47, 0x42, 0x72, 0xd2, + 0x84, 0xca, 0x61, 0xe0, 0x0d, 0xfb, 0xfe, 0x97, 0xfc, 0x54, 0x78, 0x75, 0xab, 0x69, 0xed, 0x94, + 0x59, 0x56, 0xa5, 0x2c, 0x4e, 0xdc, 0xbe, 0xf8, 0x7a, 0xc8, 0x7d, 0x39, 0xec, 0xd7, 0x73, 0xb1, + 0x45, 0x46, 0x45, 0xff, 0xb3, 0xa0, 0xfc, 0x24, 0xe4, 0x7d, 0x81, 0x11, 0x37, 0xa1, 0xc4, 0x82, + 0xab, 0x6c, 0xb8, 0x44, 0x26, 0x6f, 0xc1, 0x03, 0xc7, 0xbf, 0x14, 0x61, 0x24, 0xda, 0x3e, 0x3f, + 0xf5, 0x44, 0x0f, 0xc3, 0x95, 0xd8, 0x84, 0x96, 0x6c, 0x41, 0xf9, 0x90, 0x77, 0xcf, 0xc5, 0xc9, + 0x68, 0x20, 0xea, 0x36, 0x06, 0x49, 0x15, 0xc9, 0x6a, 0xc7, 0x7d, 0x29, 0xea, 0xf9, 0xa6, 0xb5, + 0x53, 0x65, 0xa9, 0x62, 0x12, 0x6f, 0x61, 0x0a, 0x2f, 0xa1, 0xb0, 0xcc, 0xb8, 0x7f, 0x96, 0x60, + 0x28, 0x22, 0x86, 0x31, 0x1d, 0xd9, 0x86, 0xe2, 0x13, 0x57, 0x78, 0xbd, 0xa8, 0xbe, 0xd4, 0xb4, + 0x77, 0x2a, 0x7b, 0x2b, 0xbb, 0x86, 0xbf, 0x5d, 0xd4, 0x33, 0xbd, 0x4c, 0x29, 0x3c, 0x70, 0xfa, + 0x83, 0x20, 0x94, 0x4c, 0x44, 0x83, 0xc0, 0x8f, 0x04, 0xa9, 0x81, 0xdd, 0x0e, 0x43, 0x7d, 0x76, + 0xf5, 0x49, 0x7f, 0x84, 0xda, 0x63, 0x2f, 0xe8, 0x5e, 0xb4, 0xb8, 0xe4, 0x4c, 0xfc, 0x30, 0x14, + 0x91, 0x24, 0x6b, 0x50, 0xc0, 0x2c, 0x68, 0xbb, 0x58, 0x50, 0x5a, 0x64, 0x52, 0xd3, 0x1c, 0x0b, + 0x4a, 0x8b, 0xfe, 0x48, 0x45, 0x9e, 0xc5, 0x82, 0xd2, 0x76, 0x3c, 0xb7, 0x1b, 0x53, 0x90, 0x67, + 0xb1, 0x40, 0x08, 0xe4, 0x5f, 0xb8, 0xe2, 0x4a, 0x9f, 0x1b, 0xbf, 0xa9, 0x03, 0xab, 0x99, 0xfd, + 0x35, 0xcc, 0x75, 0x28, 0xb2, 0xe0, 0xca, 0x69, 0x45, 0x75, 0xab, 0x69, 0xef, 0xe4, 0x99, 0x96, + 0x90, 0x5d, 0x4c, 0xbf, 0x5a, 0xca, 0xe1, 0x52, 0xaa, 0xa0, 0x1b, 0x50, 0x40, 0xaa, 0xd5, 0x29, + 0x53, 0x5f, 0xf5, 0x49, 0xff, 0xb7, 0xa0, 0x7c, 0xc4, 0xaf, 0x11, 0x46, 0x44, 0x3e, 0x81, 0x52, + 0x47, 0x72, 0xbf, 0xc7, 0xc3, 0x1e, 0x1a, 0x55, 0xf6, 0xde, 0x4c, 0x29, 0x4c, 0xcc, 0x76, 0x8d, + 0x4d, 0xdb, 0x97, 0xe1, 0x88, 0x25, 0x2e, 0xe4, 0x00, 0x96, 0x74, 0x4d, 0x20, 0x86, 0xca, 0x5e, + 0x73, 0x96, 0x77, 0x52, 0x36, 0xca, 0xd9, 0x38, 0x6c, 0x7e, 0x0c, 0xd5, 0xb1, 0xb0, 0x0a, 0xeb, + 0x85, 0x18, 0x99, 0x8c, 0x5c, 0x88, 0x91, 0xe2, 0xee, 0x92, 0x7b, 0xc3, 0x98, 0xe7, 0x3c, 0x8b, + 0x85, 0x83, 0xdc, 0x87, 0xd6, 0xe6, 0x01, 0x2c, 0x67, 0xa3, 0xde, 0xc5, 0x97, 0x7e, 0x07, 0xe4, + 0x30, 0x14, 0x5c, 0x0a, 0x84, 0x77, 0x24, 0xa2, 0x88, 0x9f, 0x89, 0xf9, 0x99, 0x8e, 0xb3, 0x97, + 0xcb, 0x66, 0x6f, 0x0b, 0xca, 0x4e, 0x64, 0x0e, 0x6e, 0x63, 0x5d, 0xa6, 0x0a, 0xfa, 0x08, 0x48, + 0x4b, 0x78, 0x42, 0x0a, 0xdd, 0xbf, 0xb7, 0xc4, 0xa7, 0x1d, 0x83, 0x65, 0xb1, 0x2d, 0xd9, 0x86, + 0xbc, 0x6a, 0x5d, 0x84, 0x52, 0xd9, 0x7b, 0x3d, 0x65, 0x3a, 0x99, 0x13, 0x0c, 0x0d, 0xa8, 0x6b, + 0x82, 0xea, 0x76, 0x5f, 0x70, 0xc0, 0x19, 0xa5, 0x6c, 0xb6, 0xb2, 0x27, 0xb7, 0x4a, 0x06, 0x88, + 0xde, 0xea, 0x53, 0x73, 0xd6, 0xfb, 0x6e, 0x45, 0xbf, 0xd5, 0x5a, 0xd5, 0x12, 0xc7, 0x6a, 0x35, + 0xf6, 0xc1, 0xef, 0xf9, 0x47, 0x9e, 0xc0, 0xa1, 0x62, 0xab, 0x1e, 0x8a, 0xea, 0x76, 0xd3, 0x56, + 0xb1, 0x51, 0xa0, 0xfb, 0x50, 0xec, 0x74, 0xcf, 0x45, 0x9f, 0x93, 0xb7, 0x55, 0xa1, 0xf6, 0xc4, + 0xb5, 0x88, 0x74, 0x99, 0xaf, 0x4c, 0xd0, 0xc7, 0xcc, 0x3a, 0xfd, 0xd5, 0xd2, 0xe8, 0xe7, 0x20, + 0x2a, 0xe2, 0xde, 0x51, 0x3d, 0x3f, 0x35, 0x71, 0x94, 0x9e, 0xe9, 0x65, 0xd2, 0x86, 0x9a, 0xe3, + 0x0f, 0x86, 0xb2, 0x25, 0xbe, 0x77, 0x7d, 0x57, 0xba, 0x81, 0x1f, 0xd5, 0x8b, 0xe8, 0xb2, 0x91, + 0xdd, 0x7a, 0xcc, 0x82, 0x4d, 0xb9, 0xd0, 0x9f, 0x2d, 0x58, 0x99, 0x50, 0x2e, 0xc0, 0x95, 0xbb, + 0x1d, 0xd7, 0x07, 0xc9, 0xc8, 0xb4, 0xd1, 0xb0, 0x31, 0x17, 0xcd, 0xf8, 0x04, 0xfd, 0xdd, 0x82, + 0xb5, 0x59, 0x06, 0x33, 0xd1, 0x34, 0x00, 0x9e, 0x85, 0x6e, 0x9f, 0x87, 0xa3, 0x2f, 0xc4, 0x48, + 0xdf, 0x1e, 0x19, 0x0d, 0xf9, 0x06, 0xd6, 0x27, 0x62, 0x7d, 0xd6, 0x8d, 0x29, 0x8a, 0x41, 0x3d, + 0x9c, 0x0b, 0x2a, 0xb6, 0x63, 0x73, 0xdc, 0xe9, 0xbf, 0x16, 0xbc, 0x31, 0x73, 0x29, 0xad, 0x3e, + 0x2b, 0x5b, 0xe8, 0x8f, 0xa0, 0xf6, 0x42, 0x0d, 0x86, 0x96, 0x88, 0xa4, 0xeb, 0x73, 0x65, 0xa9, + 0xcb, 0x73, 0x4a, 0x4f, 0x1c, 0x28, 0xa1, 0xee, 0x88, 0x0f, 0x34, 0xcc, 0x77, 0x16, 0xc0, 0xdc, + 0x35, 0xf6, 0x7a, 0x6e, 0x1a, 0x51, 0x81, 0xc1, 0x39, 0x6e, 0x2e, 0x05, 0x14, 0xd4, 0x44, 0x1c, + 0x73, 0xb8, 0xd3, 0x54, 0x0b, 0x60, 0xcb, 0x4c, 0x92, 0x31, 0x24, 0xb7, 0xf7, 0xe4, 0x47, 0x00, + 0xa9, 0xa9, 0x6e, 0xf7, 0x5b, 0xea, 0x33, 0x63, 0x4c, 0x9f, 0xc2, 0x96, 0x19, 0x73, 0x77, 0xd8, + 0xd0, 0x54, 0x4b, 0x2e, 0xad, 0x16, 0xda, 0x06, 0xfb, 0x39, 0x73, 0xd4, 0x55, 0x87, 0xdd, 0x6a, + 0x52, 0xa4, 0x25, 0xe5, 0xf2, 0x34, 0x88, 0xa4, 0x71, 0x51, 0xdf, 0x4a, 0xf7, 0x2c, 0x08, 0x25, + 0x22, 0xae, 0x32, 0xfc, 0xa6, 0xbf, 0x58, 0x00, 0xc7, 0x41, 0x4f, 0x74, 0x24, 0x97, 0xc3, 0x88, + 0x3c, 0xc4, 0xa8, 0x18, 0xab, 0xb2, 0x57, 0x4d, 0xcf, 0xf4, 0x9c, 0x39, 0x0c, 0xf7, 0x7b, 0x3f, + 0x73, 0x11, 0x4e, 0x4f, 0x98, 0x64, 0x89, 0x65, 0xae, 0xcb, 0x1d, 0x33, 0x50, 0x34, 0x55, 0xb5, + 0xd4, 0x3e, 0xd6, 0x6b, 0xd0, 0x9c, 0x1e, 0x43, 0xf5, 0xd0, 0x1b, 0x46, 0x52, 0x84, 0x1a, 0x8e, + 0xba, 0x49, 0x24, 0x97, 0x49, 0xfd, 0xa1, 0x40, 0xb6, 0x61, 0x09, 0x21, 0x0b, 0xa9, 0xfb, 0x76, + 0x02, 0xa8, 0x59, 0xa5, 0x1d, 0x28, 0xcc, 0x6f, 0x37, 0x02, 0x79, 0x7c, 0x83, 0x69, 0x86, 0xf0, + 0xf9, 0x55, 0x03, 0xfb, 0xc8, 0x8d, 0x53, 0x6a, 0x33, 0xf5, 0x89, 0x1a, 0x7e, 0x8d, 0x25, 0xa7, + 0x34, 0x5c, 0xdd, 0x3e, 0xab, 0x71, 0x0a, 0xd5, 0xb8, 0xbc, 0xcf, 0x3d, 0x61, 0x9e, 0x31, 0x76, + 0xe6, 0x19, 0xf3, 0x87, 0x05, 0xab, 0x4c, 0x44, 0xee, 0x4b, 0xe1, 0xf8, 0x91, 0x0c, 0x87, 0x49, + 0xfb, 0x7d, 0x1e, 0x9c, 0x3a, 0x2d, 0x8c, 0x6a, 0xb3, 0x58, 0x30, 0x39, 0xca, 0xcd, 0xcd, 0xd1, + 0xbb, 0xea, 0xe1, 0x1b, 0x84, 0x3d, 0xd5, 0x83, 0x41, 0xa8, 0x59, 0x9f, 0x30, 0xcc, 0x5a, 0x90, + 0xf7, 0x60, 0xa9, 0x13, 0x0c, 0xc3, 0x6e, 0x32, 0xa0, 0xd7, 0x53, 0xe3, 0x18, 0x55, 0xbc, 0xcc, + 0x8c, 0x19, 0xfd, 0xc9, 0x82, 0xe5, 0xec, 0xca, 0xe2, 0xc2, 0x49, 0x18, 0xca, 0xcd, 0x64, 0xc8, + 0x9e, 0xc5, 0x50, 0x3e, 0x65, 0x28, 0x7d, 0x54, 0x14, 0x32, 0x8f, 0x0a, 0x7a, 0x0e, 0x1b, 0x53, + 0xb4, 0x1d, 0x06, 0xfd, 0x81, 0xca, 0xcf, 0x7d, 0xe9, 0x5b, 0x83, 0x42, 0x3b, 0x0c, 0x35, 0x71, + 0x65, 0x16, 0x0b, 0x74, 0x1f, 0x4a, 0x27, 0xc1, 0x20, 0xf0, 0x82, 0xb3, 0x51, 0xb6, 0x00, 0xad, + 0xdb, 0x0a, 0xf0, 0x71, 0xed, 0xcf, 0x9b, 0x86, 0xf5, 0xd7, 0x4d, 0xc3, 0xfa, 0xfb, 0xa6, 0x61, + 0xfd, 0xf6, 0x4f, 0xe3, 0xb5, 0xd3, 0x22, 0xfe, 0xb2, 0xec, 0xbf, 0x0a, 0x00, 0x00, 0xff, 0xff, + 0x18, 0x9c, 0xa1, 0x75, 0xc3, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 8676a18c6..404b53e12 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -129,7 +129,7 @@ message NodeStatus { message ClusterStatus { string State = 1; - repeated URI URISet = 2; + repeated URI NodeSet = 2; } message Field { @@ -167,6 +167,6 @@ message ResizeInstructionComplete { } message Topology { - repeated URI URISet = 1; + repeated URI NodeSet = 1; } diff --git a/server.go b/server.go index 3139300c6..35cf79e4a 100644 --- a/server.go +++ b/server.go @@ -380,7 +380,7 @@ func (s *Server) LocalStatus() (proto.Message, error) { return &ns, nil } -// ClusterStatus returns the ClusterState and URISet for the cluster. +// ClusterStatus returns the ClusterState and NodeSet for the cluster. func (s *Server) ClusterStatus() (proto.Message, error) { return s.Cluster.Status(), nil } @@ -525,7 +525,7 @@ func CountOpenFiles() int { } // StatusHandler specifies the methods which an object must implement to share -// state in the cluster. These are used by the GossipNodeSet to implement the +// state in the cluster. These are used by the GossipMemberSet to implement the // LocalState and MergeRemoteState methods of memberlist.Delegate type StatusHandler interface { LocalStatus() (proto.Message, error) diff --git a/server/server.go b/server/server.go index 4a8f0558c..ad9507c0b 100644 --- a/server/server.go +++ b/server/server.go @@ -209,15 +209,15 @@ func (m *Command) SetupServer() error { // get the host portion of addr to use for binding gossipHost := uri.Host() m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipNodeSet := gossip.NewGossipNodeSet(uri.String(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) - m.Server.Cluster.NodeSet = gossipNodeSet - m.Server.Broadcaster = gossipNodeSet - m.Server.BroadcastReceiver = gossipNodeSet + gossipMemberSet := gossip.NewGossipMemberSet(uri.String(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) + m.Server.Cluster.MemberSet = gossipMemberSet + m.Server.Broadcaster = gossipMemberSet + m.Server.BroadcastReceiver = gossipMemberSet case pilosa.ClusterStatic, pilosa.ClusterNone: m.Server.Broadcaster = pilosa.NopBroadcaster - m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() + m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver - err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes) + err := m.Server.Cluster.MemberSet.(*pilosa.StaticMemberSet).Join(m.Server.Cluster.Nodes) if err != nil { return err } diff --git a/server/server_test.go b/server/server_test.go index 041026759..049801a30 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -445,18 +445,18 @@ func TestMain_SendReceiveMessage(t *testing.T) { } gossipSeed := gossipHost + ":" + freePorts[0] - topology := &pilosa.Topology{URISet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} + topology := &pilosa.Topology{NodeSet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} m0.Server.Cluster.Coordinator = m0.Server.URI m0.Server.Cluster.Topology = topology m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipNodeSet0 := gossip.NewGossipNodeSet(m0.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m0.Server, nil) - m0.Server.Cluster.NodeSet = gossipNodeSet0 - m0.Server.Broadcaster = gossipNodeSet0 + gossipMemberSet0 := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m0.Server, nil) + m0.Server.Cluster.MemberSet = gossipMemberSet0 + m0.Server.Broadcaster = gossipMemberSet0 m0.Server.Handler.Broadcaster = m0.Server.Broadcaster m0.Server.Holder.Broadcaster = m0.Server.Broadcaster - m0.Server.BroadcastReceiver = gossipNodeSet0 + m0.Server.BroadcastReceiver = gossipMemberSet0 if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { t.Fatal(err) @@ -481,12 +481,12 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Server.Cluster.Coordinator = m0.Server.URI m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipNodeSet1 := gossip.NewGossipNodeSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) - m1.Server.Cluster.NodeSet = gossipNodeSet1 - m1.Server.Broadcaster = gossipNodeSet1 + gossipMemberSet1 := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) + m1.Server.Cluster.MemberSet = gossipMemberSet1 + m1.Server.Broadcaster = gossipMemberSet1 m1.Server.Handler.Broadcaster = m1.Server.Broadcaster m1.Server.Holder.Broadcaster = m1.Server.Broadcaster - m1.Server.BroadcastReceiver = gossipNodeSet1 + m1.Server.BroadcastReceiver = gossipMemberSet1 if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { t.Fatal(err) diff --git a/test/handler.go b/test/handler.go index 6d08bc6c4..46172634c 100644 --- a/test/handler.go +++ b/test/handler.go @@ -85,8 +85,8 @@ func (s *Server) LocalStatus() (proto.Message, error) { func (s *Server) ClusterStatus() (proto.Message, error) { uri := pilosa.DefaultURI() return &internal.ClusterStatus{ - State: pilosa.NodeStateNormal, - URISet: []*internal.URI{uri.Encode()}, + State: pilosa.NodeStateNormal, + NodeSet: []*internal.URI{uri.Encode()}, }, nil } From ed7dcdcdeff91e038d290c1bf8a87303f8bd39c2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 13:09:27 -0500 Subject: [PATCH 010/311] rename NodeState to ClusterState --- cluster.go | 26 +++++++++++++------------- test/handler.go | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cluster.go b/cluster.go index f2ff60d9e..b3b1cae84 100644 --- a/cluster.go +++ b/cluster.go @@ -41,10 +41,10 @@ const ( // DefaultReplicaN is the default number of replicas per partition. DefaultReplicaN = 1 - // NodeState represents node state returned in /status endpoint for a node in the cluster. - NodeStateStarting = "STARTING" - NodeStateNormal = "NORMAL" - NodeStateResizing = "RESIZING" + // ClusterState represents the state returned in the /status endpoint. + ClusterStateStarting = "STARTING" + ClusterStateNormal = "NORMAL" + ClusterStateResizing = "RESIZING" // ResizeJob states. ResizeJobStateRunning = "RUNNING" @@ -499,7 +499,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { func (c *Cluster) Open() error { // Cluster always comes up in state STARTING until cluster membership is determined. - c.State = NodeStateStarting + c.State = ClusterStateStarting // Load topology file if it exists. if err := c.loadTopology(); err != nil { @@ -546,7 +546,7 @@ func (c *Cluster) Close() error { } func (c *Cluster) needTopologyAgreement() bool { - return c.State == NodeStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) + return c.State == ClusterStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } func (c *Cluster) haveTopologyAgreement() bool { @@ -605,7 +605,7 @@ func (c *Cluster) listenForJoins() { // Only change state to NORMAL if we have successfully added at least one host. if uriJoined { // Put the cluster back to state NORMAL and broadcast. - if err := c.setStateAndBroadcast(NodeStateNormal); err != nil { + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { c.logger().Printf("setStateAndBroadcast error: err=%s", err) } } @@ -1025,7 +1025,7 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { func (c *Cluster) considerTopology() (string, error) { // If there is no .topology file, it's safe to go to state NORMAL. if len(c.Topology.NodeSet) == 0 { - return NodeStateNormal, nil + return ClusterStateNormal, nil } // The local node (coordinator) must be in the .topology. @@ -1035,12 +1035,12 @@ func (c *Cluster) considerTopology() (string, error) { // If local node is the only thing in .topology, continue to state NORMAL. if len(c.Topology.NodeSet) == 1 { - return NodeStateNormal, nil + return ClusterStateNormal, nil } // Keep the cluster in state "STARTING" until hearing from all nodes. // Topology contains 2+ hosts. - return NodeStateStarting, nil + return ClusterStateStarting, nil } // ReceiveEvent represents an implementation of EventHandler. @@ -1071,7 +1071,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { // If the result of the previous AddNode completed the joining of nodes // in the topology, then change the state to NORMAL. if c.haveTopologyAgreement() { - return c.setStateAndBroadcast(NodeStateNormal) + return c.setStateAndBroadcast(ClusterStateNormal) } return nil @@ -1088,12 +1088,12 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { if err := c.AddNode(uri); err != nil { return err } - return c.setStateAndBroadcast(NodeStateNormal) + return c.setStateAndBroadcast(ClusterStateNormal) } // If the cluster has data, we need to change to RESIZING and // kick off the resizing process. - if err := c.setStateAndBroadcast(NodeStateResizing); err != nil { + if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return err } c.joiningURIs <- e.URI diff --git a/test/handler.go b/test/handler.go index 46172634c..e3bee5072 100644 --- a/test/handler.go +++ b/test/handler.go @@ -85,7 +85,7 @@ func (s *Server) LocalStatus() (proto.Message, error) { func (s *Server) ClusterStatus() (proto.Message, error) { uri := pilosa.DefaultURI() return &internal.ClusterStatus{ - State: pilosa.NodeStateNormal, + State: pilosa.ClusterStateNormal, NodeSet: []*internal.URI{uri.Encode()}, }, nil } From f2d28a1df9300a490b4482d8bce7f53c59a764fe Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 14:28:45 -0500 Subject: [PATCH 011/311] make note of a possible race condition --- cluster.go | 5 +++++ server.go | 38 +++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/cluster.go b/cluster.go index b3b1cae84..49f19fe5b 100644 --- a/cluster.go +++ b/cluster.go @@ -725,6 +725,11 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { srcURI := decodeURI(src.URI) + // TODO: there's a possible race condition here; + // if NodeStatus has not been shared with the joining + // node (and the schema created locally), then + // the following Frame() lookup could fail. + // Retrieve frame. f := c.Holder.Frame(src.Index, src.Frame) if f == nil { diff --git a/server.go b/server.go index 35cf79e4a..2c1bf3c69 100644 --- a/server.go +++ b/server.go @@ -396,6 +396,25 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return nil } + // Sync schema. + // Create indexes that don't exist. + for _, index := range ns.Schema.Indexes { + opt := IndexOptions{} + idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range index.Frames { + opt := decodeFrameOptions(f.Meta) + _, err := idx.CreateFrameIfNotExists(f.Name, *opt) + if err != nil { + return err + } + } + // TODO: Create inputDefinitions that don't exist. + } + // Sync maxSlices (standard). oldmaxslices := s.Holder.MaxSlices() for index, newMax := range ns.MaxSlices.Standard { @@ -428,25 +447,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { } } - // Sync schema. - // Create indexes that don't exist. - for _, index := range ns.Schema.Indexes { - opt := IndexOptions{} - idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range index.Frames { - opt := decodeFrameOptions(f.Meta) - _, err := idx.CreateFrameIfNotExists(f.Name, *opt) - if err != nil { - return err - } - } - // TODO: Create inputDefinitions that don't exist. - } - return nil } From 3539c6ffb5ab89197cbfe98c37f4c42a381d03a7 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 30 Oct 2017 15:28:12 -0500 Subject: [PATCH 012/311] Fixed overwriting standard with inverse max slice --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index 2c1bf3c69..9327e8ca8 100644 --- a/server.go +++ b/server.go @@ -443,7 +443,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { } if newMaxInverse > oldMaxInverseSlices[index] { oldMaxInverseSlices[index] = newMaxInverse - localIndex.SetRemoteMaxSlice(newMaxInverse) + localIndex.SetRemoteMaxInverseSlice(newMaxInverse) } } From fb08fd39020fbc0fb392714c960c55e583807289 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Oct 2017 15:50:22 -0500 Subject: [PATCH 013/311] endpoint to abort a cluster resize that is in progress --- cluster.go | 13 +++++++++---- handler.go | 30 ++++++++++++++++++++++++++++++ handler_test.go | 18 ++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 49f19fe5b..092233055 100644 --- a/cluster.go +++ b/cluster.go @@ -569,11 +569,15 @@ func (c *Cluster) handleJoiningHost(uri URI) error { jobResult := <-j.result switch jobResult { case ResizeJobStateDone: - c.CompleteCurrentJob(ResizeJobStateDone) + if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { + return err + } // Add uri to the cluster. return c.AddNode(uri) case ResizeJobStateAborted: - c.CompleteCurrentJob(ResizeJobStateAborted) + if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { + return err + } } return nil } @@ -691,14 +695,15 @@ func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { // CompleteCurrentJob sets the state of the current ResizeJob // then removes the pointer to currentJob. -func (c *Cluster) CompleteCurrentJob(state string) { +func (c *Cluster) CompleteCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() if c.currentJob == nil { - return + return fmt.Errorf("no resize job currently running") } c.currentJob.SetState(state) c.currentJob = nil + return nil } // followResizeInstruction is run by any node that receives a ResizeInstruction. diff --git a/handler.go b/handler.go index 0aeb9a441..88238b9ea 100644 --- a/handler.go +++ b/handler.go @@ -98,6 +98,7 @@ func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() router.HandleFunc("/", handler.handleWebUI).Methods("GET") router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET") @@ -1891,6 +1892,35 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } +//handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + var msg string + + if err := func() error { + if !h.Cluster.IsCoordinator() { + return fmt.Errorf("abort requests must be made on the coordinator node") + } + err := h.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + if err != nil { + return err + } + return nil + }(); err != nil { + msg = err.Error() + } + + // Encode response. + if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ + Info: msg, + }); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type clusterResizeAbortResponse struct { + Info string `json:"info"` +} + // InputJSONDataParser validates input json file and executes SetBit. func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef, err := index.InputDefinition(name) diff --git a/handler_test.go b/handler_test.go index 4c4b4c5ca..d6a5e7a3f 100644 --- a/handler_test.go +++ b/handler_test.go @@ -152,6 +152,24 @@ func TestHandler_Status(t *testing.T) { } } +// Ensure the handler can abort a cluster resize. +func TestHandler_ClusterResizeAbort(t *testing.T) { + + t.Run("No resize job", func(t *testing.T) { + h := test.NewHandler() + h.Cluster = test.NewCluster(1) + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"info":"no resize job currently running"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + +} + // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() From 9b7adde692da5256cedeaaf4c3e36269c398208e Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 31 Oct 2017 11:32:08 -0500 Subject: [PATCH 014/311] added endpoint protection for cluster resize --- cluster.go | 16 +++++++++++++ handler.go | 57 ++++++++++++++++++++++++++++++++++++--------- security_manager.go | 22 +++++++++++++++++ 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 security_manager.go diff --git a/cluster.go b/cluster.go index 092233055..a746a509f 100644 --- a/cluster.go +++ b/cluster.go @@ -167,6 +167,7 @@ type Cluster struct { // Close management wg sync.WaitGroup closing chan struct{} + prefect SecurityManager // The writer for any logging. LogOutput io.Writer @@ -185,6 +186,7 @@ func NewCluster() *Cluster { closing: make(chan struct{}), LogOutput: os.Stderr, + prefect: &DefaultSecurityManager{}, } } @@ -226,6 +228,20 @@ func (c *Cluster) NodeSet() []URI { } func (c *Cluster) setState(state string) { + if c.State != state { //only on new state, perform routing change + switch state { + case ClusterStateResizing: + c.prefect.SetRestricted() + case ClusterStateNormal: + c.prefect.SetNormal() + // Don't change routing for these new states + // ClusterStateStarting + // ResizeJobStateRunning + // ResizeJobStateDone + // ResizeJobStateAborted + + } + } c.State = state } diff --git a/handler.go b/handler.go index 88238b9ea..a33551abb 100644 --- a/handler.go +++ b/handler.go @@ -60,7 +60,9 @@ type Handler struct { Cluster *Cluster ClientOptions *ClientOptions - Router *mux.Router + Router *mux.Router + NormalRouter *mux.Router + RestrictedRouter *mux.Router // The execution engine for running queries. Executor interface { @@ -89,14 +91,49 @@ func NewHandler() *Handler { handler := &Handler{ LogOutput: os.Stderr, } - handler.Router = NewRouter(handler) + BuildRouters(handler) return handler } -// NewRouter creates a Gorilla Mux http router. -func NewRouter(handler *Handler) *mux.Router { +// BuildRouters creates a Gorilla Mux http routers for both normal and restricted enpoints. +func BuildRouters(handler *Handler) { router := mux.NewRouter() - router.HandleFunc("/", handler.handleWebUI).Methods("GET") + loadCommon(router, handler) + loadNormal(router, handler) + handler.NormalRouter = router + router = mux.NewRouter() + loadCommon(router, handler) + loadRestricted(router, handler) + handler.RestrictedRouter = router + handler.SetNormal() +} + +// SetNormal a method of the SecurityManager interface which provides normal URI routing +func (h *Handler) SetNormal() { + h.Router = h.NormalRouter +} + +// SetRestricted a method of the SecurityManager interface which provides restricted URI routing +func (h *Handler) SetRestricted() { + h.Router = h.RestrictedRouter +} + +func loadCommon(router *mux.Router, handler *Handler) { + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") + router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) + router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") +} + +func loadRestricted(router *mux.Router, handler *Handler) { + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") + router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted) +} + +func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") @@ -131,11 +168,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") - router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") // TODO: Apply MethodNotAllowed statuses to all endpoints. @@ -144,7 +176,10 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - return router +} + +func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { + http.Error(w, "not allowed during resize", http.StatusMethodNotAllowed) } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { diff --git a/security_manager.go b/security_manager.go new file mode 100644 index 000000000..88022de03 --- /dev/null +++ b/security_manager.go @@ -0,0 +1,22 @@ +package pilosa + +// SecurityManager provides the ability to limit access to restricted endpoints +// during cluster configuration +type SecurityManager interface { + SetRestricted() + SetNormal() +} + +// DefaultSecurityManager provides a no-op implimentation of the SecurityManager interface +type DefaultSecurityManager struct { +} + +// SetRestricted no-op +func (sdm *DefaultSecurityManager) SetRestricted() { + +} + +// SetNormal no-op +func (sdm *DefaultSecurityManager) SetNormal() { + +} From fc829b08e83c1dab58bae60a26e7892b3f53e261 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 31 Oct 2017 12:12:20 -0500 Subject: [PATCH 015/311] made endpoint available for resize --- handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler.go b/handler.go index a33551abb..808680a54 100644 --- a/handler.go +++ b/handler.go @@ -125,6 +125,7 @@ func loadCommon(router *mux.Router, handler *Handler) { router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) + router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") } @@ -141,7 +142,6 @@ func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/export", handler.handleGetExport).Methods("GET") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET") - router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET") router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST") router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") From da0184f7282bb82c17f88f279e15687f7ae077e6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 31 Oct 2017 13:56:58 -0500 Subject: [PATCH 016/311] Adjust comments. Change DefaultSecurityManager to NopSecurityManager. --- cluster.go | 28 ++++++++++++++-------------- handler.go | 10 +++++++--- security_manager.go | 14 +++++++------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/cluster.go b/cluster.go index a746a509f..8860d8405 100644 --- a/cluster.go +++ b/cluster.go @@ -186,7 +186,7 @@ func NewCluster() *Cluster { closing: make(chan struct{}), LogOutput: os.Stderr, - prefect: &DefaultSecurityManager{}, + prefect: &NopSecurityManager{}, } } @@ -228,20 +228,20 @@ func (c *Cluster) NodeSet() []URI { } func (c *Cluster) setState(state string) { - if c.State != state { //only on new state, perform routing change - switch state { - case ClusterStateResizing: - c.prefect.SetRestricted() - case ClusterStateNormal: - c.prefect.SetNormal() - // Don't change routing for these new states - // ClusterStateStarting - // ResizeJobStateRunning - // ResizeJobStateDone - // ResizeJobStateAborted - - } + // Ignore cases where the state hasn't changed. + if state == c.State { + return } + + switch state { + case ClusterStateResizing: + c.prefect.SetRestricted() + case ClusterStateNormal: + c.prefect.SetNormal() + // Don't change routing for these states: + // - ClusterStateStarting + } + c.State = state } diff --git a/handler.go b/handler.go index 808680a54..55b6e91d9 100644 --- a/handler.go +++ b/handler.go @@ -95,25 +95,29 @@ func NewHandler() *Handler { return handler } -// BuildRouters creates a Gorilla Mux http routers for both normal and restricted enpoints. +// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints. func BuildRouters(handler *Handler) { + // Normal router. router := mux.NewRouter() loadCommon(router, handler) loadNormal(router, handler) handler.NormalRouter = router + + // Restricted router. router = mux.NewRouter() loadCommon(router, handler) loadRestricted(router, handler) handler.RestrictedRouter = router + handler.SetNormal() } -// SetNormal a method of the SecurityManager interface which provides normal URI routing +// SetNormal is a method of the SecurityManager interface which provides normal URI routing. func (h *Handler) SetNormal() { h.Router = h.NormalRouter } -// SetRestricted a method of the SecurityManager interface which provides restricted URI routing +// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing. func (h *Handler) SetRestricted() { h.Router = h.RestrictedRouter } diff --git a/security_manager.go b/security_manager.go index 88022de03..2acf82986 100644 --- a/security_manager.go +++ b/security_manager.go @@ -1,22 +1,22 @@ package pilosa // SecurityManager provides the ability to limit access to restricted endpoints -// during cluster configuration +// during cluster configuration. type SecurityManager interface { SetRestricted() SetNormal() } -// DefaultSecurityManager provides a no-op implimentation of the SecurityManager interface -type DefaultSecurityManager struct { +// NopSecurityManager provides a no-op implementation of the SecurityManager interface. +type NopSecurityManager struct { } -// SetRestricted no-op -func (sdm *DefaultSecurityManager) SetRestricted() { +// SetRestricted no-op. +func (sdm *NopSecurityManager) SetRestricted() { } -// SetNormal no-op -func (sdm *DefaultSecurityManager) SetNormal() { +// SetNormal no-op. +func (sdm *NopSecurityManager) SetNormal() { } From 7d300e708042b1029b4b2a53f791b9df70f0ae86 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 3 Nov 2017 11:21:53 -0500 Subject: [PATCH 017/311] include Schema in pb.ResizeInstructions --- cluster.go | 38 +++++++- internal/private.pb.go | 209 ++++++++++++++++++++++++++--------------- internal/private.proto | 1 + 3 files changed, 166 insertions(+), 82 deletions(-) diff --git a/cluster.go b/cluster.go index 8860d8405..2841bd5bc 100644 --- a/cluster.go +++ b/cluster.go @@ -684,6 +684,8 @@ func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { toCluster.ReplicaN = c.ReplicaN toCluster.AddNodeBasicSorted(addURI) + pbSchema := c.Holder.EncodeSchema() + // Add to the ResizeJob the instructions for each index. for _, idx := range c.Holder.Indexes() { // dataDiff is map[string][]*internal.ResizeSource, where string is @@ -696,11 +698,14 @@ func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { j.URIs[uri] = true continue } + // TODO: we can probably consilidate the instructions that go to the same + // node but apply to different indexes. (i.e. don't nest this in the Indexes() loop) instr := &internal.ResizeInstruction{ JobID: j.ID, URI: uri.Encode(), Coordinator: encodeURI(c.Coordinator), Sources: sources, + Schema: pbSchema, } j.Instructions = append(j.Instructions, instr) } @@ -734,6 +739,34 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { // Stop processing on any error. if err := func() error { + + // TODO: move this schema creation code to a method on Holder. + // Sync the schema received in the resize instruction. + // Create indexes that don't exist. + for _, index := range instr.Schema.Indexes { + opt := IndexOptions{} + idx, err := c.Holder.CreateIndexIfNotExists(index.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range index.Frames { + opt := decodeFrameOptions(f.Meta) + frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) + if err != nil { + return err + } + // Create views that don't exist. + for _, v := range f.Views { + _, err := frame.CreateViewIfNotExists(v) + if err != nil { + return err + } + } + } + // TODO: Create inputDefinitions that don't exist. + } + // Create a client for calling remote nodes. client, err := NewClientFromURI(&c.URI, nil) // TODO: ClientOptions if err != nil { @@ -746,11 +779,6 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { srcURI := decodeURI(src.URI) - // TODO: there's a possible race condition here; - // if NodeStatus has not been shared with the joining - // node (and the schema created locally), then - // the following Frame() lookup could fail. - // Retrieve frame. f := c.Holder.Frame(src.Index, src.Frame) if f == nil { diff --git a/internal/private.pb.go b/internal/private.pb.go index 421179ca6..2c5355f69 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -802,6 +802,7 @@ type ResizeInstruction struct { URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` Coordinator *URI `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` + Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` } func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } @@ -837,6 +838,13 @@ func (m *ResizeInstruction) GetSources() []*ResizeSource { return nil } +func (m *ResizeInstruction) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + type ResizeSource struct { URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -2026,6 +2034,16 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { i += n } } + if m.Schema != nil { + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n16, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 + } return i, nil } @@ -2048,11 +2066,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n16, err := m.URI.MarshalTo(dAtA[i:]) + n17, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n17 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2104,11 +2122,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n17, err := m.URI.MarshalTo(dAtA[i:]) + n18, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n18 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2623,6 +2641,10 @@ func (m *ResizeInstruction) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -6292,6 +6314,39 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -6822,77 +6877,77 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1144 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0x66, 0x3c, 0xb6, 0x63, 0x97, 0xe3, 0x8d, 0xd3, 0x84, 0xc8, 0x89, 0x22, 0xaf, 0xe9, 0x03, - 0x09, 0x2b, 0x11, 0x20, 0x91, 0x10, 0x04, 0x21, 0xc1, 0xc6, 0x5e, 0xed, 0x00, 0x09, 0x4b, 0x3b, - 0xbb, 0x48, 0x1c, 0x90, 0x3a, 0x76, 0x93, 0x8c, 0x32, 0x9e, 0x31, 0x33, 0xed, 0x24, 0xde, 0x03, - 0x37, 0x38, 0xc0, 0x0b, 0x70, 0xe7, 0xcc, 0x7b, 0x70, 0xe4, 0x11, 0x50, 0x78, 0x08, 0x24, 0x2e, - 0xac, 0xba, 0xa6, 0x7b, 0x66, 0xfc, 0x17, 0x2b, 0xb9, 0x4d, 0x55, 0x57, 0x55, 0x7f, 0xfd, 0xd5, - 0x4f, 0xf7, 0x40, 0x75, 0x10, 0xba, 0x97, 0x5c, 0x8a, 0xdd, 0x41, 0x18, 0xc8, 0x80, 0x94, 0x5c, - 0x5f, 0x8a, 0xd0, 0xe7, 0x1e, 0xfd, 0x0a, 0xca, 0x8e, 0xdf, 0x13, 0xd7, 0x47, 0x42, 0x72, 0xd2, - 0x84, 0xca, 0x61, 0xe0, 0x0d, 0xfb, 0xfe, 0x97, 0xfc, 0x54, 0x78, 0x75, 0xab, 0x69, 0xed, 0x94, - 0x59, 0x56, 0xa5, 0x2c, 0x4e, 0xdc, 0xbe, 0xf8, 0x7a, 0xc8, 0x7d, 0x39, 0xec, 0xd7, 0x73, 0xb1, - 0x45, 0x46, 0x45, 0xff, 0xb3, 0xa0, 0xfc, 0x24, 0xe4, 0x7d, 0x81, 0x11, 0x37, 0xa1, 0xc4, 0x82, - 0xab, 0x6c, 0xb8, 0x44, 0x26, 0x6f, 0xc1, 0x03, 0xc7, 0xbf, 0x14, 0x61, 0x24, 0xda, 0x3e, 0x3f, - 0xf5, 0x44, 0x0f, 0xc3, 0x95, 0xd8, 0x84, 0x96, 0x6c, 0x41, 0xf9, 0x90, 0x77, 0xcf, 0xc5, 0xc9, - 0x68, 0x20, 0xea, 0x36, 0x06, 0x49, 0x15, 0xc9, 0x6a, 0xc7, 0x7d, 0x29, 0xea, 0xf9, 0xa6, 0xb5, - 0x53, 0x65, 0xa9, 0x62, 0x12, 0x6f, 0x61, 0x0a, 0x2f, 0xa1, 0xb0, 0xcc, 0xb8, 0x7f, 0x96, 0x60, - 0x28, 0x22, 0x86, 0x31, 0x1d, 0xd9, 0x86, 0xe2, 0x13, 0x57, 0x78, 0xbd, 0xa8, 0xbe, 0xd4, 0xb4, - 0x77, 0x2a, 0x7b, 0x2b, 0xbb, 0x86, 0xbf, 0x5d, 0xd4, 0x33, 0xbd, 0x4c, 0x29, 0x3c, 0x70, 0xfa, - 0x83, 0x20, 0x94, 0x4c, 0x44, 0x83, 0xc0, 0x8f, 0x04, 0xa9, 0x81, 0xdd, 0x0e, 0x43, 0x7d, 0x76, - 0xf5, 0x49, 0x7f, 0x84, 0xda, 0x63, 0x2f, 0xe8, 0x5e, 0xb4, 0xb8, 0xe4, 0x4c, 0xfc, 0x30, 0x14, - 0x91, 0x24, 0x6b, 0x50, 0xc0, 0x2c, 0x68, 0xbb, 0x58, 0x50, 0x5a, 0x64, 0x52, 0xd3, 0x1c, 0x0b, - 0x4a, 0x8b, 0xfe, 0x48, 0x45, 0x9e, 0xc5, 0x82, 0xd2, 0x76, 0x3c, 0xb7, 0x1b, 0x53, 0x90, 0x67, - 0xb1, 0x40, 0x08, 0xe4, 0x5f, 0xb8, 0xe2, 0x4a, 0x9f, 0x1b, 0xbf, 0xa9, 0x03, 0xab, 0x99, 0xfd, - 0x35, 0xcc, 0x75, 0x28, 0xb2, 0xe0, 0xca, 0x69, 0x45, 0x75, 0xab, 0x69, 0xef, 0xe4, 0x99, 0x96, - 0x90, 0x5d, 0x4c, 0xbf, 0x5a, 0xca, 0xe1, 0x52, 0xaa, 0xa0, 0x1b, 0x50, 0x40, 0xaa, 0xd5, 0x29, - 0x53, 0x5f, 0xf5, 0x49, 0xff, 0xb7, 0xa0, 0x7c, 0xc4, 0xaf, 0x11, 0x46, 0x44, 0x3e, 0x81, 0x52, - 0x47, 0x72, 0xbf, 0xc7, 0xc3, 0x1e, 0x1a, 0x55, 0xf6, 0xde, 0x4c, 0x29, 0x4c, 0xcc, 0x76, 0x8d, - 0x4d, 0xdb, 0x97, 0xe1, 0x88, 0x25, 0x2e, 0xe4, 0x00, 0x96, 0x74, 0x4d, 0x20, 0x86, 0xca, 0x5e, - 0x73, 0x96, 0x77, 0x52, 0x36, 0xca, 0xd9, 0x38, 0x6c, 0x7e, 0x0c, 0xd5, 0xb1, 0xb0, 0x0a, 0xeb, - 0x85, 0x18, 0x99, 0x8c, 0x5c, 0x88, 0x91, 0xe2, 0xee, 0x92, 0x7b, 0xc3, 0x98, 0xe7, 0x3c, 0x8b, - 0x85, 0x83, 0xdc, 0x87, 0xd6, 0xe6, 0x01, 0x2c, 0x67, 0xa3, 0xde, 0xc5, 0x97, 0x7e, 0x07, 0xe4, - 0x30, 0x14, 0x5c, 0x0a, 0x84, 0x77, 0x24, 0xa2, 0x88, 0x9f, 0x89, 0xf9, 0x99, 0x8e, 0xb3, 0x97, - 0xcb, 0x66, 0x6f, 0x0b, 0xca, 0x4e, 0x64, 0x0e, 0x6e, 0x63, 0x5d, 0xa6, 0x0a, 0xfa, 0x08, 0x48, - 0x4b, 0x78, 0x42, 0x0a, 0xdd, 0xbf, 0xb7, 0xc4, 0xa7, 0x1d, 0x83, 0x65, 0xb1, 0x2d, 0xd9, 0x86, - 0xbc, 0x6a, 0x5d, 0x84, 0x52, 0xd9, 0x7b, 0x3d, 0x65, 0x3a, 0x99, 0x13, 0x0c, 0x0d, 0xa8, 0x6b, - 0x82, 0xea, 0x76, 0x5f, 0x70, 0xc0, 0x19, 0xa5, 0x6c, 0xb6, 0xb2, 0x27, 0xb7, 0x4a, 0x06, 0x88, - 0xde, 0xea, 0x53, 0x73, 0xd6, 0xfb, 0x6e, 0x45, 0xbf, 0xd5, 0x5a, 0xd5, 0x12, 0xc7, 0x6a, 0x35, - 0xf6, 0xc1, 0xef, 0xf9, 0x47, 0x9e, 0xc0, 0xa1, 0x62, 0xab, 0x1e, 0x8a, 0xea, 0x76, 0xd3, 0x56, - 0xb1, 0x51, 0xa0, 0xfb, 0x50, 0xec, 0x74, 0xcf, 0x45, 0x9f, 0x93, 0xb7, 0x55, 0xa1, 0xf6, 0xc4, - 0xb5, 0x88, 0x74, 0x99, 0xaf, 0x4c, 0xd0, 0xc7, 0xcc, 0x3a, 0xfd, 0xd5, 0xd2, 0xe8, 0xe7, 0x20, - 0x2a, 0xe2, 0xde, 0x51, 0x3d, 0x3f, 0x35, 0x71, 0x94, 0x9e, 0xe9, 0x65, 0xd2, 0x86, 0x9a, 0xe3, - 0x0f, 0x86, 0xb2, 0x25, 0xbe, 0x77, 0x7d, 0x57, 0xba, 0x81, 0x1f, 0xd5, 0x8b, 0xe8, 0xb2, 0x91, - 0xdd, 0x7a, 0xcc, 0x82, 0x4d, 0xb9, 0xd0, 0x9f, 0x2d, 0x58, 0x99, 0x50, 0x2e, 0xc0, 0x95, 0xbb, - 0x1d, 0xd7, 0x07, 0xc9, 0xc8, 0xb4, 0xd1, 0xb0, 0x31, 0x17, 0xcd, 0xf8, 0x04, 0xfd, 0xdd, 0x82, - 0xb5, 0x59, 0x06, 0x33, 0xd1, 0x34, 0x00, 0x9e, 0x85, 0x6e, 0x9f, 0x87, 0xa3, 0x2f, 0xc4, 0x48, - 0xdf, 0x1e, 0x19, 0x0d, 0xf9, 0x06, 0xd6, 0x27, 0x62, 0x7d, 0xd6, 0x8d, 0x29, 0x8a, 0x41, 0x3d, - 0x9c, 0x0b, 0x2a, 0xb6, 0x63, 0x73, 0xdc, 0xe9, 0xbf, 0x16, 0xbc, 0x31, 0x73, 0x29, 0xad, 0x3e, - 0x2b, 0x5b, 0xe8, 0x8f, 0xa0, 0xf6, 0x42, 0x0d, 0x86, 0x96, 0x88, 0xa4, 0xeb, 0x73, 0x65, 0xa9, - 0xcb, 0x73, 0x4a, 0x4f, 0x1c, 0x28, 0xa1, 0xee, 0x88, 0x0f, 0x34, 0xcc, 0x77, 0x16, 0xc0, 0xdc, - 0x35, 0xf6, 0x7a, 0x6e, 0x1a, 0x51, 0x81, 0xc1, 0x39, 0x6e, 0x2e, 0x05, 0x14, 0xd4, 0x44, 0x1c, - 0x73, 0xb8, 0xd3, 0x54, 0x0b, 0x60, 0xcb, 0x4c, 0x92, 0x31, 0x24, 0xb7, 0xf7, 0xe4, 0x47, 0x00, - 0xa9, 0xa9, 0x6e, 0xf7, 0x5b, 0xea, 0x33, 0x63, 0x4c, 0x9f, 0xc2, 0x96, 0x19, 0x73, 0x77, 0xd8, - 0xd0, 0x54, 0x4b, 0x2e, 0xad, 0x16, 0xda, 0x06, 0xfb, 0x39, 0x73, 0xd4, 0x55, 0x87, 0xdd, 0x6a, - 0x52, 0xa4, 0x25, 0xe5, 0xf2, 0x34, 0x88, 0xa4, 0x71, 0x51, 0xdf, 0x4a, 0xf7, 0x2c, 0x08, 0x25, - 0x22, 0xae, 0x32, 0xfc, 0xa6, 0xbf, 0x58, 0x00, 0xc7, 0x41, 0x4f, 0x74, 0x24, 0x97, 0xc3, 0x88, - 0x3c, 0xc4, 0xa8, 0x18, 0xab, 0xb2, 0x57, 0x4d, 0xcf, 0xf4, 0x9c, 0x39, 0x0c, 0xf7, 0x7b, 0x3f, - 0x73, 0x11, 0x4e, 0x4f, 0x98, 0x64, 0x89, 0x65, 0xae, 0xcb, 0x1d, 0x33, 0x50, 0x34, 0x55, 0xb5, - 0xd4, 0x3e, 0xd6, 0x6b, 0xd0, 0x9c, 0x1e, 0x43, 0xf5, 0xd0, 0x1b, 0x46, 0x52, 0x84, 0x1a, 0x8e, - 0xba, 0x49, 0x24, 0x97, 0x49, 0xfd, 0xa1, 0x40, 0xb6, 0x61, 0x09, 0x21, 0x0b, 0xa9, 0xfb, 0x76, - 0x02, 0xa8, 0x59, 0xa5, 0x1d, 0x28, 0xcc, 0x6f, 0x37, 0x02, 0x79, 0x7c, 0x83, 0x69, 0x86, 0xf0, - 0xf9, 0x55, 0x03, 0xfb, 0xc8, 0x8d, 0x53, 0x6a, 0x33, 0xf5, 0x89, 0x1a, 0x7e, 0x8d, 0x25, 0xa7, - 0x34, 0x5c, 0xdd, 0x3e, 0xab, 0x71, 0x0a, 0xd5, 0xb8, 0xbc, 0xcf, 0x3d, 0x61, 0x9e, 0x31, 0x76, - 0xe6, 0x19, 0xf3, 0x87, 0x05, 0xab, 0x4c, 0x44, 0xee, 0x4b, 0xe1, 0xf8, 0x91, 0x0c, 0x87, 0x49, - 0xfb, 0x7d, 0x1e, 0x9c, 0x3a, 0x2d, 0x8c, 0x6a, 0xb3, 0x58, 0x30, 0x39, 0xca, 0xcd, 0xcd, 0xd1, - 0xbb, 0xea, 0xe1, 0x1b, 0x84, 0x3d, 0xd5, 0x83, 0x41, 0xa8, 0x59, 0x9f, 0x30, 0xcc, 0x5a, 0x90, - 0xf7, 0x60, 0xa9, 0x13, 0x0c, 0xc3, 0x6e, 0x32, 0xa0, 0xd7, 0x53, 0xe3, 0x18, 0x55, 0xbc, 0xcc, - 0x8c, 0x19, 0xfd, 0xc9, 0x82, 0xe5, 0xec, 0xca, 0xe2, 0xc2, 0x49, 0x18, 0xca, 0xcd, 0x64, 0xc8, - 0x9e, 0xc5, 0x50, 0x3e, 0x65, 0x28, 0x7d, 0x54, 0x14, 0x32, 0x8f, 0x0a, 0x7a, 0x0e, 0x1b, 0x53, - 0xb4, 0x1d, 0x06, 0xfd, 0x81, 0xca, 0xcf, 0x7d, 0xe9, 0x5b, 0x83, 0x42, 0x3b, 0x0c, 0x35, 0x71, - 0x65, 0x16, 0x0b, 0x74, 0x1f, 0x4a, 0x27, 0xc1, 0x20, 0xf0, 0x82, 0xb3, 0x51, 0xb6, 0x00, 0xad, - 0xdb, 0x0a, 0xf0, 0x71, 0xed, 0xcf, 0x9b, 0x86, 0xf5, 0xd7, 0x4d, 0xc3, 0xfa, 0xfb, 0xa6, 0x61, - 0xfd, 0xf6, 0x4f, 0xe3, 0xb5, 0xd3, 0x22, 0xfe, 0xb2, 0xec, 0xbf, 0x0a, 0x00, 0x00, 0xff, 0xff, - 0x18, 0x9c, 0xa1, 0x75, 0xc3, 0x0c, 0x00, 0x00, + // 1149 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4f, 0x6f, 0xe3, 0x44, + 0x14, 0xc7, 0x71, 0x92, 0x26, 0x2f, 0xcd, 0x36, 0x1d, 0x4a, 0x95, 0x56, 0x55, 0x36, 0xcc, 0x81, + 0x96, 0x95, 0x28, 0xd0, 0x4a, 0x08, 0x8a, 0x90, 0x60, 0x9b, 0xac, 0xd6, 0x40, 0xcb, 0x32, 0xe9, + 0x2e, 0x12, 0x07, 0xa4, 0x69, 0x32, 0xb4, 0x56, 0x1d, 0x3b, 0xd8, 0x93, 0xb6, 0xd9, 0x03, 0x37, + 0x38, 0xc0, 0x17, 0xe0, 0xce, 0x97, 0xe1, 0xc8, 0x8d, 0x2b, 0x2a, 0x1f, 0x02, 0x89, 0x0b, 0xab, + 0x79, 0x9e, 0xb1, 0x9d, 0x7f, 0x8d, 0xda, 0x9b, 0xdf, 0x9b, 0xf7, 0xde, 0xfc, 0xe6, 0xf7, 0xfe, + 0xcc, 0x18, 0xaa, 0x83, 0xd0, 0xbd, 0xe4, 0x52, 0xec, 0x0e, 0xc2, 0x40, 0x06, 0xa4, 0xe4, 0xfa, + 0x52, 0x84, 0x3e, 0xf7, 0xe8, 0x57, 0x50, 0x76, 0xfc, 0x9e, 0xb8, 0x3e, 0x12, 0x92, 0x93, 0x26, + 0x54, 0x0e, 0x03, 0x6f, 0xd8, 0xf7, 0xbf, 0xe4, 0xa7, 0xc2, 0xab, 0x5b, 0x4d, 0x6b, 0xa7, 0xcc, + 0xb2, 0x2a, 0x65, 0x71, 0xe2, 0xf6, 0xc5, 0xd7, 0x43, 0xee, 0xcb, 0x61, 0xbf, 0x9e, 0x8b, 0x2d, + 0x32, 0x2a, 0xfa, 0x9f, 0x05, 0xe5, 0x27, 0x21, 0xef, 0x0b, 0x8c, 0xb8, 0x09, 0x25, 0x16, 0x5c, + 0x65, 0xc3, 0x25, 0x32, 0x79, 0x0b, 0x1e, 0x38, 0xfe, 0xa5, 0x08, 0x23, 0xd1, 0xf6, 0xf9, 0xa9, + 0x27, 0x7a, 0x18, 0xae, 0xc4, 0x26, 0xb4, 0x64, 0x0b, 0xca, 0x87, 0xbc, 0x7b, 0x2e, 0x4e, 0x46, + 0x03, 0x51, 0xb7, 0x31, 0x48, 0xaa, 0x48, 0x56, 0x3b, 0xee, 0x4b, 0x51, 0xcf, 0x37, 0xad, 0x9d, + 0x2a, 0x4b, 0x15, 0x93, 0x78, 0x0b, 0x53, 0x78, 0x09, 0x85, 0x65, 0xc6, 0xfd, 0xb3, 0x04, 0x43, + 0x11, 0x31, 0x8c, 0xe9, 0xc8, 0x36, 0x14, 0x9f, 0xb8, 0xc2, 0xeb, 0x45, 0xf5, 0xa5, 0xa6, 0xbd, + 0x53, 0xd9, 0x5b, 0xd9, 0x35, 0xfc, 0xed, 0xa2, 0x9e, 0xe9, 0x65, 0x4a, 0xe1, 0x81, 0xd3, 0x1f, + 0x04, 0xa1, 0x64, 0x22, 0x1a, 0x04, 0x7e, 0x24, 0x48, 0x0d, 0xec, 0x76, 0x18, 0xea, 0xb3, 0xab, + 0x4f, 0xfa, 0x23, 0xd4, 0x1e, 0x7b, 0x41, 0xf7, 0xa2, 0xc5, 0x25, 0x67, 0xe2, 0x87, 0xa1, 0x88, + 0x24, 0x59, 0x83, 0x02, 0x66, 0x41, 0xdb, 0xc5, 0x82, 0xd2, 0x22, 0x93, 0x9a, 0xe6, 0x58, 0x50, + 0x5a, 0xf4, 0x47, 0x2a, 0xf2, 0x2c, 0x16, 0x94, 0xb6, 0xe3, 0xb9, 0xdd, 0x98, 0x82, 0x3c, 0x8b, + 0x05, 0x42, 0x20, 0xff, 0xc2, 0x15, 0x57, 0xfa, 0xdc, 0xf8, 0x4d, 0x1d, 0x58, 0xcd, 0xec, 0xaf, + 0x61, 0xae, 0x43, 0x91, 0x05, 0x57, 0x4e, 0x2b, 0xaa, 0x5b, 0x4d, 0x7b, 0x27, 0xcf, 0xb4, 0x84, + 0xec, 0x62, 0xfa, 0xd5, 0x52, 0x0e, 0x97, 0x52, 0x05, 0xdd, 0x80, 0x02, 0x52, 0xad, 0x4e, 0x99, + 0xfa, 0xaa, 0x4f, 0xfa, 0xbf, 0x05, 0xe5, 0x23, 0x7e, 0x8d, 0x30, 0x22, 0xf2, 0x09, 0x94, 0x3a, + 0x92, 0xfb, 0x3d, 0x1e, 0xf6, 0xd0, 0xa8, 0xb2, 0xf7, 0x66, 0x4a, 0x61, 0x62, 0xb6, 0x6b, 0x6c, + 0xda, 0xbe, 0x0c, 0x47, 0x2c, 0x71, 0x21, 0x07, 0xb0, 0xa4, 0x6b, 0x02, 0x31, 0x54, 0xf6, 0x9a, + 0xb3, 0xbc, 0x93, 0xb2, 0x51, 0xce, 0xc6, 0x61, 0xf3, 0x63, 0xa8, 0x8e, 0x85, 0x55, 0x58, 0x2f, + 0xc4, 0xc8, 0x64, 0xe4, 0x42, 0x8c, 0x14, 0x77, 0x97, 0xdc, 0x1b, 0xc6, 0x3c, 0xe7, 0x59, 0x2c, + 0x1c, 0xe4, 0x3e, 0xb4, 0x36, 0x0f, 0x60, 0x39, 0x1b, 0xf5, 0x2e, 0xbe, 0xf4, 0x3b, 0x20, 0x87, + 0xa1, 0xe0, 0x52, 0x20, 0xbc, 0x23, 0x11, 0x45, 0xfc, 0x4c, 0xcc, 0xcf, 0x74, 0x9c, 0xbd, 0x5c, + 0x36, 0x7b, 0x5b, 0x50, 0x76, 0x22, 0x73, 0x70, 0x1b, 0xeb, 0x32, 0x55, 0xd0, 0x47, 0x40, 0x5a, + 0xc2, 0x13, 0x52, 0xe8, 0xfe, 0xbd, 0x25, 0x3e, 0xed, 0x18, 0x2c, 0x8b, 0x6d, 0xc9, 0x36, 0xe4, + 0x55, 0xeb, 0x22, 0x94, 0xca, 0xde, 0xeb, 0x29, 0xd3, 0xc9, 0x9c, 0x60, 0x68, 0x40, 0x5d, 0x13, + 0x54, 0xb7, 0xfb, 0x82, 0x03, 0xce, 0x28, 0x65, 0xb3, 0x95, 0x3d, 0xb9, 0x55, 0x32, 0x40, 0xf4, + 0x56, 0x9f, 0x9a, 0xb3, 0xde, 0x77, 0x2b, 0xfa, 0xad, 0xd6, 0xaa, 0x96, 0x38, 0x56, 0xab, 0xb1, + 0x0f, 0x7e, 0xcf, 0x3f, 0xf2, 0x04, 0x0e, 0x15, 0x5b, 0xf5, 0x50, 0x54, 0xb7, 0x9b, 0xb6, 0x8a, + 0x8d, 0x02, 0xdd, 0x87, 0x62, 0xa7, 0x7b, 0x2e, 0xfa, 0x9c, 0xbc, 0xad, 0x0a, 0xb5, 0x27, 0xae, + 0x45, 0xa4, 0xcb, 0x7c, 0x65, 0x82, 0x3e, 0x66, 0xd6, 0xe9, 0xaf, 0x96, 0x46, 0x3f, 0x07, 0x51, + 0x11, 0xf7, 0x8e, 0xea, 0xf9, 0xa9, 0x89, 0xa3, 0xf4, 0x4c, 0x2f, 0x93, 0x36, 0xd4, 0x1c, 0x7f, + 0x30, 0x94, 0x2d, 0xf1, 0xbd, 0xeb, 0xbb, 0xd2, 0x0d, 0xfc, 0xa8, 0x5e, 0x44, 0x97, 0x8d, 0xec, + 0xd6, 0x63, 0x16, 0x6c, 0xca, 0x85, 0xfe, 0x6c, 0xc1, 0xca, 0x84, 0x72, 0x01, 0xae, 0xdc, 0xed, + 0xb8, 0x3e, 0x48, 0x46, 0xa6, 0x8d, 0x86, 0x8d, 0xb9, 0x68, 0xc6, 0x27, 0xe8, 0xef, 0x16, 0xac, + 0xcd, 0x32, 0x98, 0x89, 0xa6, 0x01, 0xf0, 0x2c, 0x74, 0xfb, 0x3c, 0x1c, 0x7d, 0x21, 0x46, 0xfa, + 0xf6, 0xc8, 0x68, 0xc8, 0x37, 0xb0, 0x3e, 0x11, 0xeb, 0xb3, 0x6e, 0x4c, 0x51, 0x0c, 0xea, 0xe1, + 0x5c, 0x50, 0xb1, 0x1d, 0x9b, 0xe3, 0x4e, 0xff, 0xb5, 0xe0, 0x8d, 0x99, 0x4b, 0x69, 0xf5, 0x59, + 0xd9, 0x42, 0x7f, 0x04, 0xb5, 0x17, 0x6a, 0x30, 0xb4, 0x44, 0x24, 0x5d, 0x9f, 0x2b, 0x4b, 0x5d, + 0x9e, 0x53, 0x7a, 0xe2, 0x40, 0x09, 0x75, 0x47, 0x7c, 0xa0, 0x61, 0xbe, 0xb3, 0x00, 0xe6, 0xae, + 0xb1, 0xd7, 0x73, 0xd3, 0x88, 0x0a, 0x0c, 0xce, 0x71, 0x73, 0x29, 0xa0, 0xa0, 0x26, 0xe2, 0x98, + 0xc3, 0x9d, 0xa6, 0x5a, 0x00, 0x5b, 0x66, 0x92, 0x8c, 0x21, 0xb9, 0xbd, 0x27, 0x3f, 0x02, 0x48, + 0x4d, 0x75, 0xbb, 0xdf, 0x52, 0x9f, 0x19, 0x63, 0xfa, 0x14, 0xb6, 0xcc, 0x98, 0xbb, 0xc3, 0x86, + 0xa6, 0x5a, 0x72, 0x69, 0xb5, 0xd0, 0x36, 0xd8, 0xcf, 0x99, 0xa3, 0xae, 0x3a, 0xec, 0x56, 0x93, + 0x22, 0x2d, 0x29, 0x97, 0xa7, 0x41, 0x24, 0x8d, 0x8b, 0xfa, 0x56, 0xba, 0x67, 0x41, 0x28, 0x11, + 0x71, 0x95, 0xe1, 0x37, 0xfd, 0xc5, 0x02, 0x38, 0x0e, 0x7a, 0xa2, 0x23, 0xb9, 0x1c, 0x46, 0xe4, + 0x21, 0x46, 0xc5, 0x58, 0x95, 0xbd, 0x6a, 0x7a, 0xa6, 0xe7, 0xcc, 0x61, 0xb8, 0xdf, 0xfb, 0x99, + 0x8b, 0x70, 0x7a, 0xc2, 0x24, 0x4b, 0x2c, 0x73, 0x5d, 0xee, 0x98, 0x81, 0xa2, 0xa9, 0xaa, 0xa5, + 0xf6, 0xb1, 0x5e, 0x83, 0xe6, 0xf4, 0x18, 0xaa, 0x87, 0xde, 0x30, 0x92, 0x22, 0xd4, 0x70, 0xd4, + 0x4d, 0x22, 0xb9, 0x4c, 0xea, 0x0f, 0x05, 0xb2, 0x0d, 0x4b, 0x08, 0x59, 0x48, 0xdd, 0xb7, 0x13, + 0x40, 0xcd, 0x2a, 0xed, 0x40, 0x61, 0x7e, 0xbb, 0x11, 0xc8, 0xe3, 0x1b, 0x4c, 0x33, 0x84, 0xcf, + 0xaf, 0x1a, 0xd8, 0x47, 0x6e, 0x9c, 0x52, 0x9b, 0xa9, 0x4f, 0xd4, 0xf0, 0x6b, 0x2c, 0x39, 0xa5, + 0xe1, 0xea, 0xf6, 0x59, 0x8d, 0x53, 0xa8, 0xc6, 0xe5, 0x7d, 0xee, 0x09, 0xf3, 0x8c, 0xb1, 0x33, + 0xcf, 0x98, 0xbf, 0x2c, 0x58, 0x65, 0x22, 0x72, 0x5f, 0x0a, 0xc7, 0x8f, 0x64, 0x38, 0x4c, 0xda, + 0xef, 0xf3, 0xe0, 0xd4, 0x69, 0x61, 0x54, 0x9b, 0xc5, 0x82, 0xc9, 0x51, 0x6e, 0x6e, 0x8e, 0xde, + 0x55, 0x0f, 0xdf, 0x20, 0xec, 0xa9, 0x1e, 0x0c, 0x42, 0xcd, 0xfa, 0x84, 0x61, 0xd6, 0x82, 0xbc, + 0x07, 0x4b, 0x9d, 0x60, 0x18, 0x76, 0x93, 0x01, 0xbd, 0x9e, 0x1a, 0xc7, 0xa8, 0xe2, 0x65, 0x66, + 0xcc, 0x32, 0x39, 0x2d, 0x2c, 0xc8, 0xe9, 0x4f, 0x16, 0x2c, 0x67, 0x63, 0x2c, 0x2e, 0xb1, 0x84, + 0xcb, 0xdc, 0x4c, 0x2e, 0xed, 0x59, 0x5c, 0xe6, 0x53, 0x2e, 0xd3, 0xe7, 0x47, 0x21, 0xf3, 0xfc, + 0xa0, 0xe7, 0xb0, 0x31, 0x45, 0xf0, 0x61, 0xd0, 0x1f, 0xa8, 0x4c, 0xde, 0x97, 0xe8, 0x35, 0x28, + 0xb4, 0xc3, 0x50, 0x53, 0x5c, 0x66, 0xb1, 0x40, 0xf7, 0xa1, 0x74, 0x12, 0x0c, 0x02, 0x2f, 0x38, + 0x1b, 0x65, 0x4b, 0xd5, 0xba, 0xad, 0x54, 0x1f, 0xd7, 0xfe, 0xb8, 0x69, 0x58, 0x7f, 0xde, 0x34, + 0xac, 0xbf, 0x6f, 0x1a, 0xd6, 0x6f, 0xff, 0x34, 0x5e, 0x3b, 0x2d, 0xe2, 0xcf, 0xcd, 0xfe, 0xab, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x09, 0x36, 0x2c, 0xed, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 404b53e12..56c402eca 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -150,6 +150,7 @@ message ResizeInstruction { URI URI = 2; URI Coordinator = 3; repeated ResizeSource Sources = 4; + Schema Schema = 5; } message ResizeSource { From 73b9d9fd85d742bbeced9ee7ce88bce5ad04f4a8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 3 Nov 2017 17:22:39 -0500 Subject: [PATCH 018/311] Add Cluster resize tests. Consolidate schema creation into Holder.ApplySchema. Add view names to proto schema. --- cluster.go | 57 ++++----- cluster_test.go | 248 +++++++++++++++++++++++++++++++++++++- frame.go | 17 ++- holder.go | 29 +++++ server.go | 25 ++-- test/cluster.go | 307 ++++++++++++++++++++++++++++++++++++++++++++++++ uri.go | 4 + 7 files changed, 630 insertions(+), 57 deletions(-) diff --git a/cluster.go b/cluster.go index 2841bd5bc..c438c5f7e 100644 --- a/cluster.go +++ b/cluster.go @@ -727,8 +727,8 @@ func (c *Cluster) CompleteCurrentJob(state string) error { return nil } -// followResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { +// FollowResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { go func() { // Prepare the return message. complete := &internal.ResizeInstructionComplete{ @@ -740,31 +740,9 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { // Stop processing on any error. if err := func() error { - // TODO: move this schema creation code to a method on Holder. // Sync the schema received in the resize instruction. - // Create indexes that don't exist. - for _, index := range instr.Schema.Indexes { - opt := IndexOptions{} - idx, err := c.Holder.CreateIndexIfNotExists(index.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range index.Frames { - opt := decodeFrameOptions(f.Meta) - frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) - if err != nil { - return err - } - // Create views that don't exist. - for _, v := range f.Views { - _, err := frame.CreateViewIfNotExists(v) - if err != nil { - return err - } - } - } - // TODO: Create inputDefinitions that don't exist. + if err := c.Holder.ApplySchema(instr.Schema); err != nil { + return err } // Create a client for calling remote nodes. @@ -775,7 +753,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { // Request each source file in ResizeSources. for _, src := range instr.Sources { - fmt.Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.URI) + c.logger().Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.URI) srcURI := decodeURI(src.URI) @@ -828,6 +806,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) { c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) } }() + return nil } func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { @@ -921,11 +900,10 @@ func (j *ResizeJob) setState(state string) { // Run distributes ResizeInstructions. func (j *ResizeJob) Run() error { - j.mu.RLock() - defer j.mu.RUnlock() - // Set job state to RUNNING. + j.mu.RLock() j.setState(ResizeJobStateRunning) + j.mu.RUnlock() // Job can be considered done in the case where it doesn't require any action. if !j.urisArePending() { @@ -978,6 +956,10 @@ func (j *ResizeJob) distributeResizeInstructions() error { type NodeSet []URI +func (n NodeSet) Len() int { return len(n) } +func (n NodeSet) Swap(i, j int) { n[i], n[j] = n[j], n[i] } +func (n NodeSet) Less(i, j int) bool { return n[i].String() < n[j].String() } + func (u NodeSet) ToHostPortStrings() []string { other := make([]string, 0, len(u)) for _, uri := range u { @@ -1012,7 +994,7 @@ func (t *Topology) containsURI(uri URI) bool { return false } -// AddNode adds the uri to the topology and returns true if added. +// AddURI adds the uri to the topology and returns true if added. func (t *Topology) AddURI(uri URI) bool { t.mu.Lock() defer t.mu.Unlock() @@ -1023,6 +1005,11 @@ func (t *Topology) AddURI(uri URI) bool { return true } +// Encode converts t into its internal representation. +func (t *Topology) Encode() *internal.Topology { + return encodeTopology(t) +} + // loadTopology reads the topology for the node. func (c *Cluster) loadTopology() error { buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) @@ -1117,8 +1104,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { return fmt.Errorf("host is not in topology: %v", e.URI) } - uri := e.URI - if err := c.AddNode(uri); err != nil { + if err := c.AddNode(e.URI); err != nil { return err } @@ -1138,8 +1124,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { // If the index does not yet have data, go ahead and add the node. if !c.Holder.HasData() { - uri := e.URI - if err := c.AddNode(uri); err != nil { + if err := c.AddNode(e.URI); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1161,7 +1146,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { return nil } -func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { // Ignore status updates from self (coordinator). if c.IsCoordinator() { return nil diff --git a/cluster_test.go b/cluster_test.go index 6fa3ed6e5..a51823993 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "bytes" "math/rand" "reflect" "testing" @@ -213,7 +214,7 @@ func TestCluster_Topology(t *testing.T) { t.Fatal(err) } - actual := pilosa.Nodes(c1.Nodes).URIs() + actual := c1.NodeSet() expected := []pilosa.URI{base, uri1, uri2} if !reflect.DeepEqual(actual, expected) { @@ -303,3 +304,248 @@ func TestCluster_Resize(t *testing.T) { } }) } + +// TestTestCluster ensures that general cluster functionality works as expected. +func TestCluster_ResizeStates(t *testing.T) { + + /* test conditions: + x- single node, no data, comes up in NORMAL with topology + x- single node, in topology, comes up NORMAL + x- single node, not in topology, raises error + x- two node, no data, comes up in NORMAL, with topology + x- two node, in topology, comes up NORMAL + x- two node, STARTING, not in topology, raises error + x- two node, NORMAL, not in topology, triggers resize + x- resize of nodes with data moves data appropriately + */ + + t.Run("Single node, no data", func(t *testing.T) { + tc := test.NewTestCluster(1) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + node := tc.Clusters[0] + + // Ensure that node comes up in state NORMAL. + if node.State != pilosa.ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State) + } + + expectedTop := &pilosa.Topology{ + NodeSet: []pilosa.URI{node.URI}, + } + + // Verify topology file. + if !reflect.DeepEqual(node.Topology, expectedTop) { + t.Errorf("expected topology: %v, but got: %v", expectedTop, node.Topology) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, in topology", func(t *testing.T) { + tc := test.NewTestCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &pilosa.Topology{ + NodeSet: []pilosa.URI{node.URI}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node comes up in state NORMAL. + if node.State != pilosa.ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, not in topology", func(t *testing.T) { + tc := test.NewTestCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &pilosa.Topology{ + NodeSet: []pilosa.URI{ + test.NewURIFromHostPort("some-other-host", 0), + }, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + expected := "considerTopology: coordinator http://host0:0 is not in topology: [http://some-other-host:0]" + err := tc.Open() + if err == nil || err.Error() != expected { + t.Errorf("did not receive expected error: %s", expected) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, no data", func(t *testing.T) { + tc := test.NewTestCluster(0) + tc.AddNode(false) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + tc.AddNode(false) + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State != pilosa.ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) + } else if node1.State != pilosa.ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State) + } + + expectedTop := &pilosa.Topology{ + NodeSet: []pilosa.URI{node0.URI, node1.URI}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology, expectedTop) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop, node0.Topology) + } else if !reflect.DeepEqual(node1.Topology, expectedTop) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop, node1.Topology) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { + tc := test.NewTestCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + u0 := test.NewURIFromHostPort("host0", 0) + //u1 := test.NewURIFromHostPort("host1", 0) + u2 := test.NewURIFromHostPort("host2", 0) + + // write topology to data file + top := &pilosa.Topology{ + NodeSet: []pilosa.URI{u0, u2}, + } + tc.WriteTopology(node0.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node is in state STARTING before the other node joins. + if node0.State != pilosa.ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State) + } + + // Expect an error by adding a node not in the topology. + expectedError := "host is not in topology: http://host1:0" + err := tc.AddNode(false) + if err == nil || err.Error() != expectedError { + t.Errorf("did not receive expected error: %s", expectedError) + } + + tc.AddNode(false) + node2 := tc.Clusters[2] + + // Ensure that node comes up in state NORMAL. + if node0.State != pilosa.ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) + } else if node2.State != pilosa.ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, with data", func(t *testing.T) { + tc := test.NewTestCluster(0) + tc.AddNode(false) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Add Data to node0. + tc.CreateFrame("i", "f", pilosa.FrameOptions{}) + tc.SetBit("i", "f", "standard", 1, 101, nil) + tc.SetBit("i", "f", "standard", 1, 1300000, nil) + + // AddNode needs to block until the resize process has completed. + tc.AddNode(false) + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State != pilosa.ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) + } else if node1.State != pilosa.ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State) + } + + expectedTop := &pilosa.Topology{ + NodeSet: []pilosa.URI{node0.URI, node1.URI}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology, expectedTop) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop, node0.Topology) + } else if !reflect.DeepEqual(node1.Topology, expectedTop) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop, node1.Topology) + } + + // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. + node0Frame := node0.Holder.Frame("i", "f") + node0View := node0Frame.View("standard") + node0Fragment := node0View.Fragment(1) + + node1Frame := node1.Holder.Frame("i", "f") + node1View := node1Frame.View("standard") + node1Fragment := node1View.Fragment(1) + + // Ensure checksums are the same. + orig := node0Fragment.Checksum() + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, orig) { + t.Fatalf("expected checksum to match: %x - %x", chksum, orig) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/frame.go b/frame.go index 06d081423..c57edf38e 100644 --- a/frame.go +++ b/frame.go @@ -550,6 +550,18 @@ func (f *Frame) Views() []*View { return other } +// viewNames returns a list of all views (as a string) in the frame. +func (f *Frame) viewNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + + other := make([]string, 0, len(f.views)) + for viewName, _ := range f.views { + other = append(other, viewName) + } + return other +} + // RecalculateCaches recalculates caches on every view in the frame. func (f *Frame) RecalculateCaches() { for _, view := range f.Views() { @@ -958,8 +970,9 @@ func encodeFrames(a []*Frame) []*internal.Frame { func encodeFrame(f *Frame) *internal.Frame { fo := f.options() return &internal.Frame{ - Name: f.name, - Meta: fo.Encode(), + Name: f.name, + Meta: fo.Encode(), + Views: f.viewNames(), } } diff --git a/holder.go b/holder.go index 8da92f498..ce6e052e2 100644 --- a/holder.go +++ b/holder.go @@ -187,6 +187,35 @@ func (h *Holder) Schema() []*IndexInfo { return a } +// ApplySchema applies an internal Schema to Holder. +func (h *Holder) ApplySchema(schema *internal.Schema) error { + // Create indexes that don't exist. + for _, index := range schema.Indexes { + opt := IndexOptions{} + idx, err := h.CreateIndexIfNotExists(index.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range index.Frames { + opt := decodeFrameOptions(f.Meta) + frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) + if err != nil { + return err + } + // Create views that don't exist. + for _, v := range f.Views { + _, err := frame.CreateViewIfNotExists(v) + if err != nil { + return err + } + } + } + // TODO: Create inputDefinitions that don't exist. + } + return nil +} + // EncodeMaxSlices creates and internal representation of max slices. func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { return &internal.MaxSlices{ diff --git a/server.go b/server.go index 9327e8ca8..e460e1432 100644 --- a/server.go +++ b/server.go @@ -332,12 +332,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.ClusterStatus: - err := s.Cluster.mergeClusterStatus(obj) + err := s.Cluster.MergeClusterStatus(obj) if err != nil { return err } case *internal.ResizeInstruction: - s.Cluster.followResizeInstruction(obj) + err := s.Cluster.FollowResizeInstruction(obj) + if err != nil { + return err + } case *internal.ResizeInstructionComplete: err := s.Cluster.MarkResizeInstructionComplete(obj) if err != nil { @@ -397,22 +400,8 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { } // Sync schema. - // Create indexes that don't exist. - for _, index := range ns.Schema.Indexes { - opt := IndexOptions{} - idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range index.Frames { - opt := decodeFrameOptions(f.Meta) - _, err := idx.CreateFrameIfNotExists(f.Name, *opt) - if err != nil { - return err - } - } - // TODO: Create inputDefinitions that don't exist. + if err := s.Holder.ApplySchema(ns.Schema); err != nil { + return err } // Sync maxSlices (standard). diff --git a/test/cluster.go b/test/cluster.go index 604aff14c..9d887acce 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -1,10 +1,17 @@ package test import ( + "bufio" + "bytes" "fmt" "io/ioutil" + "path/filepath" + "sort" + "time" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" ) // NewCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -62,3 +69,303 @@ func NewURIFromHostPort(host string, port uint16) pilosa.URI { uri.SetPort(port) return *uri } + +// TestCluster represents a cluster of test nodes, each of which +// has a pilosa.Cluster. +type TestCluster struct { + Clusters []*pilosa.Cluster + + common *commonClusterSettings + + resizeDone chan struct{} +} + +type commonClusterSettings struct { + NodeSet pilosa.NodeSet +} + +func (t *TestCluster) CreateIndex(name string) error { + for _, c := range t.Clusters { + if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil { + return err + } + } + return nil +} + +func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error { + for _, c := range t.Clusters { + idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) + if err != nil { + return err + } + if _, err := idx.CreateFrame(frame, opt); err != nil { + return err + } + } + return nil +} +func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { + // Determine which node should receive the SetBit. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := colID / pilosa.SliceWidth + nodes := c0.FragmentNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByURI(node.URI) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetBit(view, rowID, colID, x) + if err != nil { + return err + } + } + + return nil +} + +func (t *TestCluster) clusterByURI(uri pilosa.URI) *pilosa.Cluster { + for _, c := range t.Clusters { + if c.URI == uri { + return c + } + } + return nil +} + +// AddNode adds a node to the cluster and (potentially) starts a resize job. +func (t *TestCluster) AddNode(saveTopology bool) error { + id := len(t.Clusters) + + c, err := t.addCluster(id, saveTopology) + if err != nil { + return err + } + + // Send NodeJoin event to coordinator. + if id > 0 { + coord := t.Clusters[0] + ev := &pilosa.NodeEvent{ + Event: pilosa.NodeJoin, + URI: c.URI, + } + + //go coord.ReceiveEvent(ev) + if err := coord.ReceiveEvent(ev); err != nil { + return err + } + + // Wait for the NodeAdd job to finish. + if c.State != pilosa.ClusterStateNormal { + t.resizeDone = make(chan struct{}) + <-t.resizeDone + } + } + + return nil +} + +// WriteTopology writes the given topology to disk. +func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { + if buf, err := proto.Marshal(top.Encode()); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { + return err + } + return nil +} + +func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { + + uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) + + // add URI to common + t.common.NodeSet = append(t.common.NodeSet, uri) + sort.Sort(t.common.NodeSet) + + // create node-specific temp directory + path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + if err != nil { + return nil, err + } + + // holder + h := pilosa.NewHolder() + h.Path = path + + // cluster + c := pilosa.NewCluster() + c.ReplicaN = 1 + c.Hasher = NewModHasher() + c.Path = path + c.Topology = pilosa.NewTopology() + c.Holder = h + c.MemberSet = pilosa.NewStaticMemberSet() + c.URI = uri + c.Coordinator = t.common.NodeSet[0] // the first node is the coordinator + c.Broadcaster = t + + // add nodes + if saveTopology { + for _, u := range t.common.NodeSet { + c.AddNode(u) + } + } + + // Add this node to the TestCluster. + t.Clusters = append(t.Clusters, c) + + return c, nil +} + +// NewTestCluster returns a new instance of test.Cluster. +func NewTestCluster(n int) *TestCluster { + + tc := &TestCluster{ + common: &commonClusterSettings{}, + } + + // add clusters + for i := 0; i < n; i++ { + _, err := tc.addCluster(i, true) + if err != nil { + panic(err) + } + } + return tc +} + +// SetState sets the state of the cluster on each node. +func (t *TestCluster) SetState(state string) { + for _, c := range t.Clusters { + c.State = state + } +} + +// Open opens all clusters in the test cluster. +func (t *TestCluster) Open() error { + for _, c := range t.Clusters { + err := c.Open() + if err != nil { + return err + } + } + return nil +} + +// Close closes all clusters in the test cluster. +func (t *TestCluster) Close() error { + for _, c := range t.Clusters { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + +// TestCluster implements Broadcaster interface. + +// SendSync is a test implemenetation of Broadcaster SendSync method. +func (t *TestCluster) SendSync(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ClusterStatus: + // Apply the send message to all nodes (except the coordinator). + for _, c := range t.Clusters { + c.MergeClusterStatus(obj) + } + if obj.State == pilosa.ClusterStateNormal && t.resizeDone != nil { + close(t.resizeDone) + } + } + + return nil +} + +// SendAsync is a test implemenetation of Broadcaster SendAsync method. +func (t *TestCluster) SendAsync(pb proto.Message) error { + return nil +} + +// SendTo is a test implemenetation of Broadcaster SendTo method. +func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ResizeInstruction: + t.FollowResizeInstruction(obj) + case *internal.ResizeInstructionComplete: + coord := t.clusterByURI(to.URI) + go coord.MarkResizeInstructionComplete(obj) + } + return nil +} + +// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. +func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + + // Prepare the return message. + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + URI: instr.URI, + Error: "", + } + + // figure out which node it was meant for, then call the operation on that cluster + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + instrURI := pilosa.DecodeURI(instr.URI) + destCluster := t.clusterByURI(instrURI) + + // Sync the schema received in the resize instruction. + if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + return err + } + + for _, src := range instr.Sources { + srcURI := pilosa.DecodeURI(src.URI) + srcCluster := t.clusterByURI(srcURI) + + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + if destFragment == nil { + // Create fragment on destination if it doesn't exist. + f := destCluster.Holder.Frame(src.Index, src.Frame) + v := f.View(src.View) + var err error + destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + } + + buf := bytes.NewBuffer(nil) + + bw := bufio.NewWriter(buf) + br := bufio.NewReader(buf) + + // Get the fragment from source. + if _, err := srcFragment.WriteTo(bw); err != nil { + return err + } + + // Flush the bufio.buf to the io.Writer (buf). + bw.Flush() + + // Write data to destination. + if _, err := destFragment.ReadFrom(br); err != nil { + return err + } + } + + node := &pilosa.Node{ + URI: pilosa.DecodeURI(instr.Coordinator), + } + if err := t.SendTo(node, complete); err != nil { + return err + } + + return nil +} diff --git a/uri.go b/uri.go index 23aba4dfc..79c7cd4ac 100644 --- a/uri.go +++ b/uri.go @@ -204,6 +204,10 @@ func encodeURI(u URI) *internal.URI { } } +func DecodeURI(i *internal.URI) URI { + return decodeURI(i) +} + func decodeURI(i *internal.URI) URI { if i == nil { return URI{} From c7c1be6813e006cc5d85a748869022fb0781debe Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 3 Nov 2017 17:43:05 -0500 Subject: [PATCH 019/311] change read lock to write lock --- cluster.go | 4 +--- test/cluster.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index c438c5f7e..323283a8a 100644 --- a/cluster.go +++ b/cluster.go @@ -901,9 +901,7 @@ func (j *ResizeJob) setState(state string) { // Run distributes ResizeInstructions. func (j *ResizeJob) Run() error { // Set job state to RUNNING. - j.mu.RLock() - j.setState(ResizeJobStateRunning) - j.mu.RUnlock() + j.SetState(ResizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.urisArePending() { diff --git a/test/cluster.go b/test/cluster.go index 9d887acce..e73289827 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -160,7 +160,7 @@ func (t *TestCluster) AddNode(saveTopology bool) error { return err } - // Wait for the NodeAdd job to finish. + // Wait for the AddNode job to finish. if c.State != pilosa.ClusterStateNormal { t.resizeDone = make(chan struct{}) <-t.resizeDone From 5071fd9e5d7e3e62913c51232c66181536f12c39 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 3 Nov 2017 18:02:21 -0500 Subject: [PATCH 020/311] Fix error format on slices. These tests were failing on `Go:master` (passing on `Go:1.8` and `Go:1.9`) --- client_test.go | 2 +- executor_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client_test.go b/client_test.go index e5dffc9a5..15ebb8996 100644 --- a/client_test.go +++ b/client_test.go @@ -87,7 +87,7 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %s", num, s[i].Host(), owns) + t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns) } } diff --git a/executor_test.go b/executor_test.go index 34fe21e9a..040248de9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -781,7 +781,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %s", result[0].(*pilosa.Bitmap).Bits()) + t.Fatalf("unexpected result: %v", result[0].(*pilosa.Bitmap).Bits()) } }) From f94a2a0cfc9f7b2f512372ebb29bca61568fcb1d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 6 Nov 2017 08:38:08 -0600 Subject: [PATCH 021/311] add json tags to uri struct --- uri.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uri.go b/uri.go index 79c7cd4ac..b361cdf6a 100644 --- a/uri.go +++ b/uri.go @@ -43,9 +43,9 @@ var addressRegexp = regexp.MustCompile("^(([+a-z]+):\\/\\/)?([0-9a-z.-]+|\\[[:0- // localhost // :10101 type URI struct { - scheme string - host string - port uint16 + scheme string `json:"scheme"` + host string `json:"host"` + port uint16 `json:"port"` } // DefaultURI creates and returns the default URI. From 53fa54d3d3bb73194656167ada1545312d0e0241 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 6 Nov 2017 09:17:50 -0600 Subject: [PATCH 022/311] 891 nodeid tags (from #897) --- server.go | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/server.go b/server.go index e460e1432..cfbf4ea73 100644 --- a/server.go +++ b/server.go @@ -130,21 +130,13 @@ func (s *Server) Open() error { // Set Cluster URI. s.Cluster.URI = s.URI - /* - // Create local node if no cluster is specified. - if len(s.Cluster.Nodes) == 0 { - s.Cluster.Nodes = []*Node{ - {Scheme: s.URI.Scheme(), Host: s.URI.HostPort()}, - } + // Find the Node ID and append that tag to stats. + for i, n := range s.Cluster.Nodes { + if n.URI == s.URI { + s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) + break } - - // TODO: nodes aren't here yet. May need to merge new stats code anyway. - for i, n := range s.Cluster.Nodes { - if s.Cluster.NodeByHost(n.Host) != nil { - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) - } - } - */ + } // Open holder. s.Holder.LogOutput = s.LogOutput From 0d5a2e7bfd4470d5cf4ab7c4a3bcccda05266dce Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 6 Nov 2017 10:43:55 -0600 Subject: [PATCH 023/311] add SetFieldValue() method to TestCluster and use in tests --- cluster_test.go | 40 ++++++++++++++++++++++++++++++++++++++-- test/cluster.go | 24 ++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index a51823993..7667aa24d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -499,11 +499,30 @@ func TestCluster_ResizeStates(t *testing.T) { t.Fatal(err) } - // Add Data to node0. + // Add Bit Data to node0. tc.CreateFrame("i", "f", pilosa.FrameOptions{}) tc.SetBit("i", "f", "standard", 1, 101, nil) tc.SetBit("i", "f", "standard", 1, 1300000, nil) + // Add Field Data to node0. + tc.CreateFrame("i", "fields", pilosa.FrameOptions{ + InverseEnabled: false, + RangeEnabled: true, + CacheType: pilosa.CacheTypeNone, + Fields: []*pilosa.Field{ + { + Name: "fld0", + Type: pilosa.FieldTypeInt, + Min: -100, + Max: 100, + }, + }, + }) + tc.SetFieldValue("i", "fields", 1, "fld0", -10) + tc.SetFieldValue("i", "fields", 1, "fld0", 10) + tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) + tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) + // AddNode needs to block until the resize process has completed. tc.AddNode(false) @@ -528,6 +547,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Errorf("expected node1 topology: %v, but got: %v", expectedTop, node1.Topology) } + // Bits // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. node0Frame := node0.Holder.Frame("i", "f") node0View := node0Frame.View("standard") @@ -540,7 +560,23 @@ func TestCluster_ResizeStates(t *testing.T) { // Ensure checksums are the same. orig := node0Fragment.Checksum() if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, orig) { - t.Fatalf("expected checksum to match: %x - %x", chksum, orig) + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, orig) + } + + // Values + // Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0. + node0Frame = node0.Holder.Frame("i", "fields") + node0View = node0Frame.View("field_fld0") + node0Fragment = node0View.Fragment(1) + + node1Frame = node1.Holder.Frame("i", "fields") + node1View = node1Frame.View("field_fld0") + node1Fragment = node1View.Fragment(1) + + // Ensure checksums are the same. + orig = node0Fragment.Checksum() + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, orig) { + t.Fatalf("expected field view checksum to match: %x - %x", chksum, orig) } // Close TestCluster. diff --git a/test/cluster.go b/test/cluster.go index e73289827..248b61325 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -129,6 +129,30 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * return nil } +func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { + // Determine which node should receive the SetFieldValue. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := columnID / pilosa.SliceWidth + nodes := c0.FragmentNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByURI(node.URI) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetFieldValue(columnID, name, value) + if err != nil { + return err + } + } + + return nil +} + func (t *TestCluster) clusterByURI(uri pilosa.URI) *pilosa.Cluster { for _, c := range t.Clusters { if c.URI == uri { From 7e797efdc23f231352e9645452bbdd2f792ab4c2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 6 Nov 2017 14:32:20 -0600 Subject: [PATCH 024/311] Allow CacheType to be set for a RangeEnabled frame (to apply to the standard frame) --- cluster_test.go | 16 ++++++--- fragment.go | 5 +++ frame.go | 7 ++++ index.go | 2 -- index_test.go | 12 ------- test/cluster.go | 88 +++++++++++++++++++++++++++---------------------- 6 files changed, 72 insertions(+), 58 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 7667aa24d..911742a27 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -500,15 +500,17 @@ func TestCluster_ResizeStates(t *testing.T) { } // Add Bit Data to node0. - tc.CreateFrame("i", "f", pilosa.FrameOptions{}) + if err := tc.CreateFrame("i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } tc.SetBit("i", "f", "standard", 1, 101, nil) tc.SetBit("i", "f", "standard", 1, 1300000, nil) // Add Field Data to node0. - tc.CreateFrame("i", "fields", pilosa.FrameOptions{ + if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{ InverseEnabled: false, RangeEnabled: true, - CacheType: pilosa.CacheTypeNone, + //CacheType: pilosa.CacheTypeNone, Fields: []*pilosa.Field{ { Name: "fld0", @@ -517,14 +519,18 @@ func TestCluster_ResizeStates(t *testing.T) { Max: 100, }, }, - }) + }); err != nil { + t.Fatal(err) + } tc.SetFieldValue("i", "fields", 1, "fld0", -10) tc.SetFieldValue("i", "fields", 1, "fld0", 10) tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) // AddNode needs to block until the resize process has completed. - tc.AddNode(false) + if err := tc.AddNode(false); err != nil { + t.Fatal(err) + } node0 := tc.Clusters[0] node1 := tc.Clusters[1] diff --git a/fragment.go b/fragment.go index 172fc22a3..745d1d613 100644 --- a/fragment.go +++ b/fragment.go @@ -254,6 +254,7 @@ func (f *Fragment) openCache() error { f.cache = NewLRUCache(f.CacheSize) case CacheTypeNone: f.cache = NewNopCache() + return nil default: return ErrInvalidCacheType } @@ -1451,6 +1452,10 @@ func (f *Fragment) flushCache() error { return nil } + if f.CacheType == CacheTypeNone { + return nil + } + // Retrieve a list of row ids from the cache. ids := f.cache.IDs() diff --git a/frame.go b/frame.go index c57edf38e..58920ccc2 100644 --- a/frame.go +++ b/frame.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" @@ -584,6 +585,12 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { } view := f.newView(f.ViewPath(name), name) + + // Never keep a cache for field views. + if strings.HasPrefix(name, ViewFieldPrefix) { + view.cacheType = CacheTypeNone + } + if err := view.Open(); err != nil { return nil, err } diff --git a/index.go b/index.go index 734ea7ed5..be6ae031c 100644 --- a/index.go +++ b/index.go @@ -454,8 +454,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { if opt.RangeEnabled { if opt.InverseEnabled { return nil, ErrInverseRangeNotAllowed - } else if opt.CacheType != "" && opt.CacheType != CacheTypeNone { - return nil, ErrRangeCacheNotAllowed } } else { if len(opt.Fields) > 0 { diff --git a/index_test.go b/index_test.go index 077b2ded6..7dfec3fb9 100644 --- a/index_test.go +++ b/index_test.go @@ -136,18 +136,6 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeRanked, - }); err != pilosa.ErrRangeCacheNotAllowed { - t.Fatal(err) - } - }) - t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() diff --git a/test/cluster.go b/test/cluster.go index 248b61325..ce0563eba 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -179,7 +179,6 @@ func (t *TestCluster) AddNode(saveTopology bool) error { URI: c.URI, } - //go coord.ReceiveEvent(ev) if err := coord.ReceiveEvent(ev); err != nil { return err } @@ -320,7 +319,10 @@ func (t *TestCluster) SendAsync(pb proto.Message) error { func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { switch obj := pb.(type) { case *internal.ResizeInstruction: - t.FollowResizeInstruction(obj) + err := t.FollowResizeInstruction(obj) + if err != nil { + return err + } case *internal.ResizeInstructionComplete: coord := t.clusterByURI(to.URI) go coord.MarkResizeInstructionComplete(obj) @@ -338,50 +340,58 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) Error: "", } - // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) - instrURI := pilosa.DecodeURI(instr.URI) - destCluster := t.clusterByURI(instrURI) + // Stop processing on any error. + if err := func() error { - // Sync the schema received in the resize instruction. - if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { - return err - } + // figure out which node it was meant for, then call the operation on that cluster + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + instrURI := pilosa.DecodeURI(instr.URI) + destCluster := t.clusterByURI(instrURI) - for _, src := range instr.Sources { - srcURI := pilosa.DecodeURI(src.URI) - srcCluster := t.clusterByURI(srcURI) + // Sync the schema received in the resize instruction. + if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + return err + } - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - if destFragment == nil { - // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) - v := f.View(src.View) - var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) - if err != nil { + for _, src := range instr.Sources { + srcURI := pilosa.DecodeURI(src.URI) + srcCluster := t.clusterByURI(srcURI) + + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + if destFragment == nil { + // Create fragment on destination if it doesn't exist. + f := destCluster.Holder.Frame(src.Index, src.Frame) + v := f.View(src.View) + var err error + destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + } + + buf := bytes.NewBuffer(nil) + + bw := bufio.NewWriter(buf) + br := bufio.NewReader(buf) + + // Get the fragment from source. + if _, err := srcFragment.WriteTo(bw); err != nil { + return err + } + + // Flush the bufio.buf to the io.Writer (buf). + bw.Flush() + + // Write data to destination. + if _, err := destFragment.ReadFrom(br); err != nil { return err } } - buf := bytes.NewBuffer(nil) - - bw := bufio.NewWriter(buf) - br := bufio.NewReader(buf) - - // Get the fragment from source. - if _, err := srcFragment.WriteTo(bw); err != nil { - return err - } - - // Flush the bufio.buf to the io.Writer (buf). - bw.Flush() - - // Write data to destination. - if _, err := destFragment.ReadFrom(br); err != nil { - return err - } + return nil + }(); err != nil { + complete.Error = err.Error() } node := &pilosa.Node{ From ec5ac629cd526457ad48ee147d5b1b57304b867a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 6 Nov 2017 14:48:35 -0600 Subject: [PATCH 025/311] porting the remaining changes from PR #897 --- executor_test.go | 6 +++--- handler.go | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/executor_test.go b/executor_test.go index 040248de9..67cd2c041 100644 --- a/executor_test.go +++ b/executor_test.go @@ -305,7 +305,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } else if value != 25 { - t.Fatal("unexpected value: %v", value) + t.Fatalf("unexpected value: %v", value) } if value, exists, err := f.FieldValue(10, "field1"); err != nil { @@ -313,7 +313,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } else if value != 2 { - t.Fatal("unexpected value: %v", value) + t.Fatalf("unexpected value: %v", value) } if value, exists, err := f.FieldValue(100, "field0"); err != nil { @@ -321,7 +321,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } else if value != 10 { - t.Fatal("unexpected value: %v", value) + t.Fatalf("unexpected value: %v", value) } }) diff --git a/handler.go b/handler.go index 55b6e91d9..e3ad6422e 100644 --- a/handler.go +++ b/handler.go @@ -956,7 +956,7 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ingore this error becuase views do not exist on all nodes due to slice distribution. + // Ingore this error because views do not exist on all nodes due to slice distribution. if err != ErrInvalidView { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1201,8 +1201,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Validate that this handler owns the slice. if !h.Cluster.OwnsFragment(h.URI, req.Index, req.Slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) - http.Error(w, mesg, http.StatusPreconditionFailed) + msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) + http.Error(w, msg, http.StatusPreconditionFailed) return } @@ -1271,8 +1271,8 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) // Validate that this handler owns the slice. if !h.Cluster.OwnsFragment(h.URI, req.Index, req.Slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) - http.Error(w, mesg, http.StatusPreconditionFailed) + msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) + http.Error(w, msg, http.StatusPreconditionFailed) return } @@ -1337,8 +1337,8 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // Validate that this handler owns the slice. if !h.Cluster.OwnsFragment(h.URI, index, slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice) - http.Error(w, mesg, http.StatusPreconditionFailed) + msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice) + http.Error(w, msg, http.StatusPreconditionFailed) return } @@ -1796,7 +1796,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // Validation the input definition with the curent index's ColumnLabel. + // Validation the input definition with the current index's ColumnLabel. if err := req.Validate(index.ColumnLabel()); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return From fee1b5583888263d2a990b1ba84a947c4b1c879a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 6 Nov 2017 16:03:23 -0600 Subject: [PATCH 026/311] replace test ErrRangeCacheNotAllowed with ErrRangeCacheAllowed --- index_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/index_test.go b/index_test.go index 7dfec3fb9..5e5dce218 100644 --- a/index_test.go +++ b/index_test.go @@ -136,6 +136,18 @@ func TestIndex_CreateFrame(t *testing.T) { } }) + t.Run("ErrRangeCacheAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + + if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + CacheType: pilosa.CacheTypeRanked, + }); err != nil { + t.Fatal(err) + } + }) + t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() From 9d870197625a2ded7c415c512cb0deea9cb5c84f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 10 Nov 2017 09:10:15 -0600 Subject: [PATCH 027/311] WIP: don't block joining nodes while coordinator loads data. Implements a Holder.Peek() function, and breaks out Cluster.ListenForJoins() into a separate method that can be started after the Holder finishes loading data. TODO: - [ ] test Holder.Peek() --- cluster.go | 14 ++++++++------ holder.go | 33 ++++++++++++++++++++++++++++++++- server.go | 20 +++++++++++++++++--- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/cluster.go b/cluster.go index 32f1dee12..204e39c26 100644 --- a/cluster.go +++ b/cluster.go @@ -546,10 +546,6 @@ func (c *Cluster) Open() error { return fmt.Errorf("opening MemberSet: %v", err) } - // Listen for cluster-resize events. - c.wg.Add(1) - go func() { defer c.wg.Done(); c.listenForJoins() }() - return nil } @@ -604,6 +600,12 @@ func (c *Cluster) setStateAndBroadcast(state string) error { return c.Broadcaster.SendSync(c.Status()) } +// ListenForJoins handles cluster-resize events. +func (c *Cluster) ListenForJoins() { + c.wg.Add(1) + go func() { defer c.wg.Done(); c.listenForJoins() }() +} + func (c *Cluster) listenForJoins() { var uriJoined bool @@ -634,8 +636,8 @@ func (c *Cluster) listenForJoins() { select { case <-c.closing: return - case host := <-c.joiningURIs: - err := c.handleJoiningHost(host) + case uri := <-c.joiningURIs: + err := c.handleJoiningHost(uri) if err != nil { c.logger().Printf("handleJoiningHost error: err=%s", err) continue diff --git a/holder.go b/holder.go index 07c030135..bbda143d6 100644 --- a/holder.go +++ b/holder.go @@ -44,6 +44,7 @@ type Holder struct { // Indexes by name. indexes map[string]*Index + hasData bool Broadcaster Broadcaster // Close management @@ -77,6 +78,36 @@ func NewHolder() *Holder { } } +// Peek reads the root data directory for the holder +// without actually loading any data into memory. +func (h *Holder) Peek() error { + if err := os.MkdirAll(h.Path, 0777); err != nil { + return err + } + + // Open path to read all index directories. + f, err := os.Open(h.Path) + if err != nil { + return err + } + defer f.Close() + + fis, err := f.Readdir(0) + if err != nil { + return err + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + h.hasData = true + break + } + + return nil +} + // Open initializes the root data directory for the holder. func (h *Holder) Open() error { h.setFileLimit() @@ -149,7 +180,7 @@ func (h *Holder) Close() error { // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. func (h *Holder) HasData() bool { - return len(h.indexes) > 0 + return h.hasData || len(h.indexes) > 0 } // MaxSlices returns MaxSlice map for all indexes. diff --git a/server.go b/server.go index ac5c5d06a..06c0a81e7 100644 --- a/server.go +++ b/server.go @@ -145,10 +145,12 @@ func (s *Server) Open() error { } } - // Open holder. + // Peek at the holder to determine if there is data on disk. + // Don't actually load the data until after the Cluster + // management starts. s.Holder.LogOutput = s.LogOutput - if err := s.Holder.Open(); err != nil { - return fmt.Errorf("opening Holder: %v", err) + if err := s.Holder.Peek(); err != nil { + return fmt.Errorf("peeking at the Holder: %v", err) } // Start the BroadcastReceiver. @@ -161,6 +163,18 @@ func (s *Server) Open() error { return fmt.Errorf("opening Cluster: %v", err) } + // Open holder. + if err := s.Holder.Open(); err != nil { + return fmt.Errorf("opening Holder: %v", err) + } + + // Listen for joining nodes. + // This needs to start after the Holder has opened so that nodes can join + // the cluster without waiting for data to load on the coordinator. Before + // this starts, the joins are queued up in the Cluster.joiningURIs buffered + // channel. + s.Cluster.ListenForJoins() + // Create default HTTP client s.createDefaultClient() From 453bbc996c0ad3342a9683ae92b1eceed52618f3 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 10 Nov 2017 10:38:49 -0600 Subject: [PATCH 028/311] adjust Holder.Peek() and add tests for it --- holder.go | 13 ++++++------ holder_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 4 +--- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/holder.go b/holder.go index bbda143d6..0e3d3953c 100644 --- a/holder.go +++ b/holder.go @@ -80,21 +80,20 @@ func NewHolder() *Holder { // Peek reads the root data directory for the holder // without actually loading any data into memory. -func (h *Holder) Peek() error { - if err := os.MkdirAll(h.Path, 0777); err != nil { - return err - } +// HasData is returned, and h.hasData is set. +func (h *Holder) Peek() bool { + h.hasData = false // Open path to read all index directories. f, err := os.Open(h.Path) if err != nil { - return err + return false } defer f.Close() fis, err := f.Readdir(0) if err != nil { - return err + return false } for _, fi := range fis { @@ -105,7 +104,7 @@ func (h *Holder) Peek() error { break } - return nil + return h.hasData } // Open initializes the root data directory for the holder. diff --git a/holder_test.go b/holder_test.go index 0427c577e..f182d3496 100644 --- a/holder_test.go +++ b/holder_test.go @@ -266,6 +266,60 @@ func TestHolder_Open(t *testing.T) { }) } +func TestHolder_HasData(t *testing.T) { + t.Run("IndexDirectory", func(t *testing.T) { + h := test.MustOpenHolder() + defer h.Close() + + if h.HasData() { + t.Fatal("expected HasData to return false") + } + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + if !h.HasData() { + t.Fatal("expected HasData to return true") + } + }) + + t.Run("Peek", func(t *testing.T) { + h := test.NewHolder() + + if hasData := h.Peek(); hasData != false { + t.Fatal("expected Peek to return false") + } else if h.HasData() { + t.Fatal("expected HasData to return false") + } + + // Create an index directory to indicate data exists. + if err := os.Mkdir(h.IndexPath("test"), 0777); err != nil { + t.Fatal(err) + } + + if hasData := h.Peek(); hasData != true { + t.Fatal("expected Peek to return true") + } else if !h.HasData() { + t.Fatal("expected HasData to return true") + } + }) + + t.Run("Peek at missing directory", func(t *testing.T) { + h := test.NewHolder() + + // Ensure that hasData is false when trying to peek into + // a directory that doesn't exist. + h.Path = "bad-path" + + if hasData := h.Peek(); hasData != false { + t.Fatal("expected Peek to return false") + } else if h.HasData() { + t.Fatal("expected HasData to return false") + } + }) +} + /* func TestHolder_Schema(t *testing.T) { t.Run("Schema", func(t *testing.T) { diff --git a/server.go b/server.go index 06c0a81e7..da6cc8a1a 100644 --- a/server.go +++ b/server.go @@ -149,9 +149,7 @@ func (s *Server) Open() error { // Don't actually load the data until after the Cluster // management starts. s.Holder.LogOutput = s.LogOutput - if err := s.Holder.Peek(); err != nil { - return fmt.Errorf("peeking at the Holder: %v", err) - } + s.Holder.Peek() // Start the BroadcastReceiver. if err := s.BroadcastReceiver.Start(s); err != nil { From 212628b2203d34fb25fd5f4bf7012ade4f0be2dc Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 10 Nov 2017 11:38:16 -0600 Subject: [PATCH 029/311] adjust tests to include Cluster.ListenForJoins() --- cluster_test.go | 11 ----------- test/cluster.go | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index a51823993..1e5427df0 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -308,17 +308,6 @@ func TestCluster_Resize(t *testing.T) { // TestTestCluster ensures that general cluster functionality works as expected. func TestCluster_ResizeStates(t *testing.T) { - /* test conditions: - x- single node, no data, comes up in NORMAL with topology - x- single node, in topology, comes up NORMAL - x- single node, not in topology, raises error - x- two node, no data, comes up in NORMAL, with topology - x- two node, in topology, comes up NORMAL - x- two node, STARTING, not in topology, raises error - x- two node, NORMAL, not in topology, triggers resize - x- resize of nodes with data moves data appropriately - */ - t.Run("Single node, no data", func(t *testing.T) { tc := test.NewTestCluster(1) diff --git a/test/cluster.go b/test/cluster.go index e73289827..82c45aef6 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -255,6 +255,13 @@ func (t *TestCluster) Open() error { return err } } + + // Start the listener on the coordinator. + if len(t.Clusters) == 0 { + return nil + } + t.Clusters[0].ListenForJoins() + return nil } From 426992bc59658e142dc0b7970dd1c387504931f5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 15 Nov 2017 09:06:07 -0600 Subject: [PATCH 030/311] add set-coordinator endpoint --- broadcast.go | 5 + cluster.go | 10 ++ cluster_test.go | 31 +++- handler.go | 64 +++++++- handler_test.go | 1 + internal/private.pb.go | 342 ++++++++++++++++++++++++++++++++--------- internal/private.proto | 5 + server.go | 2 + 8 files changed, 384 insertions(+), 76 deletions(-) diff --git a/broadcast.go b/broadcast.go index e0da4756b..4a5b8e8f7 100644 --- a/broadcast.go +++ b/broadcast.go @@ -125,6 +125,7 @@ const ( MessageTypeClusterStatus = 9 MessageTypeResizeInstruction = 10 MessageTypeResizeInstructionComplete = 11 + MessageTypeSetCoordinator = 12 ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -153,6 +154,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeResizeInstruction case *internal.ResizeInstructionComplete: typ = MessageTypeResizeInstructionComplete + case *internal.SetCoordinatorMessage: + typ = MessageTypeSetCoordinator default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -191,6 +194,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.ResizeInstruction{} case MessageTypeResizeInstructionComplete: m = &internal.ResizeInstructionComplete{} + case MessageTypeSetCoordinator: + m = &internal.SetCoordinatorMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/cluster.go b/cluster.go index 204e39c26..fa6651eab 100644 --- a/cluster.go +++ b/cluster.go @@ -200,6 +200,16 @@ func (c *Cluster) IsCoordinator() bool { return c.Coordinator == c.URI } +// SetCoordinator updates the Coordinator to new if it is +// currently old. Returns true if the Coordinator changed. +func (c *Cluster) SetCoordinator(oldURI, newURI URI) bool { + if c.Coordinator == oldURI && oldURI != newURI { + c.Coordinator = newURI + return true + } + return false +} + // AddNode adds a node to the Cluster and updates and saves the // new topology. func (c *Cluster) AddNode(uri URI) error { diff --git a/cluster_test.go b/cluster_test.go index 1e5427df0..0c26b96f2 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -305,7 +305,7 @@ func TestCluster_Resize(t *testing.T) { }) } -// TestTestCluster ensures that general cluster functionality works as expected. +// Ensure that general cluster functionality works as expected. func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, no data", func(t *testing.T) { @@ -538,3 +538,32 @@ func TestCluster_ResizeStates(t *testing.T) { } }) } + +// Ensures that coordinator can be changed. +func TestCluster_SetCoordinator(t *testing.T) { + t.Run("SetCoordinator", func(t *testing.T) { + c := test.NewCluster(1) + oldURI, err := pilosa.NewURIFromAddress("localhost:8888") + if err != nil { + t.Fatal(err) + } + c.Coordinator = *oldURI + + newURI, err := pilosa.NewURIFromAddress("localhost:9999") + if err != nil { + t.Fatal(err) + } + + // Set coordinator to the same value. + c.SetCoordinator(c.Coordinator, *oldURI) + if c.Coordinator != *oldURI { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, *oldURI) + } + + // Set coordinator to a new value. + c.SetCoordinator(c.Coordinator, *newURI) + if c.Coordinator != *newURI { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, *newURI) + } + }) +} diff --git a/handler.go b/handler.go index 91d128e7c..e0177ba35 100644 --- a/handler.go +++ b/handler.go @@ -123,6 +123,7 @@ func (h *Handler) SetRestricted() { } func loadCommon(router *mux.Router, handler *Handler) { + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") @@ -140,7 +141,6 @@ func loadRestricted(router *mux.Router, handler *Handler) { func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET") @@ -1927,7 +1927,67 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -//handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +// handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request. +func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req setCoordinatorRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + oldURI := h.Cluster.Coordinator + + var newURI *URI + if err := func() error { + newURI, err = NewURIFromAddress(req.Address) + if err != nil { + return fmt.Errorf("problem with set-coordinator address: %s", err) + } + + //if !Nodes(h.Cluster.Nodes).ContainsURI(*newURI) { + // return fmt.Errorf("set-coordinator node does not exist: %s", newURI) + //} + + // Send the set-coordinator message to all nodes. + err := h.Broadcaster.SendSync( + &internal.SetCoordinatorMessage{ + Old: (&h.Cluster.Coordinator).Encode(), + New: newURI.Encode(), + }) + if err != nil { + return fmt.Errorf("problem sending SetCoordinator message: %s", err) + } + + // Set Coordinator on local node. + h.Cluster.SetCoordinator(oldURI, *newURI) + + return nil + }(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: &oldURI, + New: newURI, + }); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type setCoordinatorRequest struct { + Address string `json:"address"` +} + +type setCoordinatorResponse struct { + Old *URI `json:"old"` + New *URI `json:"new"` +} + +// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { var msg string diff --git a/handler_test.go b/handler_test.go index d6a5e7a3f..05116ef8c 100644 --- a/handler_test.go +++ b/handler_test.go @@ -158,6 +158,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { h := test.NewHandler() h.Cluster = test.NewCluster(1) + h.SetRestricted() w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) diff --git a/internal/private.pb.go b/internal/private.pb.go index 2c5355f69..172bc0be2 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -36,6 +36,7 @@ ResizeInstruction ResizeSource ResizeInstructionComplete + SetCoordinatorMessage Topology */ package internal @@ -927,6 +928,30 @@ func (m *ResizeInstructionComplete) GetError() string { return "" } +type SetCoordinatorMessage struct { + Old *URI `protobuf:"bytes,1,opt,name=Old" json:"Old,omitempty"` + New *URI `protobuf:"bytes,2,opt,name=New" json:"New,omitempty"` +} + +func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } +func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*SetCoordinatorMessage) ProtoMessage() {} +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } + +func (m *SetCoordinatorMessage) GetOld() *URI { + if m != nil { + return m.Old + } + return nil +} + +func (m *SetCoordinatorMessage) GetNew() *URI { + if m != nil { + return m.New + } + return nil +} + type Topology struct { NodeSet []*URI `protobuf:"bytes,1,rep,name=NodeSet" json:"NodeSet,omitempty"` } @@ -934,7 +959,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } func (m *Topology) GetNodeSet() []*URI { if m != nil { @@ -972,6 +997,7 @@ func init() { proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") proto.RegisterType((*ResizeSource)(nil), "internal.ResizeSource") proto.RegisterType((*ResizeInstructionComplete)(nil), "internal.ResizeInstructionComplete") + proto.RegisterType((*SetCoordinatorMessage)(nil), "internal.SetCoordinatorMessage") proto.RegisterType((*Topology)(nil), "internal.Topology") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -2137,6 +2163,44 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *SetCoordinatorMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Old != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Old.Size())) + n19, err := m.Old.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n19 + } + if m.New != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n20, err := m.New.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n20 + } + return i, nil +} + func (m *Topology) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2690,6 +2754,20 @@ func (m *ResizeInstructionComplete) Size() (n int) { return n } +func (m *SetCoordinatorMessage) Size() (n int) { + var l int + _ = l + if m.Old != nil { + l = m.Old.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.New != nil { + l = m.New.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *Topology) Size() (n int) { var l int _ = l @@ -6688,6 +6766,122 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } return nil } +func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetCoordinatorMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetCoordinatorMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Old", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Old == nil { + m.Old = &URI{} + } + if err := m.Old.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field New", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.New == nil { + m.New = &URI{} + } + if err := m.New.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Topology) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -6877,77 +7071,79 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1149 bytes of a gzipped FileDescriptorProto + // 1179 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4f, 0x6f, 0xe3, 0x44, - 0x14, 0xc7, 0x71, 0x92, 0x26, 0x2f, 0xcd, 0x36, 0x1d, 0x4a, 0x95, 0x56, 0x55, 0x36, 0xcc, 0x81, - 0x96, 0x95, 0x28, 0xd0, 0x4a, 0x08, 0x8a, 0x90, 0x60, 0x9b, 0xac, 0xd6, 0x40, 0xcb, 0x32, 0xe9, - 0x2e, 0x12, 0x07, 0xa4, 0x69, 0x32, 0xb4, 0x56, 0x1d, 0x3b, 0xd8, 0x93, 0xb6, 0xd9, 0x03, 0x37, - 0x38, 0xc0, 0x17, 0xe0, 0xce, 0x97, 0xe1, 0xc8, 0x8d, 0x2b, 0x2a, 0x1f, 0x02, 0x89, 0x0b, 0xab, - 0x79, 0x9e, 0xb1, 0x9d, 0x7f, 0x8d, 0xda, 0x9b, 0xdf, 0x9b, 0xf7, 0xde, 0xfc, 0xe6, 0xf7, 0xfe, - 0xcc, 0x18, 0xaa, 0x83, 0xd0, 0xbd, 0xe4, 0x52, 0xec, 0x0e, 0xc2, 0x40, 0x06, 0xa4, 0xe4, 0xfa, - 0x52, 0x84, 0x3e, 0xf7, 0xe8, 0x57, 0x50, 0x76, 0xfc, 0x9e, 0xb8, 0x3e, 0x12, 0x92, 0x93, 0x26, - 0x54, 0x0e, 0x03, 0x6f, 0xd8, 0xf7, 0xbf, 0xe4, 0xa7, 0xc2, 0xab, 0x5b, 0x4d, 0x6b, 0xa7, 0xcc, - 0xb2, 0x2a, 0x65, 0x71, 0xe2, 0xf6, 0xc5, 0xd7, 0x43, 0xee, 0xcb, 0x61, 0xbf, 0x9e, 0x8b, 0x2d, - 0x32, 0x2a, 0xfa, 0x9f, 0x05, 0xe5, 0x27, 0x21, 0xef, 0x0b, 0x8c, 0xb8, 0x09, 0x25, 0x16, 0x5c, - 0x65, 0xc3, 0x25, 0x32, 0x79, 0x0b, 0x1e, 0x38, 0xfe, 0xa5, 0x08, 0x23, 0xd1, 0xf6, 0xf9, 0xa9, - 0x27, 0x7a, 0x18, 0xae, 0xc4, 0x26, 0xb4, 0x64, 0x0b, 0xca, 0x87, 0xbc, 0x7b, 0x2e, 0x4e, 0x46, - 0x03, 0x51, 0xb7, 0x31, 0x48, 0xaa, 0x48, 0x56, 0x3b, 0xee, 0x4b, 0x51, 0xcf, 0x37, 0xad, 0x9d, - 0x2a, 0x4b, 0x15, 0x93, 0x78, 0x0b, 0x53, 0x78, 0x09, 0x85, 0x65, 0xc6, 0xfd, 0xb3, 0x04, 0x43, - 0x11, 0x31, 0x8c, 0xe9, 0xc8, 0x36, 0x14, 0x9f, 0xb8, 0xc2, 0xeb, 0x45, 0xf5, 0xa5, 0xa6, 0xbd, - 0x53, 0xd9, 0x5b, 0xd9, 0x35, 0xfc, 0xed, 0xa2, 0x9e, 0xe9, 0x65, 0x4a, 0xe1, 0x81, 0xd3, 0x1f, - 0x04, 0xa1, 0x64, 0x22, 0x1a, 0x04, 0x7e, 0x24, 0x48, 0x0d, 0xec, 0x76, 0x18, 0xea, 0xb3, 0xab, - 0x4f, 0xfa, 0x23, 0xd4, 0x1e, 0x7b, 0x41, 0xf7, 0xa2, 0xc5, 0x25, 0x67, 0xe2, 0x87, 0xa1, 0x88, - 0x24, 0x59, 0x83, 0x02, 0x66, 0x41, 0xdb, 0xc5, 0x82, 0xd2, 0x22, 0x93, 0x9a, 0xe6, 0x58, 0x50, - 0x5a, 0xf4, 0x47, 0x2a, 0xf2, 0x2c, 0x16, 0x94, 0xb6, 0xe3, 0xb9, 0xdd, 0x98, 0x82, 0x3c, 0x8b, - 0x05, 0x42, 0x20, 0xff, 0xc2, 0x15, 0x57, 0xfa, 0xdc, 0xf8, 0x4d, 0x1d, 0x58, 0xcd, 0xec, 0xaf, - 0x61, 0xae, 0x43, 0x91, 0x05, 0x57, 0x4e, 0x2b, 0xaa, 0x5b, 0x4d, 0x7b, 0x27, 0xcf, 0xb4, 0x84, - 0xec, 0x62, 0xfa, 0xd5, 0x52, 0x0e, 0x97, 0x52, 0x05, 0xdd, 0x80, 0x02, 0x52, 0xad, 0x4e, 0x99, - 0xfa, 0xaa, 0x4f, 0xfa, 0xbf, 0x05, 0xe5, 0x23, 0x7e, 0x8d, 0x30, 0x22, 0xf2, 0x09, 0x94, 0x3a, - 0x92, 0xfb, 0x3d, 0x1e, 0xf6, 0xd0, 0xa8, 0xb2, 0xf7, 0x66, 0x4a, 0x61, 0x62, 0xb6, 0x6b, 0x6c, - 0xda, 0xbe, 0x0c, 0x47, 0x2c, 0x71, 0x21, 0x07, 0xb0, 0xa4, 0x6b, 0x02, 0x31, 0x54, 0xf6, 0x9a, - 0xb3, 0xbc, 0x93, 0xb2, 0x51, 0xce, 0xc6, 0x61, 0xf3, 0x63, 0xa8, 0x8e, 0x85, 0x55, 0x58, 0x2f, - 0xc4, 0xc8, 0x64, 0xe4, 0x42, 0x8c, 0x14, 0x77, 0x97, 0xdc, 0x1b, 0xc6, 0x3c, 0xe7, 0x59, 0x2c, - 0x1c, 0xe4, 0x3e, 0xb4, 0x36, 0x0f, 0x60, 0x39, 0x1b, 0xf5, 0x2e, 0xbe, 0xf4, 0x3b, 0x20, 0x87, - 0xa1, 0xe0, 0x52, 0x20, 0xbc, 0x23, 0x11, 0x45, 0xfc, 0x4c, 0xcc, 0xcf, 0x74, 0x9c, 0xbd, 0x5c, - 0x36, 0x7b, 0x5b, 0x50, 0x76, 0x22, 0x73, 0x70, 0x1b, 0xeb, 0x32, 0x55, 0xd0, 0x47, 0x40, 0x5a, - 0xc2, 0x13, 0x52, 0xe8, 0xfe, 0xbd, 0x25, 0x3e, 0xed, 0x18, 0x2c, 0x8b, 0x6d, 0xc9, 0x36, 0xe4, - 0x55, 0xeb, 0x22, 0x94, 0xca, 0xde, 0xeb, 0x29, 0xd3, 0xc9, 0x9c, 0x60, 0x68, 0x40, 0x5d, 0x13, - 0x54, 0xb7, 0xfb, 0x82, 0x03, 0xce, 0x28, 0x65, 0xb3, 0x95, 0x3d, 0xb9, 0x55, 0x32, 0x40, 0xf4, - 0x56, 0x9f, 0x9a, 0xb3, 0xde, 0x77, 0x2b, 0xfa, 0xad, 0xd6, 0xaa, 0x96, 0x38, 0x56, 0xab, 0xb1, - 0x0f, 0x7e, 0xcf, 0x3f, 0xf2, 0x04, 0x0e, 0x15, 0x5b, 0xf5, 0x50, 0x54, 0xb7, 0x9b, 0xb6, 0x8a, - 0x8d, 0x02, 0xdd, 0x87, 0x62, 0xa7, 0x7b, 0x2e, 0xfa, 0x9c, 0xbc, 0xad, 0x0a, 0xb5, 0x27, 0xae, - 0x45, 0xa4, 0xcb, 0x7c, 0x65, 0x82, 0x3e, 0x66, 0xd6, 0xe9, 0xaf, 0x96, 0x46, 0x3f, 0x07, 0x51, - 0x11, 0xf7, 0x8e, 0xea, 0xf9, 0xa9, 0x89, 0xa3, 0xf4, 0x4c, 0x2f, 0x93, 0x36, 0xd4, 0x1c, 0x7f, - 0x30, 0x94, 0x2d, 0xf1, 0xbd, 0xeb, 0xbb, 0xd2, 0x0d, 0xfc, 0xa8, 0x5e, 0x44, 0x97, 0x8d, 0xec, - 0xd6, 0x63, 0x16, 0x6c, 0xca, 0x85, 0xfe, 0x6c, 0xc1, 0xca, 0x84, 0x72, 0x01, 0xae, 0xdc, 0xed, - 0xb8, 0x3e, 0x48, 0x46, 0xa6, 0x8d, 0x86, 0x8d, 0xb9, 0x68, 0xc6, 0x27, 0xe8, 0xef, 0x16, 0xac, - 0xcd, 0x32, 0x98, 0x89, 0xa6, 0x01, 0xf0, 0x2c, 0x74, 0xfb, 0x3c, 0x1c, 0x7d, 0x21, 0x46, 0xfa, - 0xf6, 0xc8, 0x68, 0xc8, 0x37, 0xb0, 0x3e, 0x11, 0xeb, 0xb3, 0x6e, 0x4c, 0x51, 0x0c, 0xea, 0xe1, - 0x5c, 0x50, 0xb1, 0x1d, 0x9b, 0xe3, 0x4e, 0xff, 0xb5, 0xe0, 0x8d, 0x99, 0x4b, 0x69, 0xf5, 0x59, - 0xd9, 0x42, 0x7f, 0x04, 0xb5, 0x17, 0x6a, 0x30, 0xb4, 0x44, 0x24, 0x5d, 0x9f, 0x2b, 0x4b, 0x5d, - 0x9e, 0x53, 0x7a, 0xe2, 0x40, 0x09, 0x75, 0x47, 0x7c, 0xa0, 0x61, 0xbe, 0xb3, 0x00, 0xe6, 0xae, - 0xb1, 0xd7, 0x73, 0xd3, 0x88, 0x0a, 0x0c, 0xce, 0x71, 0x73, 0x29, 0xa0, 0xa0, 0x26, 0xe2, 0x98, - 0xc3, 0x9d, 0xa6, 0x5a, 0x00, 0x5b, 0x66, 0x92, 0x8c, 0x21, 0xb9, 0xbd, 0x27, 0x3f, 0x02, 0x48, - 0x4d, 0x75, 0xbb, 0xdf, 0x52, 0x9f, 0x19, 0x63, 0xfa, 0x14, 0xb6, 0xcc, 0x98, 0xbb, 0xc3, 0x86, - 0xa6, 0x5a, 0x72, 0x69, 0xb5, 0xd0, 0x36, 0xd8, 0xcf, 0x99, 0xa3, 0xae, 0x3a, 0xec, 0x56, 0x93, - 0x22, 0x2d, 0x29, 0x97, 0xa7, 0x41, 0x24, 0x8d, 0x8b, 0xfa, 0x56, 0xba, 0x67, 0x41, 0x28, 0x11, - 0x71, 0x95, 0xe1, 0x37, 0xfd, 0xc5, 0x02, 0x38, 0x0e, 0x7a, 0xa2, 0x23, 0xb9, 0x1c, 0x46, 0xe4, - 0x21, 0x46, 0xc5, 0x58, 0x95, 0xbd, 0x6a, 0x7a, 0xa6, 0xe7, 0xcc, 0x61, 0xb8, 0xdf, 0xfb, 0x99, - 0x8b, 0x70, 0x7a, 0xc2, 0x24, 0x4b, 0x2c, 0x73, 0x5d, 0xee, 0x98, 0x81, 0xa2, 0xa9, 0xaa, 0xa5, - 0xf6, 0xb1, 0x5e, 0x83, 0xe6, 0xf4, 0x18, 0xaa, 0x87, 0xde, 0x30, 0x92, 0x22, 0xd4, 0x70, 0xd4, - 0x4d, 0x22, 0xb9, 0x4c, 0xea, 0x0f, 0x05, 0xb2, 0x0d, 0x4b, 0x08, 0x59, 0x48, 0xdd, 0xb7, 0x13, - 0x40, 0xcd, 0x2a, 0xed, 0x40, 0x61, 0x7e, 0xbb, 0x11, 0xc8, 0xe3, 0x1b, 0x4c, 0x33, 0x84, 0xcf, - 0xaf, 0x1a, 0xd8, 0x47, 0x6e, 0x9c, 0x52, 0x9b, 0xa9, 0x4f, 0xd4, 0xf0, 0x6b, 0x2c, 0x39, 0xa5, - 0xe1, 0xea, 0xf6, 0x59, 0x8d, 0x53, 0xa8, 0xc6, 0xe5, 0x7d, 0xee, 0x09, 0xf3, 0x8c, 0xb1, 0x33, - 0xcf, 0x98, 0xbf, 0x2c, 0x58, 0x65, 0x22, 0x72, 0x5f, 0x0a, 0xc7, 0x8f, 0x64, 0x38, 0x4c, 0xda, - 0xef, 0xf3, 0xe0, 0xd4, 0x69, 0x61, 0x54, 0x9b, 0xc5, 0x82, 0xc9, 0x51, 0x6e, 0x6e, 0x8e, 0xde, - 0x55, 0x0f, 0xdf, 0x20, 0xec, 0xa9, 0x1e, 0x0c, 0x42, 0xcd, 0xfa, 0x84, 0x61, 0xd6, 0x82, 0xbc, - 0x07, 0x4b, 0x9d, 0x60, 0x18, 0x76, 0x93, 0x01, 0xbd, 0x9e, 0x1a, 0xc7, 0xa8, 0xe2, 0x65, 0x66, - 0xcc, 0x32, 0x39, 0x2d, 0x2c, 0xc8, 0xe9, 0x4f, 0x16, 0x2c, 0x67, 0x63, 0x2c, 0x2e, 0xb1, 0x84, - 0xcb, 0xdc, 0x4c, 0x2e, 0xed, 0x59, 0x5c, 0xe6, 0x53, 0x2e, 0xd3, 0xe7, 0x47, 0x21, 0xf3, 0xfc, - 0xa0, 0xe7, 0xb0, 0x31, 0x45, 0xf0, 0x61, 0xd0, 0x1f, 0xa8, 0x4c, 0xde, 0x97, 0xe8, 0x35, 0x28, - 0xb4, 0xc3, 0x50, 0x53, 0x5c, 0x66, 0xb1, 0x40, 0xf7, 0xa1, 0x74, 0x12, 0x0c, 0x02, 0x2f, 0x38, - 0x1b, 0x65, 0x4b, 0xd5, 0xba, 0xad, 0x54, 0x1f, 0xd7, 0xfe, 0xb8, 0x69, 0x58, 0x7f, 0xde, 0x34, - 0xac, 0xbf, 0x6f, 0x1a, 0xd6, 0x6f, 0xff, 0x34, 0x5e, 0x3b, 0x2d, 0xe2, 0xcf, 0xcd, 0xfe, 0xab, - 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x09, 0x36, 0x2c, 0xed, 0x0c, 0x00, 0x00, + 0x14, 0xc7, 0x71, 0x92, 0x26, 0x2f, 0x9b, 0xdd, 0xec, 0xd0, 0xad, 0xd2, 0xaa, 0x4a, 0xc3, 0x1c, + 0x68, 0x59, 0x89, 0x02, 0xad, 0x84, 0xa0, 0x08, 0x09, 0xb6, 0xc9, 0x6a, 0x0d, 0xb4, 0x5d, 0x26, + 0xdd, 0x45, 0x70, 0x40, 0x9a, 0x26, 0x43, 0x6b, 0xd5, 0xb1, 0x83, 0x3d, 0x69, 0x9b, 0x3d, 0x70, + 0x83, 0x03, 0x7c, 0x01, 0xee, 0x7c, 0x19, 0x8e, 0xdc, 0xb8, 0xa2, 0xf2, 0x21, 0x90, 0xb8, 0x80, + 0xe6, 0x79, 0xc6, 0x76, 0xfe, 0x35, 0x6a, 0x6f, 0x7e, 0x6f, 0xde, 0x7b, 0xf3, 0x9b, 0xdf, 0xfb, + 0x33, 0x63, 0xa8, 0x0e, 0x42, 0xf7, 0x82, 0x4b, 0xb1, 0x3d, 0x08, 0x03, 0x19, 0x90, 0x92, 0xeb, + 0x4b, 0x11, 0xfa, 0xdc, 0xa3, 0x47, 0x50, 0x76, 0xfc, 0x9e, 0xb8, 0x3a, 0x10, 0x92, 0x93, 0x26, + 0x54, 0xf6, 0x03, 0x6f, 0xd8, 0xf7, 0xbf, 0xe0, 0x27, 0xc2, 0xab, 0x5b, 0x4d, 0x6b, 0xab, 0xcc, + 0xb2, 0x2a, 0x65, 0x71, 0xec, 0xf6, 0xc5, 0x97, 0x43, 0xee, 0xcb, 0x61, 0xbf, 0x9e, 0x8b, 0x2d, + 0x32, 0x2a, 0xfa, 0xaf, 0x05, 0xe5, 0xa7, 0x21, 0xef, 0x0b, 0x8c, 0xb8, 0x06, 0x25, 0x16, 0x5c, + 0x66, 0xc3, 0x25, 0x32, 0x79, 0x13, 0xee, 0x3b, 0xfe, 0x85, 0x08, 0x23, 0xd1, 0xf6, 0xf9, 0x89, + 0x27, 0x7a, 0x18, 0xae, 0xc4, 0x26, 0xb4, 0x64, 0x1d, 0xca, 0xfb, 0xbc, 0x7b, 0x26, 0x8e, 0x47, + 0x03, 0x51, 0xb7, 0x31, 0x48, 0xaa, 0x48, 0x56, 0x3b, 0xee, 0x2b, 0x51, 0xcf, 0x37, 0xad, 0xad, + 0x2a, 0x4b, 0x15, 0x93, 0x78, 0x0b, 0x53, 0x78, 0x09, 0x85, 0x7b, 0x8c, 0xfb, 0xa7, 0x09, 0x86, + 0x22, 0x62, 0x18, 0xd3, 0x91, 0x4d, 0x28, 0x3e, 0x75, 0x85, 0xd7, 0x8b, 0xea, 0x4b, 0x4d, 0x7b, + 0xab, 0xb2, 0xf3, 0x60, 0xdb, 0xf0, 0xb7, 0x8d, 0x7a, 0xa6, 0x97, 0x29, 0x85, 0xfb, 0x4e, 0x7f, + 0x10, 0x84, 0x92, 0x89, 0x68, 0x10, 0xf8, 0x91, 0x20, 0x35, 0xb0, 0xdb, 0x61, 0xa8, 0xcf, 0xae, + 0x3e, 0xe9, 0x0f, 0x50, 0x7b, 0xe2, 0x05, 0xdd, 0xf3, 0x16, 0x97, 0x9c, 0x89, 0xef, 0x87, 0x22, + 0x92, 0x64, 0x19, 0x0a, 0x98, 0x05, 0x6d, 0x17, 0x0b, 0x4a, 0x8b, 0x4c, 0x6a, 0x9a, 0x63, 0x41, + 0x69, 0xd1, 0x1f, 0xa9, 0xc8, 0xb3, 0x58, 0x50, 0xda, 0x8e, 0xe7, 0x76, 0x63, 0x0a, 0xf2, 0x2c, + 0x16, 0x08, 0x81, 0xfc, 0x4b, 0x57, 0x5c, 0xea, 0x73, 0xe3, 0x37, 0x75, 0xe0, 0x61, 0x66, 0x7f, + 0x0d, 0x73, 0x05, 0x8a, 0x2c, 0xb8, 0x74, 0x5a, 0x51, 0xdd, 0x6a, 0xda, 0x5b, 0x79, 0xa6, 0x25, + 0x64, 0x17, 0xd3, 0xaf, 0x96, 0x72, 0xb8, 0x94, 0x2a, 0xe8, 0x2a, 0x14, 0x90, 0x6a, 0x75, 0xca, + 0xd4, 0x57, 0x7d, 0xd2, 0xff, 0x2c, 0x28, 0x1f, 0xf0, 0x2b, 0x84, 0x11, 0x91, 0x8f, 0xa1, 0xd4, + 0x91, 0xdc, 0xef, 0xf1, 0xb0, 0x87, 0x46, 0x95, 0x9d, 0x37, 0x52, 0x0a, 0x13, 0xb3, 0x6d, 0x63, + 0xd3, 0xf6, 0x65, 0x38, 0x62, 0x89, 0x0b, 0xd9, 0x83, 0x25, 0x5d, 0x13, 0x88, 0xa1, 0xb2, 0xd3, + 0x9c, 0xe5, 0x9d, 0x94, 0x8d, 0x72, 0x36, 0x0e, 0x6b, 0x1f, 0x41, 0x75, 0x2c, 0xac, 0xc2, 0x7a, + 0x2e, 0x46, 0x26, 0x23, 0xe7, 0x62, 0xa4, 0xb8, 0xbb, 0xe0, 0xde, 0x30, 0xe6, 0x39, 0xcf, 0x62, + 0x61, 0x2f, 0xf7, 0x81, 0xb5, 0xb6, 0x07, 0xf7, 0xb2, 0x51, 0x6f, 0xe3, 0x4b, 0xbf, 0x05, 0xb2, + 0x1f, 0x0a, 0x2e, 0x05, 0xc2, 0x3b, 0x10, 0x51, 0xc4, 0x4f, 0xc5, 0xfc, 0x4c, 0xc7, 0xd9, 0xcb, + 0x65, 0xb3, 0xb7, 0x0e, 0x65, 0x27, 0x32, 0x07, 0xb7, 0xb1, 0x2e, 0x53, 0x05, 0x7d, 0x0c, 0xa4, + 0x25, 0x3c, 0x21, 0x85, 0xee, 0xdf, 0x1b, 0xe2, 0xd3, 0x8e, 0xc1, 0xb2, 0xd8, 0x96, 0x6c, 0x42, + 0x5e, 0xb5, 0x2e, 0x42, 0xa9, 0xec, 0xbc, 0x9e, 0x32, 0x9d, 0xcc, 0x09, 0x86, 0x06, 0xd4, 0x35, + 0x41, 0x75, 0xbb, 0x2f, 0x38, 0xe0, 0x8c, 0x52, 0x36, 0x5b, 0xd9, 0x93, 0x5b, 0x25, 0x03, 0x44, + 0x6f, 0xf5, 0x89, 0x39, 0xeb, 0x5d, 0xb7, 0xa2, 0xdf, 0x68, 0xad, 0x6a, 0x89, 0x43, 0xb5, 0x1a, + 0xfb, 0xe0, 0xf7, 0xfc, 0x23, 0x4f, 0xe0, 0x50, 0xb1, 0x55, 0x0f, 0x45, 0x75, 0xbb, 0x69, 0xab, + 0xd8, 0x28, 0xd0, 0x5d, 0x28, 0x76, 0xba, 0x67, 0xa2, 0xcf, 0xc9, 0x5b, 0xaa, 0x50, 0x7b, 0xe2, + 0x4a, 0x44, 0xba, 0xcc, 0x1f, 0x4c, 0xd0, 0xc7, 0xcc, 0x3a, 0xfd, 0xc5, 0xd2, 0xe8, 0xe7, 0x20, + 0x2a, 0xe2, 0xde, 0x51, 0x3d, 0x3f, 0x35, 0x71, 0x94, 0x9e, 0xe9, 0x65, 0xd2, 0x86, 0x9a, 0xe3, + 0x0f, 0x86, 0xb2, 0x25, 0xbe, 0x73, 0x7d, 0x57, 0xba, 0x81, 0x1f, 0xd5, 0x8b, 0xe8, 0xb2, 0x9a, + 0xdd, 0x7a, 0xcc, 0x82, 0x4d, 0xb9, 0xd0, 0x9f, 0x2c, 0x78, 0x30, 0xa1, 0x5c, 0x80, 0x2b, 0x77, + 0x33, 0xae, 0xf7, 0x93, 0x91, 0x69, 0xa3, 0x61, 0x63, 0x2e, 0x9a, 0xf1, 0x09, 0xfa, 0x9b, 0x05, + 0xcb, 0xb3, 0x0c, 0x66, 0xa2, 0x69, 0x00, 0x3c, 0x0f, 0xdd, 0x3e, 0x0f, 0x47, 0x9f, 0x8b, 0x91, + 0xbe, 0x3d, 0x32, 0x1a, 0xf2, 0x15, 0xac, 0x4c, 0xc4, 0xfa, 0xb4, 0x1b, 0x53, 0x14, 0x83, 0xda, + 0x98, 0x0b, 0x2a, 0xb6, 0x63, 0x73, 0xdc, 0xe9, 0x3f, 0x16, 0x3c, 0x9a, 0xb9, 0x94, 0x56, 0x9f, + 0x95, 0x2d, 0xf4, 0xc7, 0x50, 0x7b, 0xa9, 0x06, 0x43, 0x4b, 0x44, 0xd2, 0xf5, 0xb9, 0xb2, 0xd4, + 0xe5, 0x39, 0xa5, 0x27, 0x0e, 0x94, 0x50, 0x77, 0xc0, 0x07, 0x1a, 0xe6, 0xdb, 0x0b, 0x60, 0x6e, + 0x1b, 0x7b, 0x3d, 0x37, 0x8d, 0xa8, 0xc0, 0xe0, 0x1c, 0x37, 0x97, 0x02, 0x0a, 0x6a, 0x22, 0x8e, + 0x39, 0xdc, 0x6a, 0xaa, 0x05, 0xb0, 0x6e, 0x26, 0xc9, 0x18, 0x92, 0x9b, 0x7b, 0xf2, 0x43, 0x80, + 0xd4, 0x54, 0xb7, 0xfb, 0x0d, 0xf5, 0x99, 0x31, 0xa6, 0xcf, 0x60, 0xdd, 0x8c, 0xb9, 0x5b, 0x6c, + 0x68, 0xaa, 0x25, 0x97, 0x56, 0x0b, 0x6d, 0x83, 0xfd, 0x82, 0x39, 0xea, 0xaa, 0xc3, 0x6e, 0x35, + 0x29, 0xd2, 0x92, 0x72, 0x79, 0x16, 0x44, 0xd2, 0xb8, 0xa8, 0x6f, 0xa5, 0x7b, 0x1e, 0x84, 0x12, + 0x11, 0x57, 0x19, 0x7e, 0xd3, 0x9f, 0x2d, 0x80, 0xc3, 0xa0, 0x27, 0x3a, 0x92, 0xcb, 0x61, 0x44, + 0x36, 0x30, 0x2a, 0xc6, 0xaa, 0xec, 0x54, 0xd3, 0x33, 0xbd, 0x60, 0x0e, 0xc3, 0xfd, 0xde, 0xcb, + 0x5c, 0x84, 0xd3, 0x13, 0x26, 0x59, 0x62, 0x99, 0xeb, 0x72, 0xcb, 0x0c, 0x14, 0x4d, 0x55, 0x2d, + 0xb5, 0x8f, 0xf5, 0x1a, 0x34, 0xa7, 0x87, 0x50, 0xdd, 0xf7, 0x86, 0x91, 0x14, 0xa1, 0x86, 0xa3, + 0x6e, 0x12, 0xc9, 0x65, 0x52, 0x7f, 0x28, 0x90, 0x4d, 0x58, 0x42, 0xc8, 0x42, 0xea, 0xbe, 0x9d, + 0x00, 0x6a, 0x56, 0x69, 0x07, 0x0a, 0xf3, 0xdb, 0x8d, 0x40, 0x1e, 0xdf, 0x60, 0x9a, 0x21, 0x7c, + 0x7e, 0xd5, 0xc0, 0x3e, 0x70, 0xe3, 0x94, 0xda, 0x4c, 0x7d, 0xa2, 0x86, 0x5f, 0x61, 0xc9, 0x29, + 0x0d, 0x57, 0xb7, 0xcf, 0xc3, 0x38, 0x85, 0x6a, 0x5c, 0xde, 0xe5, 0x9e, 0x30, 0xcf, 0x18, 0x3b, + 0xf3, 0x8c, 0xf9, 0xd3, 0x82, 0x87, 0x4c, 0x44, 0xee, 0x2b, 0xe1, 0xf8, 0x91, 0x0c, 0x87, 0x49, + 0xfb, 0x7d, 0x16, 0x9c, 0x38, 0x2d, 0x8c, 0x6a, 0xb3, 0x58, 0x30, 0x39, 0xca, 0xcd, 0xcd, 0xd1, + 0x3b, 0xea, 0xe1, 0x1b, 0x84, 0x3d, 0xd5, 0x83, 0x41, 0xa8, 0x59, 0x9f, 0x30, 0xcc, 0x5a, 0x90, + 0x77, 0x61, 0xa9, 0x13, 0x0c, 0xc3, 0x6e, 0x32, 0xa0, 0x57, 0x52, 0xe3, 0x18, 0x55, 0xbc, 0xcc, + 0x8c, 0x59, 0x26, 0xa7, 0x85, 0x05, 0x39, 0xfd, 0xd1, 0x82, 0x7b, 0xd9, 0x18, 0x8b, 0x4b, 0x2c, + 0xe1, 0x32, 0x37, 0x93, 0x4b, 0x7b, 0x16, 0x97, 0xf9, 0x94, 0xcb, 0xf4, 0xf9, 0x51, 0xc8, 0x3c, + 0x3f, 0xe8, 0x19, 0xac, 0x4e, 0x11, 0xbc, 0x1f, 0xf4, 0x07, 0x2a, 0x93, 0x77, 0x25, 0x7a, 0x19, + 0x0a, 0xed, 0x30, 0xd4, 0x14, 0x97, 0x59, 0x2c, 0xd0, 0xaf, 0xe1, 0x51, 0x47, 0xc8, 0x0c, 0xbf, + 0xa6, 0x48, 0x36, 0xc0, 0x3e, 0xf2, 0x7a, 0x73, 0x4e, 0x7e, 0xe4, 0xf5, 0x94, 0xc1, 0xa1, 0xb8, + 0x9c, 0xb3, 0xe1, 0xa1, 0xb8, 0xa4, 0xbb, 0x50, 0x3a, 0x0e, 0x06, 0x81, 0x17, 0x9c, 0x8e, 0xb2, + 0x5d, 0x60, 0xdd, 0xd4, 0x05, 0x4f, 0x6a, 0xbf, 0x5f, 0x37, 0xac, 0x3f, 0xae, 0x1b, 0xd6, 0x5f, + 0xd7, 0x0d, 0xeb, 0xd7, 0xbf, 0x1b, 0xaf, 0x9d, 0x14, 0xf1, 0xbf, 0x69, 0xf7, 0xff, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x80, 0x0a, 0xbc, 0x80, 0x48, 0x0d, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 56c402eca..a97ac0e2d 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -167,6 +167,11 @@ message ResizeInstructionComplete { string Error = 3; } +message SetCoordinatorMessage { + URI Old = 1; + URI New = 2; +} + message Topology { repeated URI NodeSet = 1; } diff --git a/server.go b/server.go index da6cc8a1a..55fd62249 100644 --- a/server.go +++ b/server.go @@ -356,6 +356,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.SetCoordinatorMessage: + s.Cluster.SetCoordinator(DecodeURI(obj.Old), DecodeURI(obj.New)) } return nil From 2517b994a11db5cb7cf8ca7bbd4890401f918db7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 14 Nov 2017 13:28:00 -0600 Subject: [PATCH 031/311] Temporarily disable Go master CI as builds are failing due to possible Go bug (See #956) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd97d53e8..2377d30f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: go go: - 1.8 - 1.9 - - master env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" From a98b862fcae859101068f7b259d2b457361bd5f8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 20 Nov 2017 18:26:36 -0600 Subject: [PATCH 032/311] add /cluster/resize/remove-node endpoint --- cluster.go | 383 ++++++++++++++++++++++++++++++--------- cluster_internal_test.go | 233 +++++++++++++++++++++++- cluster_test.go | 75 -------- gossip/gossip.go | 2 +- handler.go | 52 ++++++ index.go | 1 - server.go | 17 ++ 7 files changed, 597 insertions(+), 166 deletions(-) diff --git a/cluster.go b/cluster.go index fa6651eab..befc66c1a 100644 --- a/cluster.go +++ b/cluster.go @@ -51,6 +51,9 @@ const ( // Final states. ResizeJobStateDone = "DONE" ResizeJobStateAborted = "ABORTED" + + ResizeJobActionAdd = "ADD" + ResizeJobActionRemove = "REMOVE" ) // Node represents a node in the cluster. @@ -127,6 +130,12 @@ func (h ByHost) Len() int { return len(h) } func (h ByHost) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h ByHost) Less(i, j int) bool { return h[i].URI.String() < h[j].URI.String() } +// nodeAction represents a node that is joining or leaving the cluster. +type nodeAction struct { + uri URI + action string +} + // Cluster represents a collection of nodes. type Cluster struct { URI URI @@ -158,7 +167,7 @@ type Cluster struct { Holder *Holder Broadcaster Broadcaster - joiningURIs chan URI + joiningLeavingNodes chan nodeAction mu sync.RWMutex jobs map[int64]*ResizeJob @@ -181,9 +190,9 @@ func NewCluster() *Cluster { ReplicaN: DefaultReplicaN, EventReceiver: NopEventReceiver, - joiningURIs: make(chan URI, 10), // buffered channel - jobs: make(map[int64]*ResizeJob), - closing: make(chan struct{}), + joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel + jobs: make(map[int64]*ResizeJob), + closing: make(chan struct{}), LogOutput: os.Stderr, prefect: &NopSecurityManager{}, @@ -215,7 +224,7 @@ func (c *Cluster) SetCoordinator(oldURI, newURI URI) bool { func (c *Cluster) AddNode(uri URI) error { // add to cluster - _, added := c.AddNodeBasicSorted(uri) + _, added := c.addNodeBasicSorted(uri) if !added { return nil } @@ -232,6 +241,27 @@ func (c *Cluster) AddNode(uri URI) error { return c.saveTopology() } +// RemoveNode removes a node from the Cluster and updates and saves the +// new topology. +func (c *Cluster) RemoveNode(uri URI) error { + // remove from cluster + removed := c.removeNodeBasicSorted(uri) + if !removed { + return nil + } + + // remove from topology + if c.Topology == nil { + return fmt.Errorf("Cluster.Topology is nil") + } + if !c.Topology.RemoveURI(uri) { + return nil + } + + // save topology + return c.saveTopology() +} + // NodeSet returns the list of uris in the cluster. func (c *Cluster) NodeSet() []URI { return Nodes(c.Nodes).URIs() @@ -278,9 +308,19 @@ func (c *Cluster) NodeByURI(uri URI) *Node { return nil } -// AddNodeBasicSorted adds a node to the cluster, sorted by uri. +// nodePositionByURI returns the position of the node in slice c.Nodes. +func (c *Cluster) nodePositionByURI(uri URI) int { + for i, n := range c.Nodes { + if n.URI == uri { + return i + } + } + return -1 +} + +// addNodeBasicSorted adds a node to the cluster, sorted by uri. // Returns a pointer to the node and true if the node was added. -func (c *Cluster) AddNodeBasicSorted(uri URI) (*Node, bool) { +func (c *Cluster) addNodeBasicSorted(uri URI) (*Node, bool) { n := c.NodeByURI(uri) if n != nil { return n, false @@ -295,6 +335,21 @@ func (c *Cluster) AddNodeBasicSorted(uri URI) (*Node, bool) { return n, true } +// removeNodeBasicSorted removes a node from the cluster, maintaining +// the sort order. Returns true if the node was removed. +func (c *Cluster) removeNodeBasicSorted(uri URI) bool { + i := c.nodePositionByURI(uri) + if i < 0 { + return false + } + + copy(c.Nodes[i:], c.Nodes[i+1:]) + c.Nodes[len(c.Nodes)-1] = nil + c.Nodes = c.Nodes[:len(c.Nodes)-1] + + return true +} + // frag is a struct of basic fragment information. type frag struct { frame string @@ -363,8 +418,8 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { - f := c.FragmentNodes(idx, i) - for _, n := range f { + nodes := c.FragmentNodes(idx, i) + for _, n := range nodes { // for each frame/view combination: for frame, views := range frameViews { for _, view := range views { @@ -376,21 +431,70 @@ func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFram return t } -// DataDiff returns a list of ResizeSources - for each host in the `to` cluster - +// diff compares c with another cluster and determines if a node is being +// added or removed. An error is returned for any case other than where +// exactly one node is added or removed. +func (c *Cluster) diff(other *Cluster) (action string, uri URI, err error) { + lenFrom := len(c.Nodes) + lenTo := len(other.Nodes) + // Determine if a node is being added or removed. + if lenFrom == lenTo { + return action, uri, errors.New("clusters are the same size") + } + if lenFrom < lenTo { + // Adding a node. + if lenTo-lenFrom > 1 { + return action, uri, errors.New("adding more than one node at a time is not supported") + } + action = ResizeJobActionAdd + // Determine the URI that is being added. + for _, n := range other.Nodes { + if c.NodeByURI(n.URI) == nil { + uri = n.URI + break + } + } + } else if len(c.Nodes) > len(other.Nodes) { + // Removing a node. + if lenFrom-lenTo > 1 { + return action, uri, errors.New("removing more than one node at a time is not supported") + } + action = ResizeJobActionRemove + // Determine the URI that is being removed. + for _, n := range c.Nodes { + if other.NodeByURI(n.URI) == nil { + uri = n.URI + break + } + } + } + return action, uri, nil +} + +// fragSources returns a list of ResizeSources - for each node in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[URI][]*internal.ResizeSource { +func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[URI][]*internal.ResizeSource, error) { m := make(map[URI][]*internal.ResizeSource) + // Determine if a node is being added or removed. + action, diffURI, err := c.diff(to) + if err != nil { + return nil, err + } + // Initialize the map with all the nodes in `to`. for _, n := range to.Nodes { m[n.URI] = nil } - // For now, we want our source to be confined to the primary fragment - // (i.e. don't use replicas as source data). So if it's not already, - // base our source fragments on a cluster with replica = 1. + // If a node is being added, the source can be confined to the + // primary fragments (i.e. no need to use replicas as source data). + // In this case, source fragments can be based on a cluster with + // replica = 1. + // If a node is being removed, however, then it will most likely + // require that a replica fragment be the source data. srcCluster := c - if c.ReplicaN > 1 { + if action == ResizeJobActionAdd && c.ReplicaN > 1 { srcCluster = NewCluster() srcCluster.Nodes = Nodes(c.Nodes).Clone() srcCluster.Hasher = c.Hasher @@ -408,6 +512,10 @@ func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[URI][]*internal.ResizeSo // srcHostsByFrag is the inverse representation of srcFrags. srcHostsByFrag := make(map[frag]URI) for uri, frags := range srcFrags { + // If a node is being removed, don't consider it as a source. + if action == ResizeJobActionRemove && uri == diffURI { + continue + } for _, frag := range frags { srcHostsByFrag[frag] = uri } @@ -427,18 +535,28 @@ func (c *Cluster) DataDiff(to *Cluster, idx *Index) map[URI][]*internal.ResizeSo for host, diff := range diffs { m[host] = []*internal.ResizeSource{} for _, frag := range diff { + // If there is no valid source URI for a fragment, + // it likely means that the replica factor was not + // high enough for the remaining nodes to contain + // the fragment. + srcHost, ok := srcHostsByFrag[frag] + if !ok { + return nil, errors.New("not enough data to perform resize") + } + src := &internal.ResizeSource{ - URI: (srcHostsByFrag[frag]).Encode(), + URI: (srcHost).Encode(), Index: idx.Name(), Frame: frag.frame, View: frag.view, Slice: frag.slice, } + m[host] = append(m[host], src) } } - return m + return m, nil } // Partition returns the partition that a slice belongs to. @@ -575,8 +693,8 @@ func (c *Cluster) haveTopologyAgreement() bool { return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } -func (c *Cluster) handleJoiningHost(uri URI) error { - j, err := c.GenerateResizeJob(uri) +func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { + j, err := c.generateResizeJob(nodeAction) if err != nil { return err } @@ -594,8 +712,12 @@ func (c *Cluster) handleJoiningHost(uri URI) error { if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { return err } - // Add uri to the cluster. - return c.AddNode(uri) + // Add/remove uri to/from the cluster. + if j.action == ResizeJobActionRemove { + return c.RemoveNode(nodeAction.uri) + } else if j.action == ResizeJobActionAdd { + return c.AddNode(nodeAction.uri) + } case ResizeJobStateAborted: if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { return err @@ -623,10 +745,10 @@ func (c *Cluster) listenForJoins() { // Handle all pending joins before changing state back to NORMAL. select { - case uri := <-c.joiningURIs: - err := c.handleJoiningHost(uri) + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) if err != nil { - c.logger().Printf("handleJoiningHost error: err=%s", err) + c.logger().Printf("handleNodeAction error: err=%s", err) continue } uriJoined = true @@ -646,10 +768,10 @@ func (c *Cluster) listenForJoins() { select { case <-c.closing: return - case uri := <-c.joiningURIs: - err := c.handleJoiningHost(uri) + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) if err != nil { - c.logger().Printf("handleJoiningHost error: err=%s", err) + c.logger().Printf("handleNodeAction error: err=%s", err) continue } uriJoined = true @@ -658,14 +780,17 @@ func (c *Cluster) listenForJoins() { } } -// GenerateResizeJob creates a new ResizeJob based on the new host being -// added. It also saves a reference to the ResizeJob in the `jobs` map +// generateResizeJob creates a new ResizeJob based on the new node being +// added/removed. It also saves a reference to the ResizeJob in the `jobs` map // for future lookup by JobID. -func (c *Cluster) GenerateResizeJob(addURI URI) (*ResizeJob, error) { +func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { c.mu.Lock() defer c.mu.Unlock() - j := c.generateResizeJob(addURI) + j, err := c.generateResizeJobByAction(nodeAction) + if err != nil { + return nil, err + } // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -679,51 +804,57 @@ func (c *Cluster) GenerateResizeJob(addURI URI) (*ResizeJob, error) { return j, nil } -// generateResizeJob returns a ResizeJob with instructions based on -// the difference between Cluster and a new Cluster containing addHost. +// generateResizeJobByAction returns a ResizeJob with instructions based on +// the difference between Cluster and a new Cluster with/without uri. // Broadcaster is associated to the ResizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. -func (c *Cluster) generateResizeJob(addURI URI) *ResizeJob { +func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { - j := NewResizeJob(addURI, Nodes(c.Nodes).URIs()) + j := NewResizeJob(Nodes(c.Nodes).URIs(), nodeAction.uri, nodeAction.action) j.Broadcaster = c.Broadcaster - // toCluster is a clone of Cluster with the new node added for comparison. + // toCluster is a clone of Cluster with the new node added/removed for comparison. toCluster := NewCluster() toCluster.Nodes = Nodes(c.Nodes).Clone() toCluster.Hasher = c.Hasher toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN - toCluster.AddNodeBasicSorted(addURI) + if nodeAction.action == ResizeJobActionRemove { + toCluster.removeNodeBasicSorted(nodeAction.uri) + } else if nodeAction.action == ResizeJobActionAdd { + toCluster.addNodeBasicSorted(nodeAction.uri) + } pbSchema := c.Holder.EncodeSchema() // Add to the ResizeJob the instructions for each index. for _, idx := range c.Holder.Indexes() { - // dataDiff is map[string][]*internal.ResizeSource, where string is - // a host in toCluster. - dataDiff := c.DataDiff(toCluster, idx) + // fragSources is map[URI][]*internal.ResizeSource. + fragSources, err := c.fragSources(toCluster, idx) + if err != nil { + return nil, err + } - for uri, sources := range dataDiff { + for u, sources := range fragSources { // If a host doesn't need to request data, mark it as complete. if len(sources) == 0 { - j.URIs[uri] = true + j.URIs[u] = true continue } - // TODO: we can probably consilidate the instructions that go to the same + // TODO: we can probably consolidate the instructions that go to the same // node but apply to different indexes. (i.e. don't nest this in the Indexes() loop) instr := &internal.ResizeInstruction{ JobID: j.ID, - URI: uri.Encode(), + URI: u.Encode(), Coordinator: encodeURI(c.Coordinator), Sources: sources, - Schema: pbSchema, + Schema: pbSchema, // Include the schema to ensure it's in sync on the receiving node. } j.Instructions = append(j.Instructions, instr) } } - return j + return j, nil } // CompleteCurrentJob sets the state of the current ResizeJob @@ -862,6 +993,7 @@ type ResizeJob struct { Instructions []*internal.ResizeInstruction Broadcaster Broadcaster + action string result chan string mu sync.RWMutex @@ -869,22 +1001,33 @@ type ResizeJob struct { } // NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(addURI URI, existingURIs []URI) *ResizeJob { +func NewResizeJob(existingURIs []URI, uri URI, action string) *ResizeJob { // Build a map of uris to track their resize status. - uris := make(map[URI]bool) - // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. - for _, u := range existingURIs { - uris[u] = false + uris := make(map[URI]bool) + + if action == ResizeJobActionRemove { + for _, u := range existingURIs { + // Exclude the removed node from the map. + if u == uri { + continue + } + uris[u] = false + } + } else if action == ResizeJobActionAdd { + for _, u := range existingURIs { + uris[u] = false + } + // Include the added node in the map for tracking. + uris[uri] = false } - // Include the added node in the map for tracking. - uris[addURI] = false return &ResizeJob{ ID: rand.Int63(), URIs: uris, + action: action, result: make(chan string), } } @@ -1009,6 +1152,15 @@ func (t *Topology) containsURI(uri URI) bool { return false } +func (t *Topology) positionByURI(uri URI) int { + for i, turi := range t.NodeSet { + if turi == uri { + return i + } + } + return -1 +} + // AddURI adds the uri to the topology and returns true if added. func (t *Topology) AddURI(uri URI) bool { t.mu.Lock() @@ -1020,6 +1172,23 @@ func (t *Topology) AddURI(uri URI) bool { return true } +// RemoveURI removes the uri from the topology and returns true if removed. +func (t *Topology) RemoveURI(uri URI) bool { + t.mu.Lock() + defer t.mu.Unlock() + + i := t.positionByURI(uri) + if i < 0 { + return false + } + + copy(t.NodeSet[i:], t.NodeSet[i+1:]) + t.NodeSet[len(t.NodeSet)-1] = URI{} + t.NodeSet = t.NodeSet[:len(t.NodeSet)-1] + + return true +} + // Encode converts t into its internal representation. func (t *Topology) Encode() *internal.Topology { return encodeTopology(t) @@ -1112,52 +1281,94 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { if !c.IsCoordinator() { return nil } + return c.nodeJoin(e.URI) + case NodeLeave: + // Automatic nodeLeave is intentionally not implemented. + case NodeUpdate: + // NodeUpdate is intentionally not implemented. + } - if c.needTopologyAgreement() { - // A host that is not part of the topology can't be added to the STARTING cluster. - if !c.Topology.ContainsURI(e.URI) { - return fmt.Errorf("host is not in topology: %v", e.URI) - } + return nil +} - if err := c.AddNode(e.URI); err != nil { - return err - } - - // If the result of the previous AddNode completed the joining of nodes - // in the topology, then change the state to NORMAL. - if c.haveTopologyAgreement() { - return c.setStateAndBroadcast(ClusterStateNormal) - } - - return nil +func (c *Cluster) nodeJoin(uri URI) error { + if c.needTopologyAgreement() { + // A host that is not part of the topology can't be added to the STARTING cluster. + if !c.Topology.ContainsURI(uri) { + return fmt.Errorf("host is not in topology: %v", uri) } - // Don't do anything else if the cluster already contains the node. - if c.NodeByURI(e.URI) != nil { - return nil + if err := c.AddNode(uri); err != nil { + return err } - // If the index does not yet have data, go ahead and add the node. - if !c.Holder.HasData() { - if err := c.AddNode(e.URI); err != nil { - return err - } + // If the result of the previous AddNode completed the joining of nodes + // in the topology, then change the state to NORMAL. + if c.haveTopologyAgreement() { return c.setStateAndBroadcast(ClusterStateNormal) } - // If the cluster has data, we need to change to RESIZING and - // kick off the resizing process. - if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { + return nil + } + + // Don't do anything else if the cluster already contains the node. + if c.NodeByURI(uri) != nil { + return nil + } + + // If the holder does not yet contain data, go ahead and add the node. + if !c.Holder.HasData() { + if err := c.AddNode(uri); err != nil { return err } - c.joiningURIs <- e.URI - - case NodeLeave: - // TODO: implement this - case NodeUpdate: - // TODO: implement this + return c.setStateAndBroadcast(ClusterStateNormal) } + // If the cluster has data, we need to change to RESIZING and + // kick off the resizing process. + if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { + return err + } + c.joiningLeavingNodes <- nodeAction{uri, ResizeJobActionAdd} + + return nil +} + +// NodeLeave initiates the removal of a node from the cluster. +func (c *Cluster) NodeLeave(uri URI) error { + // Refuse the request if this is not the coordinator. + if !c.IsCoordinator() { + return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.Coordinator) + } + + if c.State != ClusterStateNormal { + return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.State) + } + + return c.nodeLeave(uri) +} + +func (c *Cluster) nodeLeave(uri URI) error { + // Don't do anything else if the cluster doesn't contain the node. + if c.NodeByURI(uri) == nil { + return nil + } + + // If the holder does not yet contain data, go ahead and remove the node. + if !c.Holder.HasData() { + if err := c.RemoveNode(uri); err != nil { + return err + } + return c.setStateAndBroadcast(ClusterStateNormal) + } + + // If the cluster has data then change state to RESIZING and + // kick off the resizing process. + if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { + return err + } + c.joiningLeavingNodes <- nodeAction{uri, ResizeJobActionRemove} + return nil } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 4ef65c958..75eb86f85 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -15,14 +15,16 @@ package pilosa import ( + "io/ioutil" "reflect" "testing" + + "github.com/pilosa/pilosa/internal" ) // Ensure that fragCombos creates the correct fragment mapping. func TestFragCombos(t *testing.T) { - c := NewCluster() uri0, err := NewURIFromAddress("host0") if err != nil { t.Fatal(err) @@ -31,8 +33,10 @@ func TestFragCombos(t *testing.T) { if err != nil { t.Fatal(err) } - c.AddNodeBasicSorted(*uri0) - c.AddNodeBasicSorted(*uri1) + + c := NewCluster() + c.addNodeBasicSorted(*uri0) + c.addNodeBasicSorted(*uri1) tests := []struct { idx string @@ -68,3 +72,226 @@ func TestFragCombos(t *testing.T) { } } + +// newIndexWithTempPath returns a new instance of Index. +func newIndexWithTempPath(name string) *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := NewIndex(path, name) + if err != nil { + panic(err) + } + return index +} + +// Ensure that fragSources creates the correct fragment mapping. +func TestFragSources(t *testing.T) { + + uri0, err := NewURIFromAddress("host0") + if err != nil { + t.Fatal(err) + } + uri1, err := NewURIFromAddress("host1") + if err != nil { + t.Fatal(err) + } + uri2, err := NewURIFromAddress("host2") + if err != nil { + t.Fatal(err) + } + uri3, err := NewURIFromAddress("host3") + if err != nil { + t.Fatal(err) + } + + c1 := NewCluster() + c1.ReplicaN = 1 + c1.addNodeBasicSorted(*uri0) + c1.addNodeBasicSorted(*uri1) + + c2 := NewCluster() + c2.ReplicaN = 1 + c2.addNodeBasicSorted(*uri0) + c2.addNodeBasicSorted(*uri1) + c2.addNodeBasicSorted(*uri2) + + c3 := NewCluster() + c3.ReplicaN = 2 + c3.addNodeBasicSorted(*uri0) + c3.addNodeBasicSorted(*uri1) + + c4 := NewCluster() + c4.ReplicaN = 2 + c4.addNodeBasicSorted(*uri0) + c4.addNodeBasicSorted(*uri1) + c4.addNodeBasicSorted(*uri2) + + c5 := NewCluster() + c5.ReplicaN = 2 + c5.addNodeBasicSorted(*uri0) + c5.addNodeBasicSorted(*uri1) + c5.addNodeBasicSorted(*uri2) + c5.addNodeBasicSorted(*uri3) + + idx := newIndexWithTempPath("i") + frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{}) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 101, nil) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 1300000, nil) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 2600000, nil) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 3900000, nil) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + from *Cluster + to *Cluster + idx *Index + expected map[URI][]*internal.ResizeSource + err string + }{ + { + from: c1, + to: c2, + idx: idx, + expected: map[URI][]*internal.ResizeSource{ + URI{"http", "host0", 10101}: []*internal.ResizeSource{}, + URI{"http", "host1", 10101}: []*internal.ResizeSource{}, + URI{"http", "host2", 10101}: []*internal.ResizeSource{ + {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(0)}, + {&internal.URI{"http", "host1", 10101}, "i", "f", "standard", uint64(2)}, + }, + }, + err: "", + }, + { + from: c4, + to: c3, + idx: idx, + expected: map[URI][]*internal.ResizeSource{ + URI{"http", "host0", 10101}: []*internal.ResizeSource{ + {&internal.URI{"http", "host1", 10101}, "i", "f", "standard", uint64(1)}, + }, + URI{"http", "host1", 10101}: []*internal.ResizeSource{ + {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(0)}, + {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(2)}, + }, + }, + err: "", + }, + { + from: c5, + to: c4, + idx: idx, + expected: map[URI][]*internal.ResizeSource{ + URI{"http", "host0", 10101}: []*internal.ResizeSource{ + {&internal.URI{"http", "host2", 10101}, "i", "f", "standard", uint64(0)}, + {&internal.URI{"http", "host2", 10101}, "i", "f", "standard", uint64(2)}, + }, + URI{"http", "host1", 10101}: []*internal.ResizeSource{ + {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(3)}, + }, + URI{"http", "host2", 10101}: []*internal.ResizeSource{}, + }, + err: "", + }, + { + from: c2, + to: c4, + idx: idx, + expected: nil, + err: "clusters are the same size", + }, + { + from: c1, + to: c5, + idx: idx, + expected: nil, + err: "adding more than one node at a time is not supported", + }, + { + from: c5, + to: c1, + idx: idx, + expected: nil, + err: "removing more than one node at a time is not supported", + }, + } + for _, test := range tests { + + actual, err := (test.from).fragSources(test.to, test.idx) + if test.err != "" { + if err.Error() != test.err { + t.Fatalf("expected error: %s", test.err) + } + } else { + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("expected: %v, but got: %v", test.expected, actual) + } + } + } +} + +// Ensure that fragSources creates the correct fragment mapping. +func TestResizeJob(t *testing.T) { + + uri0, err := NewURIFromAddress("host0") + if err != nil { + t.Fatal(err) + } + uri1, err := NewURIFromAddress("host1") + if err != nil { + t.Fatal(err) + } + uri2, err := NewURIFromAddress("host2") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + existingURIs []URI + uri URI + action string + expectedURIs map[URI]bool + }{ + { + existingURIs: []URI{*uri0, *uri1}, + uri: *uri2, + action: ResizeJobActionAdd, + expectedURIs: map[URI]bool{*uri0: false, *uri1: false, *uri2: false}, + }, + { + existingURIs: []URI{*uri0, *uri1, *uri2}, + uri: *uri2, + action: ResizeJobActionRemove, + expectedURIs: map[URI]bool{*uri0: false, *uri1: false}, + }, + } + for _, test := range tests { + + actual := NewResizeJob(test.existingURIs, test.uri, test.action) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actual.URIs, test.expectedURIs) { + t.Errorf("expected: %v, but got: %v", test.expectedURIs, actual.URIs) + } + } +} diff --git a/cluster_test.go b/cluster_test.go index 0c26b96f2..ca60bf15d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -23,7 +23,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" ) @@ -231,80 +230,6 @@ func TestCluster_Topology(t *testing.T) { }) } -// Ensure DataDiff can generate the expected source map. -func TestCluster_Resize(t *testing.T) { - // Given two clusters, ensure DataDiff can determine the sources of data - // needed in order to respond to queries. - t.Run("DataDiff", func(t *testing.T) { - - // Holder - h1 := test.NewHolder() - i, err := h1.CreateIndex("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - f, err := i.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) - if err != nil { - t.Fatal(err) - } - _, err = f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } - _, err = f.CreateViewIfNotExists("inverse") - if err != nil { - t.Fatal(err) - } - - // Set max slices. - i.SetRemoteMaxSlice(2) - i.SetRemoteMaxInverseSlice(5) - - // Cluster 1 - c1 := test.NewCluster(3) - c1.ReplicaN = 2 - - // Cluster 2 - c2 := test.NewCluster(4) - c2.ReplicaN = 2 - - u0 := test.NewURIFromHostPort("host0", 0) - u1 := test.NewURIFromHostPort("host1", 0) - u2 := test.NewURIFromHostPort("host2", 0) - u3 := test.NewURIFromHostPort("host3", 0) - - uri0 := u0.Encode() - uri1 := u1.Encode() - uri2 := u2.Encode() - //uri3 := u3.Encode() - - expected := map[pilosa.URI][]*internal.ResizeSource{ - u0: []*internal.ResizeSource{ - {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 5}, - }, - u1: []*internal.ResizeSource{ - {URI: uri2, Index: "i", Frame: "f", View: "v", Slice: 0}, - {URI: uri2, Index: "i", Frame: "f", View: "inverse", Slice: 0}, - }, - u2: []*internal.ResizeSource{ - {URI: uri0, Index: "i", Frame: "f", View: "inverse", Slice: 3}, - }, - u3: []*internal.ResizeSource{ - {URI: uri0, Index: "i", Frame: "f", View: "v", Slice: 1}, - {URI: uri1, Index: "i", Frame: "f", View: "v", Slice: 2}, - {URI: uri0, Index: "i", Frame: "f", View: "inverse", Slice: 1}, - {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 2}, - {URI: uri1, Index: "i", Frame: "f", View: "inverse", Slice: 5}, - }, - } - - actual := c1.DataDiff(c2, i) - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) -} - // Ensure that general cluster functionality works as expected. func TestCluster_ResizeStates(t *testing.T) { diff --git a/gossip/gossip.go b/gossip/gossip.go index f1599bf2d..2c5c85faf 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -148,7 +148,7 @@ func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSe g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) g.config.memberlistConfig.AdvertisePort = gossipPort - g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. // TODO travis: change this from 0 + //g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. // TODO travis: change this from 0 g.config.memberlistConfig.Delegate = g g.config.memberlistConfig.SecretKey = secretKey g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) diff --git a/handler.go b/handler.go index e0177ba35..943cd13e2 100644 --- a/handler.go +++ b/handler.go @@ -141,6 +141,7 @@ func loadRestricted(router *mux.Router, handler *Handler) { func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET") @@ -1987,6 +1988,57 @@ type setCoordinatorResponse struct { New *URI `json:"new"` } +// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. +func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req removeNodeRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var removeURI *URI + if err := func() error { + removeURI, err = NewURIFromAddress(req.Address) + if err != nil { + return fmt.Errorf("problem with remove node address: %s", err) + } + + // TODO: make sure the address is in the cluster + + // TODO: prevent removing the coordinator node + + // Start the resize process (similar to NodeJoin) + // TODO: this currently blocks. we should leverage listenForJoins() in cluster + // by converted it to a channel of nodeAction {URI, action}. + err := h.Cluster.NodeLeave(*removeURI) + if err != nil { + return err + } + + return nil + }(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(removeNodeResponse{ + Remove: removeURI, + }); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type removeNodeRequest struct { + Address string `json:"address"` +} + +type removeNodeResponse struct { + Remove *URI `json:"remove"` +} + // handlePostClusterResizeAbort handles POST /cluster/resize/abort request. func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { var msg string diff --git a/index.go b/index.go index 95b09577d..22e6ab062 100644 --- a/index.go +++ b/index.go @@ -780,7 +780,6 @@ func (i *Index) openInputDefinitions() error { return nil } } - } return nil } diff --git a/server.go b/server.go index 55fd62249..c5965912a 100644 --- a/server.go +++ b/server.go @@ -411,6 +411,23 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return nil } + // If this node is still STARTING, don't apply remote status. + // There is an issue where starting up a cluster with existing + // data will error on `flock: resource temporarily unavailable`. + // This is because the ApplySchema creates/opens indexes before + // Holder.Open() has run. When Holder.Open() runs later, the + // fragment files are locked. + // TODO: There is still a race condition where the coordinator + // changes state to NORMAL, broadcasts that to the remote node, + // the remote node receives a `NodeStatus` (with schema) before + // running `Holder.Open()`. In that case, state would be NORMAL, + // meaning this check wouldn't pass, and `Holder.Open()` still + // hasn't run. We may need to track whether `Holder.Open()` has + // run, and use that to determine if we bail here. + if s.Cluster.State == ClusterStateStarting { + return nil + } + // Sync schema. if err := s.Holder.ApplySchema(ns.Schema); err != nil { return err From a13db570ccacea63af33444d7b4ff9e462b47eb7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 21 Nov 2017 16:23:20 -0600 Subject: [PATCH 033/311] WIP: Wait for nodeState on Holder.Open(). Implement prefect in Cluster so a node can start http listener in a restricted mode. Adjust tests; particularly start test.Holder in state Normal. TODO: - [ ] fix test TestMain_SendReceiveMessage - [ ] add additional tests for `nodeState` --- broadcast.go | 5 + cluster.go | 78 +++++++- cluster_test.go | 20 +- handler.go | 4 +- internal/private.pb.go | 413 ++++++++++++++++++++++++++++++----------- internal/private.proto | 5 + security_manager.go | 8 +- server.go | 52 +++--- test/handler.go | 2 + 9 files changed, 427 insertions(+), 160 deletions(-) diff --git a/broadcast.go b/broadcast.go index 4a5b8e8f7..ec0e53c8d 100644 --- a/broadcast.go +++ b/broadcast.go @@ -126,6 +126,7 @@ const ( MessageTypeResizeInstruction = 10 MessageTypeResizeInstructionComplete = 11 MessageTypeSetCoordinator = 12 + MessageTypeNodeState = 13 ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -156,6 +157,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeResizeInstructionComplete case *internal.SetCoordinatorMessage: typ = MessageTypeSetCoordinator + case *internal.NodeStateMessage: + typ = MessageTypeNodeState default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -196,6 +199,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.ResizeInstructionComplete{} case MessageTypeSetCoordinator: m = &internal.SetCoordinatorMessage{} + case MessageTypeNodeState: + m = &internal.NodeStateMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/cluster.go b/cluster.go index befc66c1a..171615b4d 100644 --- a/cluster.go +++ b/cluster.go @@ -46,6 +46,10 @@ const ( ClusterStateNormal = "NORMAL" ClusterStateResizing = "RESIZING" + // NodeState represents the state of a node during startup. + NodeStateLoading = "LOADING" + NodeStateReady = "READY" + // ResizeJob states. ResizeJobStateRunning = "RUNNING" // Final states. @@ -285,6 +289,41 @@ func (c *Cluster) setState(state string) { c.State = state } +func (c *Cluster) setNodeState(state string) { + if c.IsCoordinator() { + c.Topology.nodeStates[c.URI] = state + return + } + + // Send node state to coordinator. + ns := &internal.NodeStateMessage{ + URI: c.URI.Encode(), + State: state, + } + + node := &Node{ + URI: c.Coordinator, + } + if err := c.Broadcaster.SendTo(node, ns); err != nil { + c.logger().Printf("sending node state error: err=%s", err) + } +} + +func (c *Cluster) ReceiveNodeState(uri URI, state string) error { + if !c.IsCoordinator() { + return nil + } + + c.Topology.nodeStates[uri] = state + + // Set cluster state to NORMAL. + if c.haveTopologyAgreement() && c.allNodesReady() { + return c.setStateAndBroadcast(ClusterStateNormal) + } + + return nil +} + // localNode is not being used. //func (c *Cluster) localNode() *Node { // return c.NodeByURI(c.URI) @@ -693,6 +732,15 @@ func (c *Cluster) haveTopologyAgreement() bool { return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } +func (c *Cluster) allNodesReady() bool { + for _, uri := range c.Topology.NodeSet { + if c.Topology.nodeStates[uri] != NodeStateReady { + return false + } + } + return true +} + func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { j, err := c.generateResizeJob(nodeAction) if err != nil { @@ -1130,10 +1178,16 @@ func (u NodeSet) ToStrings() []string { type Topology struct { mu sync.RWMutex NodeSet []URI + + // nodeStates holds the state of each node according to + // the coordinator. Used during startup and data load. + nodeStates map[URI]string } func NewTopology() *Topology { - return &Topology{} + return &Topology{ + nodeStates: make(map[URI]string), + } } // ContainsURI returns true if uri matches one of the topology's uris. @@ -1241,9 +1295,9 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { return nil, nil } - t := &Topology{ - NodeSet: decodeURIs(topology.NodeSet), - } + t := NewTopology() + t.NodeSet = decodeURIs(topology.NodeSet) + return t, nil } @@ -1302,9 +1356,19 @@ func (c *Cluster) nodeJoin(uri URI) error { return err } - // If the result of the previous AddNode completed the joining of nodes - // in the topology, then change the state to NORMAL. - if c.haveTopologyAgreement() { + // Only change to normal if there is no existing data. Otherwise, + // the coordinator needs to wait to receive READY messages (nodeStates) + // from remote nodes before setting the cluster to state NORMAL. + if !c.Holder.HasData() { + // If the result of the previous AddNode completed the joining of nodes + // in the topology, then change the state to NORMAL. + if c.haveTopologyAgreement() { + return c.setStateAndBroadcast(ClusterStateNormal) + } + return nil + } + + if c.haveTopologyAgreement() && c.allNodesReady() { return c.setStateAndBroadcast(ClusterStateNormal) } diff --git a/cluster_test.go b/cluster_test.go index ca60bf15d..b13638bf4 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -253,8 +253,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Verify topology file. - if !reflect.DeepEqual(node.Topology, expectedTop) { - t.Errorf("expected topology: %v, but got: %v", expectedTop, node.Topology) + if !reflect.DeepEqual(node.Topology.NodeSet, expectedTop.NodeSet) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeSet, node.Topology.NodeSet) } // Close TestCluster. @@ -344,10 +344,10 @@ func TestCluster_ResizeStates(t *testing.T) { } // Verify topology file. - if !reflect.DeepEqual(node0.Topology, expectedTop) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop, node0.Topology) - } else if !reflect.DeepEqual(node1.Topology, expectedTop) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop, node1.Topology) + if !reflect.DeepEqual(node0.Topology.NodeSet, expectedTop.NodeSet) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeSet, node0.Topology.NodeSet) + } else if !reflect.DeepEqual(node1.Topology.NodeSet, expectedTop.NodeSet) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeSet, node1.Topology.NodeSet) } // Close TestCluster. @@ -436,10 +436,10 @@ func TestCluster_ResizeStates(t *testing.T) { } // Verify topology file. - if !reflect.DeepEqual(node0.Topology, expectedTop) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop, node0.Topology) - } else if !reflect.DeepEqual(node1.Topology, expectedTop) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop, node1.Topology) + if !reflect.DeepEqual(node0.Topology.NodeSet, expectedTop.NodeSet) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeSet, node0.Topology.NodeSet) + } else if !reflect.DeepEqual(node1.Topology.NodeSet, expectedTop.NodeSet) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeSet, node1.Topology.NodeSet) } // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. diff --git a/handler.go b/handler.go index 943cd13e2..1ef7be7f9 100644 --- a/handler.go +++ b/handler.go @@ -109,7 +109,7 @@ func BuildRouters(handler *Handler) { loadRestricted(router, handler) handler.RestrictedRouter = router - handler.SetNormal() + handler.SetRestricted() } // SetNormal is a method of the SecurityManager interface which provides normal URI routing. @@ -184,7 +184,7 @@ func loadNormal(router *mux.Router, handler *Handler) { } func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { - http.Error(w, "not allowed during resize", http.StatusMethodNotAllowed) + http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.Cluster.State), http.StatusMethodNotAllowed) } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { diff --git a/internal/private.pb.go b/internal/private.pb.go index 172bc0be2..0b43b5cc9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -29,6 +29,7 @@ CreateInputDefinitionMessage DeleteInputDefinitionMessage URI + NodeStateMessage NodeStatus ClusterStatus Field @@ -670,6 +671,30 @@ func (m *URI) GetPort() uint32 { return 0 } +type NodeStateMessage struct { + URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` +} + +func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } +func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } +func (*NodeStateMessage) ProtoMessage() {} +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } + +func (m *NodeStateMessage) GetURI() *URI { + if m != nil { + return m.URI + } + return nil +} + +func (m *NodeStateMessage) GetState() string { + if m != nil { + return m.State + } + return "" +} + type NodeStatus struct { URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` @@ -679,7 +704,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *NodeStatus) GetURI() *URI { if m != nil { @@ -710,7 +735,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *ClusterStatus) GetState() string { if m != nil { @@ -736,7 +761,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *Field) GetName() string { if m != nil { @@ -775,7 +800,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -809,7 +834,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -857,7 +882,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *ResizeSource) GetURI() *URI { if m != nil { @@ -904,7 +929,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{27} + return fileDescriptorPrivate, []int{28} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -936,7 +961,7 @@ type SetCoordinatorMessage struct { func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } func (m *SetCoordinatorMessage) GetOld() *URI { if m != nil { @@ -959,7 +984,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func (m *Topology) GetNodeSet() []*URI { if m != nil { @@ -990,6 +1015,7 @@ func init() { proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") proto.RegisterType((*URI)(nil), "internal.URI") + proto.RegisterType((*NodeStateMessage)(nil), "internal.NodeStateMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") proto.RegisterType((*Field)(nil), "internal.Field") @@ -1848,6 +1874,40 @@ func (m *URI) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.URI != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n11, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n11 + } + if len(m.State) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } + return i, nil +} + func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1867,32 +1927,32 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n11, err := m.URI.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n11 - } - if m.MaxSlices != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n12, err := m.MaxSlices.MarshalTo(dAtA[i:]) + n12, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n12 } - if m.Schema != nil { - dAtA[i] = 0x1a + if m.MaxSlices != nil { + dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n13, err := m.Schema.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) + n13, err := m.MaxSlices.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n13 } + if m.Schema != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n14, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n14 + } return i, nil } @@ -2032,21 +2092,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n14, err := m.URI.MarshalTo(dAtA[i:]) + n15, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n15 } if m.Coordinator != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n15, err := m.Coordinator.MarshalTo(dAtA[i:]) + n16, err := m.Coordinator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n16 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -2064,11 +2124,11 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n16, err := m.Schema.MarshalTo(dAtA[i:]) + n17, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n17 } return i, nil } @@ -2092,11 +2152,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n17, err := m.URI.MarshalTo(dAtA[i:]) + n18, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n18 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2148,11 +2208,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n18, err := m.URI.MarshalTo(dAtA[i:]) + n19, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n19 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2182,21 +2242,21 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Old.Size())) - n19, err := m.Old.MarshalTo(dAtA[i:]) + n20, err := m.Old.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n20 } if m.New != nil { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n20, err := m.New.MarshalTo(dAtA[i:]) + n21, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n21 } return i, nil } @@ -2613,6 +2673,20 @@ func (m *URI) Size() (n int) { return n } +func (m *NodeStateMessage) Size() (n int) { + var l int + _ = l + if m.URI != nil { + l = m.URI.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *NodeStatus) Size() (n int) { var l int _ = l @@ -5705,6 +5779,118 @@ func (m *URI) Unmarshal(dAtA []byte) error { } return nil } +func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeStateMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeStateMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.URI == nil { + m.URI = &URI{} + } + if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7071,79 +7257,80 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1179 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4f, 0x6f, 0xe3, 0x44, - 0x14, 0xc7, 0x71, 0x92, 0x26, 0x2f, 0x9b, 0xdd, 0xec, 0xd0, 0xad, 0xd2, 0xaa, 0x4a, 0xc3, 0x1c, - 0x68, 0x59, 0x89, 0x02, 0xad, 0x84, 0xa0, 0x08, 0x09, 0xb6, 0xc9, 0x6a, 0x0d, 0xb4, 0x5d, 0x26, - 0xdd, 0x45, 0x70, 0x40, 0x9a, 0x26, 0x43, 0x6b, 0xd5, 0xb1, 0x83, 0x3d, 0x69, 0x9b, 0x3d, 0x70, - 0x83, 0x03, 0x7c, 0x01, 0xee, 0x7c, 0x19, 0x8e, 0xdc, 0xb8, 0xa2, 0xf2, 0x21, 0x90, 0xb8, 0x80, - 0xe6, 0x79, 0xc6, 0x76, 0xfe, 0x35, 0x6a, 0x6f, 0x7e, 0x6f, 0xde, 0x7b, 0xf3, 0x9b, 0xdf, 0xfb, - 0x33, 0x63, 0xa8, 0x0e, 0x42, 0xf7, 0x82, 0x4b, 0xb1, 0x3d, 0x08, 0x03, 0x19, 0x90, 0x92, 0xeb, - 0x4b, 0x11, 0xfa, 0xdc, 0xa3, 0x47, 0x50, 0x76, 0xfc, 0x9e, 0xb8, 0x3a, 0x10, 0x92, 0x93, 0x26, - 0x54, 0xf6, 0x03, 0x6f, 0xd8, 0xf7, 0xbf, 0xe0, 0x27, 0xc2, 0xab, 0x5b, 0x4d, 0x6b, 0xab, 0xcc, - 0xb2, 0x2a, 0x65, 0x71, 0xec, 0xf6, 0xc5, 0x97, 0x43, 0xee, 0xcb, 0x61, 0xbf, 0x9e, 0x8b, 0x2d, - 0x32, 0x2a, 0xfa, 0xaf, 0x05, 0xe5, 0xa7, 0x21, 0xef, 0x0b, 0x8c, 0xb8, 0x06, 0x25, 0x16, 0x5c, - 0x66, 0xc3, 0x25, 0x32, 0x79, 0x13, 0xee, 0x3b, 0xfe, 0x85, 0x08, 0x23, 0xd1, 0xf6, 0xf9, 0x89, - 0x27, 0x7a, 0x18, 0xae, 0xc4, 0x26, 0xb4, 0x64, 0x1d, 0xca, 0xfb, 0xbc, 0x7b, 0x26, 0x8e, 0x47, - 0x03, 0x51, 0xb7, 0x31, 0x48, 0xaa, 0x48, 0x56, 0x3b, 0xee, 0x2b, 0x51, 0xcf, 0x37, 0xad, 0xad, - 0x2a, 0x4b, 0x15, 0x93, 0x78, 0x0b, 0x53, 0x78, 0x09, 0x85, 0x7b, 0x8c, 0xfb, 0xa7, 0x09, 0x86, - 0x22, 0x62, 0x18, 0xd3, 0x91, 0x4d, 0x28, 0x3e, 0x75, 0x85, 0xd7, 0x8b, 0xea, 0x4b, 0x4d, 0x7b, - 0xab, 0xb2, 0xf3, 0x60, 0xdb, 0xf0, 0xb7, 0x8d, 0x7a, 0xa6, 0x97, 0x29, 0x85, 0xfb, 0x4e, 0x7f, - 0x10, 0x84, 0x92, 0x89, 0x68, 0x10, 0xf8, 0x91, 0x20, 0x35, 0xb0, 0xdb, 0x61, 0xa8, 0xcf, 0xae, - 0x3e, 0xe9, 0x0f, 0x50, 0x7b, 0xe2, 0x05, 0xdd, 0xf3, 0x16, 0x97, 0x9c, 0x89, 0xef, 0x87, 0x22, - 0x92, 0x64, 0x19, 0x0a, 0x98, 0x05, 0x6d, 0x17, 0x0b, 0x4a, 0x8b, 0x4c, 0x6a, 0x9a, 0x63, 0x41, - 0x69, 0xd1, 0x1f, 0xa9, 0xc8, 0xb3, 0x58, 0x50, 0xda, 0x8e, 0xe7, 0x76, 0x63, 0x0a, 0xf2, 0x2c, - 0x16, 0x08, 0x81, 0xfc, 0x4b, 0x57, 0x5c, 0xea, 0x73, 0xe3, 0x37, 0x75, 0xe0, 0x61, 0x66, 0x7f, - 0x0d, 0x73, 0x05, 0x8a, 0x2c, 0xb8, 0x74, 0x5a, 0x51, 0xdd, 0x6a, 0xda, 0x5b, 0x79, 0xa6, 0x25, - 0x64, 0x17, 0xd3, 0xaf, 0x96, 0x72, 0xb8, 0x94, 0x2a, 0xe8, 0x2a, 0x14, 0x90, 0x6a, 0x75, 0xca, - 0xd4, 0x57, 0x7d, 0xd2, 0xff, 0x2c, 0x28, 0x1f, 0xf0, 0x2b, 0x84, 0x11, 0x91, 0x8f, 0xa1, 0xd4, - 0x91, 0xdc, 0xef, 0xf1, 0xb0, 0x87, 0x46, 0x95, 0x9d, 0x37, 0x52, 0x0a, 0x13, 0xb3, 0x6d, 0x63, - 0xd3, 0xf6, 0x65, 0x38, 0x62, 0x89, 0x0b, 0xd9, 0x83, 0x25, 0x5d, 0x13, 0x88, 0xa1, 0xb2, 0xd3, - 0x9c, 0xe5, 0x9d, 0x94, 0x8d, 0x72, 0x36, 0x0e, 0x6b, 0x1f, 0x41, 0x75, 0x2c, 0xac, 0xc2, 0x7a, - 0x2e, 0x46, 0x26, 0x23, 0xe7, 0x62, 0xa4, 0xb8, 0xbb, 0xe0, 0xde, 0x30, 0xe6, 0x39, 0xcf, 0x62, - 0x61, 0x2f, 0xf7, 0x81, 0xb5, 0xb6, 0x07, 0xf7, 0xb2, 0x51, 0x6f, 0xe3, 0x4b, 0xbf, 0x05, 0xb2, - 0x1f, 0x0a, 0x2e, 0x05, 0xc2, 0x3b, 0x10, 0x51, 0xc4, 0x4f, 0xc5, 0xfc, 0x4c, 0xc7, 0xd9, 0xcb, - 0x65, 0xb3, 0xb7, 0x0e, 0x65, 0x27, 0x32, 0x07, 0xb7, 0xb1, 0x2e, 0x53, 0x05, 0x7d, 0x0c, 0xa4, - 0x25, 0x3c, 0x21, 0x85, 0xee, 0xdf, 0x1b, 0xe2, 0xd3, 0x8e, 0xc1, 0xb2, 0xd8, 0x96, 0x6c, 0x42, - 0x5e, 0xb5, 0x2e, 0x42, 0xa9, 0xec, 0xbc, 0x9e, 0x32, 0x9d, 0xcc, 0x09, 0x86, 0x06, 0xd4, 0x35, - 0x41, 0x75, 0xbb, 0x2f, 0x38, 0xe0, 0x8c, 0x52, 0x36, 0x5b, 0xd9, 0x93, 0x5b, 0x25, 0x03, 0x44, - 0x6f, 0xf5, 0x89, 0x39, 0xeb, 0x5d, 0xb7, 0xa2, 0xdf, 0x68, 0xad, 0x6a, 0x89, 0x43, 0xb5, 0x1a, - 0xfb, 0xe0, 0xf7, 0xfc, 0x23, 0x4f, 0xe0, 0x50, 0xb1, 0x55, 0x0f, 0x45, 0x75, 0xbb, 0x69, 0xab, - 0xd8, 0x28, 0xd0, 0x5d, 0x28, 0x76, 0xba, 0x67, 0xa2, 0xcf, 0xc9, 0x5b, 0xaa, 0x50, 0x7b, 0xe2, - 0x4a, 0x44, 0xba, 0xcc, 0x1f, 0x4c, 0xd0, 0xc7, 0xcc, 0x3a, 0xfd, 0xc5, 0xd2, 0xe8, 0xe7, 0x20, - 0x2a, 0xe2, 0xde, 0x51, 0x3d, 0x3f, 0x35, 0x71, 0x94, 0x9e, 0xe9, 0x65, 0xd2, 0x86, 0x9a, 0xe3, - 0x0f, 0x86, 0xb2, 0x25, 0xbe, 0x73, 0x7d, 0x57, 0xba, 0x81, 0x1f, 0xd5, 0x8b, 0xe8, 0xb2, 0x9a, - 0xdd, 0x7a, 0xcc, 0x82, 0x4d, 0xb9, 0xd0, 0x9f, 0x2c, 0x78, 0x30, 0xa1, 0x5c, 0x80, 0x2b, 0x77, - 0x33, 0xae, 0xf7, 0x93, 0x91, 0x69, 0xa3, 0x61, 0x63, 0x2e, 0x9a, 0xf1, 0x09, 0xfa, 0x9b, 0x05, - 0xcb, 0xb3, 0x0c, 0x66, 0xa2, 0x69, 0x00, 0x3c, 0x0f, 0xdd, 0x3e, 0x0f, 0x47, 0x9f, 0x8b, 0x91, - 0xbe, 0x3d, 0x32, 0x1a, 0xf2, 0x15, 0xac, 0x4c, 0xc4, 0xfa, 0xb4, 0x1b, 0x53, 0x14, 0x83, 0xda, - 0x98, 0x0b, 0x2a, 0xb6, 0x63, 0x73, 0xdc, 0xe9, 0x3f, 0x16, 0x3c, 0x9a, 0xb9, 0x94, 0x56, 0x9f, - 0x95, 0x2d, 0xf4, 0xc7, 0x50, 0x7b, 0xa9, 0x06, 0x43, 0x4b, 0x44, 0xd2, 0xf5, 0xb9, 0xb2, 0xd4, - 0xe5, 0x39, 0xa5, 0x27, 0x0e, 0x94, 0x50, 0x77, 0xc0, 0x07, 0x1a, 0xe6, 0xdb, 0x0b, 0x60, 0x6e, - 0x1b, 0x7b, 0x3d, 0x37, 0x8d, 0xa8, 0xc0, 0xe0, 0x1c, 0x37, 0x97, 0x02, 0x0a, 0x6a, 0x22, 0x8e, - 0x39, 0xdc, 0x6a, 0xaa, 0x05, 0xb0, 0x6e, 0x26, 0xc9, 0x18, 0x92, 0x9b, 0x7b, 0xf2, 0x43, 0x80, - 0xd4, 0x54, 0xb7, 0xfb, 0x0d, 0xf5, 0x99, 0x31, 0xa6, 0xcf, 0x60, 0xdd, 0x8c, 0xb9, 0x5b, 0x6c, - 0x68, 0xaa, 0x25, 0x97, 0x56, 0x0b, 0x6d, 0x83, 0xfd, 0x82, 0x39, 0xea, 0xaa, 0xc3, 0x6e, 0x35, - 0x29, 0xd2, 0x92, 0x72, 0x79, 0x16, 0x44, 0xd2, 0xb8, 0xa8, 0x6f, 0xa5, 0x7b, 0x1e, 0x84, 0x12, - 0x11, 0x57, 0x19, 0x7e, 0xd3, 0x9f, 0x2d, 0x80, 0xc3, 0xa0, 0x27, 0x3a, 0x92, 0xcb, 0x61, 0x44, - 0x36, 0x30, 0x2a, 0xc6, 0xaa, 0xec, 0x54, 0xd3, 0x33, 0xbd, 0x60, 0x0e, 0xc3, 0xfd, 0xde, 0xcb, - 0x5c, 0x84, 0xd3, 0x13, 0x26, 0x59, 0x62, 0x99, 0xeb, 0x72, 0xcb, 0x0c, 0x14, 0x4d, 0x55, 0x2d, - 0xb5, 0x8f, 0xf5, 0x1a, 0x34, 0xa7, 0x87, 0x50, 0xdd, 0xf7, 0x86, 0x91, 0x14, 0xa1, 0x86, 0xa3, - 0x6e, 0x12, 0xc9, 0x65, 0x52, 0x7f, 0x28, 0x90, 0x4d, 0x58, 0x42, 0xc8, 0x42, 0xea, 0xbe, 0x9d, - 0x00, 0x6a, 0x56, 0x69, 0x07, 0x0a, 0xf3, 0xdb, 0x8d, 0x40, 0x1e, 0xdf, 0x60, 0x9a, 0x21, 0x7c, - 0x7e, 0xd5, 0xc0, 0x3e, 0x70, 0xe3, 0x94, 0xda, 0x4c, 0x7d, 0xa2, 0x86, 0x5f, 0x61, 0xc9, 0x29, - 0x0d, 0x57, 0xb7, 0xcf, 0xc3, 0x38, 0x85, 0x6a, 0x5c, 0xde, 0xe5, 0x9e, 0x30, 0xcf, 0x18, 0x3b, - 0xf3, 0x8c, 0xf9, 0xd3, 0x82, 0x87, 0x4c, 0x44, 0xee, 0x2b, 0xe1, 0xf8, 0x91, 0x0c, 0x87, 0x49, - 0xfb, 0x7d, 0x16, 0x9c, 0x38, 0x2d, 0x8c, 0x6a, 0xb3, 0x58, 0x30, 0x39, 0xca, 0xcd, 0xcd, 0xd1, - 0x3b, 0xea, 0xe1, 0x1b, 0x84, 0x3d, 0xd5, 0x83, 0x41, 0xa8, 0x59, 0x9f, 0x30, 0xcc, 0x5a, 0x90, - 0x77, 0x61, 0xa9, 0x13, 0x0c, 0xc3, 0x6e, 0x32, 0xa0, 0x57, 0x52, 0xe3, 0x18, 0x55, 0xbc, 0xcc, - 0x8c, 0x59, 0x26, 0xa7, 0x85, 0x05, 0x39, 0xfd, 0xd1, 0x82, 0x7b, 0xd9, 0x18, 0x8b, 0x4b, 0x2c, - 0xe1, 0x32, 0x37, 0x93, 0x4b, 0x7b, 0x16, 0x97, 0xf9, 0x94, 0xcb, 0xf4, 0xf9, 0x51, 0xc8, 0x3c, - 0x3f, 0xe8, 0x19, 0xac, 0x4e, 0x11, 0xbc, 0x1f, 0xf4, 0x07, 0x2a, 0x93, 0x77, 0x25, 0x7a, 0x19, - 0x0a, 0xed, 0x30, 0xd4, 0x14, 0x97, 0x59, 0x2c, 0xd0, 0xaf, 0xe1, 0x51, 0x47, 0xc8, 0x0c, 0xbf, - 0xa6, 0x48, 0x36, 0xc0, 0x3e, 0xf2, 0x7a, 0x73, 0x4e, 0x7e, 0xe4, 0xf5, 0x94, 0xc1, 0xa1, 0xb8, - 0x9c, 0xb3, 0xe1, 0xa1, 0xb8, 0xa4, 0xbb, 0x50, 0x3a, 0x0e, 0x06, 0x81, 0x17, 0x9c, 0x8e, 0xb2, - 0x5d, 0x60, 0xdd, 0xd4, 0x05, 0x4f, 0x6a, 0xbf, 0x5f, 0x37, 0xac, 0x3f, 0xae, 0x1b, 0xd6, 0x5f, - 0xd7, 0x0d, 0xeb, 0xd7, 0xbf, 0x1b, 0xaf, 0x9d, 0x14, 0xf1, 0xbf, 0x69, 0xf7, 0xff, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x80, 0x0a, 0xbc, 0x80, 0x48, 0x0d, 0x00, 0x00, + // 1192 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0x23, 0x45, + 0x13, 0x7e, 0xc7, 0x63, 0x3b, 0x76, 0x79, 0xbd, 0xeb, 0xf4, 0x9b, 0x8d, 0x9c, 0x28, 0xf2, 0x9a, + 0x3e, 0x90, 0xb0, 0x12, 0x01, 0x12, 0x09, 0x41, 0x10, 0x12, 0x6c, 0xec, 0xd5, 0x0e, 0x90, 0x64, + 0x69, 0x67, 0x17, 0xc1, 0x01, 0xa9, 0x63, 0x37, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xd3, 0x4e, 0xe2, + 0x3d, 0x70, 0x83, 0x03, 0xfc, 0x01, 0xee, 0xfc, 0x19, 0x8e, 0xdc, 0xb8, 0xa2, 0xf0, 0x23, 0x90, + 0xb8, 0x80, 0xba, 0xa6, 0x7b, 0x66, 0xfc, 0x15, 0x93, 0xdc, 0xa6, 0xaa, 0xab, 0xaa, 0x9f, 0x7e, + 0xea, 0xa3, 0x7b, 0xa0, 0x3a, 0x08, 0xdd, 0x0b, 0x2e, 0xc5, 0xf6, 0x20, 0x0c, 0x64, 0x40, 0x4a, + 0xae, 0x2f, 0x45, 0xe8, 0x73, 0x8f, 0x1e, 0x41, 0xd9, 0xf1, 0x7b, 0xe2, 0xea, 0x40, 0x48, 0x4e, + 0x9a, 0x50, 0xd9, 0x0f, 0xbc, 0x61, 0xdf, 0xff, 0x8c, 0x9f, 0x08, 0xaf, 0x6e, 0x35, 0xad, 0xad, + 0x32, 0xcb, 0xaa, 0x94, 0xc5, 0xb1, 0xdb, 0x17, 0x9f, 0x0f, 0xb9, 0x2f, 0x87, 0xfd, 0x7a, 0x2e, + 0xb6, 0xc8, 0xa8, 0xe8, 0xdf, 0x16, 0x94, 0x9f, 0x86, 0xbc, 0x2f, 0x30, 0xe2, 0x3a, 0x94, 0x58, + 0x70, 0x99, 0x0d, 0x97, 0xc8, 0xe4, 0x75, 0xb8, 0xef, 0xf8, 0x17, 0x22, 0x8c, 0x44, 0xdb, 0xe7, + 0x27, 0x9e, 0xe8, 0x61, 0xb8, 0x12, 0x9b, 0xd0, 0x92, 0x0d, 0x28, 0xef, 0xf3, 0xee, 0x99, 0x38, + 0x1e, 0x0d, 0x44, 0xdd, 0xc6, 0x20, 0xa9, 0x22, 0x59, 0xed, 0xb8, 0xaf, 0x44, 0x3d, 0xdf, 0xb4, + 0xb6, 0xaa, 0x2c, 0x55, 0x4c, 0xe2, 0x2d, 0x4c, 0xe1, 0x25, 0x14, 0xee, 0x31, 0xee, 0x9f, 0x26, + 0x18, 0x8a, 0x88, 0x61, 0x4c, 0x47, 0x36, 0xa1, 0xf8, 0xd4, 0x15, 0x5e, 0x2f, 0xaa, 0x2f, 0x35, + 0xed, 0xad, 0xca, 0xce, 0x83, 0x6d, 0xc3, 0xdf, 0x36, 0xea, 0x99, 0x5e, 0xa6, 0x14, 0xee, 0x3b, + 0xfd, 0x41, 0x10, 0x4a, 0x26, 0xa2, 0x41, 0xe0, 0x47, 0x82, 0xd4, 0xc0, 0x6e, 0x87, 0xa1, 0x3e, + 0xbb, 0xfa, 0xa4, 0xdf, 0x41, 0xed, 0x89, 0x17, 0x74, 0xcf, 0x5b, 0x5c, 0x72, 0x26, 0xbe, 0x1d, + 0x8a, 0x48, 0x92, 0x15, 0x28, 0x60, 0x16, 0xb4, 0x5d, 0x2c, 0x28, 0x2d, 0x32, 0xa9, 0x69, 0x8e, + 0x05, 0xa5, 0x45, 0x7f, 0xa4, 0x22, 0xcf, 0x62, 0x41, 0x69, 0x3b, 0x9e, 0xdb, 0x8d, 0x29, 0xc8, + 0xb3, 0x58, 0x20, 0x04, 0xf2, 0x2f, 0x5d, 0x71, 0xa9, 0xcf, 0x8d, 0xdf, 0xd4, 0x81, 0xe5, 0xcc, + 0xfe, 0x1a, 0xe6, 0x2a, 0x14, 0x59, 0x70, 0xe9, 0xb4, 0xa2, 0xba, 0xd5, 0xb4, 0xb7, 0xf2, 0x4c, + 0x4b, 0xc8, 0x2e, 0xa6, 0x5f, 0x2d, 0xe5, 0x70, 0x29, 0x55, 0xd0, 0x35, 0x28, 0x20, 0xd5, 0xea, + 0x94, 0xa9, 0xaf, 0xfa, 0xa4, 0xff, 0x58, 0x50, 0x3e, 0xe0, 0x57, 0x08, 0x23, 0x22, 0x1f, 0x42, + 0xa9, 0x23, 0xb9, 0xdf, 0xe3, 0x61, 0x0f, 0x8d, 0x2a, 0x3b, 0xaf, 0xa5, 0x14, 0x26, 0x66, 0xdb, + 0xc6, 0xa6, 0xed, 0xcb, 0x70, 0xc4, 0x12, 0x17, 0xb2, 0x07, 0x4b, 0xba, 0x26, 0x10, 0x43, 0x65, + 0xa7, 0x39, 0xcb, 0x3b, 0x29, 0x1b, 0xe5, 0x6c, 0x1c, 0xd6, 0x3f, 0x80, 0xea, 0x58, 0x58, 0x85, + 0xf5, 0x5c, 0x8c, 0x4c, 0x46, 0xce, 0xc5, 0x48, 0x71, 0x77, 0xc1, 0xbd, 0x61, 0xcc, 0x73, 0x9e, + 0xc5, 0xc2, 0x5e, 0xee, 0x3d, 0x6b, 0x7d, 0x0f, 0xee, 0x65, 0xa3, 0xde, 0xc6, 0x97, 0x7e, 0x0d, + 0x64, 0x3f, 0x14, 0x5c, 0x0a, 0x84, 0x77, 0x20, 0xa2, 0x88, 0x9f, 0x8a, 0xf9, 0x99, 0x8e, 0xb3, + 0x97, 0xcb, 0x66, 0x6f, 0x03, 0xca, 0x4e, 0x64, 0x0e, 0x6e, 0x63, 0x5d, 0xa6, 0x0a, 0xfa, 0x18, + 0x48, 0x4b, 0x78, 0x42, 0x0a, 0xdd, 0xbf, 0x37, 0xc4, 0xa7, 0x1d, 0x83, 0x65, 0xb1, 0x2d, 0xd9, + 0x84, 0xbc, 0x6a, 0x5d, 0x84, 0x52, 0xd9, 0xf9, 0x7f, 0xca, 0x74, 0x32, 0x27, 0x18, 0x1a, 0x50, + 0xd7, 0x04, 0xd5, 0xed, 0xbe, 0xe0, 0x80, 0x33, 0x4a, 0xd9, 0x6c, 0x65, 0x4f, 0x6e, 0x95, 0x0c, + 0x10, 0xbd, 0xd5, 0x47, 0xe6, 0xac, 0x77, 0xdd, 0x8a, 0x7e, 0xa5, 0xb5, 0xaa, 0x25, 0x0e, 0xd5, + 0x6a, 0xec, 0x83, 0xdf, 0xf3, 0x8f, 0x3c, 0x81, 0x43, 0xc5, 0x56, 0x3d, 0x14, 0xd5, 0xed, 0xa6, + 0xad, 0x62, 0xa3, 0x40, 0x77, 0xa1, 0xd8, 0xe9, 0x9e, 0x89, 0x3e, 0x27, 0x6f, 0xa8, 0x42, 0xed, + 0x89, 0x2b, 0x11, 0xe9, 0x32, 0x7f, 0x30, 0x41, 0x1f, 0x33, 0xeb, 0xf4, 0x27, 0x4b, 0xa3, 0x9f, + 0x83, 0xa8, 0x88, 0x7b, 0x47, 0xf5, 0xfc, 0xd4, 0xc4, 0x51, 0x7a, 0xa6, 0x97, 0x49, 0x1b, 0x6a, + 0x8e, 0x3f, 0x18, 0xca, 0x96, 0xf8, 0xc6, 0xf5, 0x5d, 0xe9, 0x06, 0x7e, 0x54, 0x2f, 0xa2, 0xcb, + 0x5a, 0x76, 0xeb, 0x31, 0x0b, 0x36, 0xe5, 0x42, 0x7f, 0xb0, 0xe0, 0xc1, 0x84, 0x72, 0x01, 0xae, + 0xdc, 0xcd, 0xb8, 0xde, 0x4d, 0x46, 0xa6, 0x8d, 0x86, 0x8d, 0xb9, 0x68, 0xc6, 0x27, 0xe8, 0x2f, + 0x16, 0xac, 0xcc, 0x32, 0x98, 0x89, 0xa6, 0x01, 0xf0, 0x3c, 0x74, 0xfb, 0x3c, 0x1c, 0x7d, 0x2a, + 0x46, 0xfa, 0xf6, 0xc8, 0x68, 0xc8, 0x17, 0xb0, 0x3a, 0x11, 0xeb, 0xe3, 0x6e, 0x4c, 0x51, 0x0c, + 0xea, 0xd1, 0x5c, 0x50, 0xb1, 0x1d, 0x9b, 0xe3, 0x4e, 0xff, 0xb2, 0xe0, 0xe1, 0xcc, 0xa5, 0xb4, + 0xfa, 0xac, 0x6c, 0xa1, 0x3f, 0x86, 0xda, 0x4b, 0x35, 0x18, 0x5a, 0x22, 0x92, 0xae, 0xcf, 0x95, + 0xa5, 0x2e, 0xcf, 0x29, 0x3d, 0x71, 0xa0, 0x84, 0xba, 0x03, 0x3e, 0xd0, 0x30, 0xdf, 0x5c, 0x00, + 0x73, 0xdb, 0xd8, 0xeb, 0xb9, 0x69, 0x44, 0x05, 0x06, 0xe7, 0xb8, 0xb9, 0x14, 0x50, 0x50, 0x13, + 0x71, 0xcc, 0xe1, 0x56, 0x53, 0x2d, 0x80, 0x0d, 0x33, 0x49, 0xc6, 0x90, 0xdc, 0xdc, 0x93, 0xef, + 0x03, 0xa4, 0xa6, 0xba, 0xdd, 0x6f, 0xa8, 0xcf, 0x8c, 0x31, 0x7d, 0x06, 0x1b, 0x66, 0xcc, 0xdd, + 0x62, 0x43, 0x53, 0x2d, 0xb9, 0xb4, 0x5a, 0x68, 0x1b, 0xec, 0x17, 0xcc, 0x51, 0x57, 0x1d, 0x76, + 0xab, 0x49, 0x91, 0x96, 0x94, 0xcb, 0xb3, 0x20, 0x92, 0xc6, 0x45, 0x7d, 0x2b, 0xdd, 0xf3, 0x20, + 0x94, 0x88, 0xb8, 0xca, 0xf0, 0x9b, 0x3a, 0x50, 0x3b, 0x0c, 0x7a, 0xa2, 0x23, 0xb9, 0x4c, 0x26, + 0xd1, 0x23, 0x0c, 0x8d, 0x01, 0x2b, 0x3b, 0xd5, 0xf4, 0x60, 0x2f, 0x98, 0xc3, 0x70, 0x53, 0x35, + 0xe0, 0x95, 0x83, 0x19, 0x4a, 0x28, 0xd0, 0x1f, 0x2d, 0x00, 0x13, 0x6b, 0x18, 0x2d, 0x8e, 0xf2, + 0x4e, 0xe6, 0x4e, 0x9d, 0x1e, 0x56, 0xc9, 0x12, 0xcb, 0xdc, 0xbc, 0x5b, 0x66, 0x36, 0x69, 0xd6, + 0x6b, 0xa9, 0x7d, 0xac, 0xd7, 0xe7, 0xe7, 0xf4, 0x10, 0xaa, 0xfb, 0xde, 0x30, 0x92, 0x22, 0xd4, + 0x70, 0x12, 0xcc, 0x56, 0x06, 0x33, 0xd9, 0x84, 0x25, 0x84, 0x2c, 0xa4, 0x1e, 0x01, 0x13, 0x40, + 0xcd, 0x2a, 0xed, 0x40, 0x61, 0x7e, 0xe7, 0x12, 0xc8, 0xe3, 0x73, 0x4e, 0x93, 0x8d, 0x2f, 0xb9, + 0x1a, 0xd8, 0x07, 0x6e, 0x5c, 0x1d, 0x36, 0x53, 0x9f, 0xa8, 0xe1, 0x57, 0x58, 0xbd, 0x4a, 0xc3, + 0xd5, 0x45, 0xb6, 0x1c, 0x57, 0x83, 0x9a, 0xbc, 0x77, 0xb9, 0x72, 0xcc, 0x8b, 0xc8, 0xce, 0xbc, + 0x88, 0x7e, 0xb7, 0x60, 0x99, 0x89, 0xc8, 0x7d, 0x25, 0x1c, 0x3f, 0x92, 0xe1, 0x30, 0xe9, 0xe4, + 0x4f, 0x82, 0x13, 0xa7, 0x85, 0x51, 0x6d, 0x16, 0x0b, 0x26, 0x47, 0xb9, 0xb9, 0x39, 0x7a, 0x4b, + 0xbd, 0xa1, 0x83, 0xb0, 0xa7, 0xda, 0x39, 0x08, 0x35, 0xeb, 0x13, 0x86, 0x59, 0x0b, 0xf2, 0x36, + 0x2c, 0x75, 0x82, 0x61, 0xd8, 0x4d, 0x66, 0xfd, 0x6a, 0x6a, 0x1c, 0xa3, 0x8a, 0x97, 0x99, 0x31, + 0xcb, 0xe4, 0xb4, 0xb0, 0x20, 0xa7, 0xdf, 0x5b, 0x70, 0x2f, 0x1b, 0xe3, 0x3f, 0x15, 0x6a, 0xcc, + 0x65, 0x6e, 0x26, 0x97, 0xf6, 0x2c, 0x2e, 0xf3, 0x29, 0x97, 0xe9, 0x4b, 0xa6, 0x90, 0x79, 0xc9, + 0xd0, 0x33, 0x58, 0x9b, 0x22, 0x78, 0x3f, 0xe8, 0x0f, 0x54, 0x26, 0xef, 0x4a, 0xf4, 0x0a, 0x14, + 0xda, 0x61, 0xa8, 0x29, 0x2e, 0xb3, 0x58, 0xa0, 0x5f, 0xc2, 0xc3, 0x8e, 0x90, 0x19, 0x7e, 0x33, + 0x2d, 0x7a, 0xe4, 0xf5, 0xe6, 0x9c, 0xfc, 0xc8, 0xeb, 0x29, 0x83, 0x43, 0x71, 0x39, 0x67, 0xc3, + 0x43, 0x71, 0x49, 0x77, 0xa1, 0x74, 0x1c, 0x0c, 0x02, 0x2f, 0x38, 0x1d, 0x65, 0xbb, 0xc0, 0xba, + 0xa9, 0x0b, 0x9e, 0xd4, 0x7e, 0xbd, 0x6e, 0x58, 0xbf, 0x5d, 0x37, 0xac, 0x3f, 0xae, 0x1b, 0xd6, + 0xcf, 0x7f, 0x36, 0xfe, 0x77, 0x52, 0xc4, 0x5f, 0xb0, 0xdd, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, + 0x50, 0x2b, 0xa7, 0xda, 0x93, 0x0d, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index a97ac0e2d..bb1cadfea 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -121,6 +121,11 @@ message URI { uint32 Port = 3; } +message NodeStateMessage { + URI URI = 1; + string State = 2; +} + message NodeStatus { URI URI = 1; MaxSlices MaxSlices = 2; diff --git a/security_manager.go b/security_manager.go index 2acf82986..fc174866a 100644 --- a/security_manager.go +++ b/security_manager.go @@ -12,11 +12,7 @@ type NopSecurityManager struct { } // SetRestricted no-op. -func (sdm *NopSecurityManager) SetRestricted() { - -} +func (sdm *NopSecurityManager) SetRestricted() {} // SetNormal no-op. -func (sdm *NopSecurityManager) SetNormal() { - -} +func (sdm *NopSecurityManager) SetNormal() {} diff --git a/server.go b/server.go index c5965912a..3944fb0bf 100644 --- a/server.go +++ b/server.go @@ -151,28 +151,6 @@ func (s *Server) Open() error { s.Holder.LogOutput = s.LogOutput s.Holder.Peek() - // Start the BroadcastReceiver. - if err := s.BroadcastReceiver.Start(s); err != nil { - return fmt.Errorf("starting BroadcastReceiver: %v", err) - } - - // Open Cluster management. - if err := s.Cluster.Open(); err != nil { - return fmt.Errorf("opening Cluster: %v", err) - } - - // Open holder. - if err := s.Holder.Open(); err != nil { - return fmt.Errorf("opening Holder: %v", err) - } - - // Listen for joining nodes. - // This needs to start after the Holder has opened so that nodes can join - // the cluster without waiting for data to load on the coordinator. Before - // this starts, the joins are queued up in the Cluster.joiningURIs buffered - // channel. - s.Cluster.ListenForJoins() - // Create default HTTP client s.createDefaultClient() @@ -191,6 +169,8 @@ func (s *Server) Open() error { s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput + s.Cluster.prefect = s.Handler + // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster @@ -202,6 +182,29 @@ func (s *Server) Open() error { } }() + // Start the BroadcastReceiver. + if err := s.BroadcastReceiver.Start(s); err != nil { + return fmt.Errorf("starting BroadcastReceiver: %v", err) + } + + // Open Cluster management. + if err := s.Cluster.Open(); err != nil { + return fmt.Errorf("opening Cluster: %v", err) + } + + // Open holder. + if err := s.Holder.Open(); err != nil { + return fmt.Errorf("opening Holder: %v", err) + } + s.Cluster.setNodeState(NodeStateReady) + + // Listen for joining nodes. + // This needs to start after the Holder has opened so that nodes can join + // the cluster without waiting for data to load on the coordinator. Before + // this starts, the joins are queued up in the Cluster.joiningURIs buffered + // channel. + s.Cluster.ListenForJoins() + // Start background monitoring. s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() @@ -358,6 +361,11 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { } case *internal.SetCoordinatorMessage: s.Cluster.SetCoordinator(DecodeURI(obj.Old), DecodeURI(obj.New)) + case *internal.NodeStateMessage: + err := s.Cluster.ReceiveNodeState(DecodeURI(obj.URI), obj.State) + if err != nil { + return err + } } return nil diff --git a/test/handler.go b/test/handler.go index e3bee5072..6c6703f84 100644 --- a/test/handler.go +++ b/test/handler.go @@ -32,6 +32,8 @@ func NewHandler() *Handler { // Handler test messages can no-op. h.Broadcaster = pilosa.NopBroadcaster + h.SetNormal() + return h } From bd511dae8026b4ddfd02880bb403942e55e8fa1b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Nov 2017 13:42:08 -0600 Subject: [PATCH 034/311] ensure that holder opens before node is deemed ready --- cluster.go | 48 ++++++++++++++++++++++++++++-------------------- holder.go | 7 +++++++ server.go | 8 +++++--- test/cluster.go | 9 +++++++-- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/cluster.go b/cluster.go index 171615b4d..7ea161215 100644 --- a/cluster.go +++ b/cluster.go @@ -286,13 +286,13 @@ func (c *Cluster) setState(state string) { // - ClusterStateStarting } + c.logger().Printf("Change cluster state from %s to %s", c.State, state) c.State = state } -func (c *Cluster) setNodeState(state string) { +func (c *Cluster) SetNodeState(state string) error { if c.IsCoordinator() { - c.Topology.nodeStates[c.URI] = state - return + return c.ReceiveNodeState(c.URI, state) } // Send node state to coordinator. @@ -305,8 +305,10 @@ func (c *Cluster) setNodeState(state string) { URI: c.Coordinator, } if err := c.Broadcaster.SendTo(node, ns); err != nil { - c.logger().Printf("sending node state error: err=%s", err) + return fmt.Errorf("sending node state error: err=%s", err) } + + return nil } func (c *Cluster) ReceiveNodeState(uri URI, state string) error { @@ -691,18 +693,15 @@ func (c *Cluster) Open() error { // Only the coordinator needs to consider the .topology file. if c.IsCoordinator() { - state, err := c.considerTopology() + err := c.considerTopology() if err != nil { return fmt.Errorf("considerTopology: %v", err) } - // Add the local node to the cluster and update state. - c.AddNode(c.URI) - c.setState(state) - } else { - // Add the local node to the cluster. - c.AddNode(c.URI) } + // Add the local node to the cluster. + c.AddNode(c.URI) + // Start the EventReceiver. if err := c.EventReceiver.Start(c); err != nil { return fmt.Errorf("starting EventReceiver: %v", err) @@ -921,6 +920,10 @@ func (c *Cluster) CompleteCurrentJob(state string) error { // FollowResizeInstruction is run by any node that receives a ResizeInstruction. func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { go func() { + + // Make sure the holder has opened. + <-c.Holder.opened + // Prepare the return message. complete := &internal.ResizeInstructionComplete{ JobID: instr.JobID, @@ -1273,6 +1276,11 @@ func (c *Cluster) loadTopology() error { // saveTopology writes the current topology to disk. func (c *Cluster) saveTopology() error { + + if err := os.MkdirAll(c.Path, 0777); err != nil { + return err + } + if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { return err } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { @@ -1301,25 +1309,25 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { return t, nil } -func (c *Cluster) considerTopology() (string, error) { - // If there is no .topology file, it's safe to go to state NORMAL. +func (c *Cluster) considerTopology() error { + // If there is no .topology file, it's safe to proceed. if len(c.Topology.NodeSet) == 0 { - return ClusterStateNormal, nil + return nil } // The local node (coordinator) must be in the .topology. if !c.Topology.ContainsURI(c.Coordinator) { - return "", fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.NodeSet) + return fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.NodeSet) } - // If local node is the only thing in .topology, continue to state NORMAL. - if len(c.Topology.NodeSet) == 1 { - return ClusterStateNormal, nil - } + // If local node is the only thing in .topology, continue. + //if len(c.Topology.NodeSet) == 1 { + // return nil + //} // Keep the cluster in state "STARTING" until hearing from all nodes. // Topology contains 2+ hosts. - return ClusterStateStarting, nil + return nil } // ReceiveEvent represents an implementation of EventHandler. diff --git a/holder.go b/holder.go index 0e3d3953c..cd700e586 100644 --- a/holder.go +++ b/holder.go @@ -46,6 +46,9 @@ type Holder struct { indexes map[string]*Index hasData bool + // opened channel is closed once Open() completes. + opened chan struct{} + Broadcaster Broadcaster // Close management wg sync.WaitGroup @@ -69,6 +72,8 @@ func NewHolder() *Holder { indexes: make(map[string]*Index), closing: make(chan struct{}, 0), + opened: make(chan struct{}), + Broadcaster: NopBroadcaster, Stats: NopStatsClient, @@ -156,6 +161,8 @@ func (h *Holder) Open() error { go func() { defer h.wg.Done(); h.monitorCacheFlush() }() h.Stats.Open() + + close(h.opened) return nil } diff --git a/server.go b/server.go index 3944fb0bf..b677147bd 100644 --- a/server.go +++ b/server.go @@ -196,13 +196,15 @@ func (s *Server) Open() error { if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } - s.Cluster.setNodeState(NodeStateReady) + if err := s.Cluster.SetNodeState(NodeStateReady); err != nil { + return fmt.Errorf("setting nodeState: %v", err) + } // Listen for joining nodes. // This needs to start after the Holder has opened so that nodes can join // the cluster without waiting for data to load on the coordinator. Before - // this starts, the joins are queued up in the Cluster.joiningURIs buffered - // channel. + // this starts, the joins are queued up in the Cluster.joiningLeavingNodes + // buffered channel. s.Cluster.ListenForJoins() // Start background monitoring. diff --git a/test/cluster.go b/test/cluster.go index 82c45aef6..449f50580 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -250,8 +250,13 @@ func (t *TestCluster) SetState(state string) { // Open opens all clusters in the test cluster. func (t *TestCluster) Open() error { for _, c := range t.Clusters { - err := c.Open() - if err != nil { + if err := c.Open(); err != nil { + return err + } + if err := c.Holder.Open(); err != nil { + return err + } + if err := c.SetNodeState(pilosa.NodeStateReady); err != nil { return err } } From f2c32f8ec9a5ca8cd375dfdeb40b3be7eb37b5bf Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Nov 2017 15:04:26 -0600 Subject: [PATCH 035/311] add logging support to ResizeJob --- cluster.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 7ea161215..39bf5afff 100644 --- a/cluster.go +++ b/cluster.go @@ -831,6 +831,7 @@ func (c *Cluster) listenForJoins() { // added/removed. It also saves a reference to the ResizeJob in the `jobs` map // for future lookup by JobID. func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { + c.logger().Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -838,6 +839,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, err } + c.logger().Printf("generated ResizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1049,6 +1051,14 @@ type ResizeJob struct { mu sync.RWMutex state string + + // The writer for any logging. + LogOutput io.Writer +} + +// logger returns a logger for the resize job. +func (j *ResizeJob) logger() *log.Logger { + return log.New(j.LogOutput, "", log.LstdFlags) } // NewResizeJob returns a new instance of ResizeJob. @@ -1076,10 +1086,11 @@ func NewResizeJob(existingURIs []URI, uri URI, action string) *ResizeJob { } return &ResizeJob{ - ID: rand.Int63(), - URIs: uris, - action: action, - result: make(chan string), + ID: rand.Int63(), + URIs: uris, + action: action, + result: make(chan string), + LogOutput: os.Stderr, } } @@ -1141,6 +1152,7 @@ func (j *ResizeJob) urisArePending() bool { } func (j *ResizeJob) distributeResizeInstructions() error { + j.logger().Printf("distributeResizeInstructions for job %d", j.ID) // Loop through the ResizeInstructions in ResizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create @@ -1148,6 +1160,7 @@ func (j *ResizeJob) distributeResizeInstructions() error { node := &Node{ URI: decodeURI(instr.URI), } + j.logger().Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { return err } From 013cd0cd95d3690fe3f956f0aa097551455be61a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Nov 2017 17:08:26 -0600 Subject: [PATCH 036/311] refactor resize instruction logic to support multi-index. wait to open holder on non-coordinator nodes. --- cluster.go | 42 ++++++++++++++++++++++++++---------------- server.go | 20 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/cluster.go b/cluster.go index 39bf5afff..d66883f5e 100644 --- a/cluster.go +++ b/cluster.go @@ -876,33 +876,43 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, pbSchema := c.Holder.EncodeSchema() - // Add to the ResizeJob the instructions for each index. + // multiIndex is a map of sources for each node in toCluster. + // Initialize the map with all the nodes in toCluster. + multiIndex := make(map[URI][]*internal.ResizeSource) + for _, n := range toCluster.Nodes { + multiIndex[n.URI] = nil + } + + // Add to m the instructions for each index. for _, idx := range c.Holder.Indexes() { - // fragSources is map[URI][]*internal.ResizeSource. fragSources, err := c.fragSources(toCluster, idx) if err != nil { return nil, err } for u, sources := range fragSources { - // If a host doesn't need to request data, mark it as complete. - if len(sources) == 0 { - j.URIs[u] = true - continue + for _, src := range sources { + multiIndex[u] = append(multiIndex[u], src) } - // TODO: we can probably consolidate the instructions that go to the same - // node but apply to different indexes. (i.e. don't nest this in the Indexes() loop) - instr := &internal.ResizeInstruction{ - JobID: j.ID, - URI: u.Encode(), - Coordinator: encodeURI(c.Coordinator), - Sources: sources, - Schema: pbSchema, // Include the schema to ensure it's in sync on the receiving node. - } - j.Instructions = append(j.Instructions, instr) } } + for u, sources := range multiIndex { + // If a host doesn't need to request data, mark it as complete. + if len(sources) == 0 { + j.URIs[u] = true + continue + } + instr := &internal.ResizeInstruction{ + JobID: j.ID, + URI: u.Encode(), + Coordinator: encodeURI(c.Coordinator), + Sources: sources, + Schema: pbSchema, // Include the schema to ensure it's in sync on the receiving node. + } + j.Instructions = append(j.Instructions, instr) + } + return j, nil } diff --git a/server.go b/server.go index b677147bd..7328eac80 100644 --- a/server.go +++ b/server.go @@ -51,6 +51,11 @@ type Server struct { wg sync.WaitGroup closing chan struct{} + // joining is held open until this node + // receives ClusterStatus from the coordinator. + joining chan struct{} + joined bool + // Data storage and HTTP interface. Holder *Holder Handler *Handler @@ -84,6 +89,7 @@ type Server struct { func NewServer() *Server { s := &Server{ closing: make(chan struct{}), + joining: make(chan struct{}), Holder: NewHolder(), Handler: NewHandler(), @@ -192,6 +198,12 @@ func (s *Server) Open() error { return fmt.Errorf("opening Cluster: %v", err) } + // If not coordinator then wait for ClusterStatus from coordinator. + if !s.Cluster.IsCoordinator() { + s.Logger().Printf("wait for joining to complete") + <-s.joining + } + // Open holder. if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) @@ -216,6 +228,13 @@ func (s *Server) Open() error { return nil } +func (s *Server) markAsJoined() { + if !s.joined { + s.joined = true + close(s.joining) + } +} + // Close closes the server and waits for it to shutdown. func (s *Server) Close() error { // Notify goroutines to stop. @@ -351,6 +370,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + s.markAsJoined() case *internal.ResizeInstruction: err := s.Cluster.FollowResizeInstruction(obj) if err != nil { From ae63adfaac349b0de7da156966c9e2891e4db380 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 27 Nov 2017 10:32:49 -0600 Subject: [PATCH 037/311] let remote nodes know that its safe to open Holder when launching based on existing taxonomy --- cluster.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/cluster.go b/cluster.go index d66883f5e..9f30c0f94 100644 --- a/cluster.go +++ b/cluster.go @@ -301,10 +301,7 @@ func (c *Cluster) SetNodeState(state string) error { State: state, } - node := &Node{ - URI: c.Coordinator, - } - if err := c.Broadcaster.SendTo(node, ns); err != nil { + if err := c.sendTo(c.Coordinator, ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -779,6 +776,14 @@ func (c *Cluster) setStateAndBroadcast(state string) error { return c.Broadcaster.SendSync(c.Status()) } +func (c *Cluster) sendTo(to URI, msg proto.Message) error { + node := &Node{URI: to} + if err := c.Broadcaster.SendTo(node, msg); err != nil { + return err + } + return nil +} + // ListenForJoins handles cluster-resize events. func (c *Cluster) ListenForJoins() { c.wg.Add(1) @@ -876,14 +881,14 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, pbSchema := c.Holder.EncodeSchema() - // multiIndex is a map of sources for each node in toCluster. - // Initialize the map with all the nodes in toCluster. + // multiIndex is a map of sources initialized with all the nodes in toCluster. multiIndex := make(map[URI][]*internal.ResizeSource) + for _, n := range toCluster.Nodes { multiIndex[n.URI] = nil } - // Add to m the instructions for each index. + // Add to multiIndex the instructions for each index. for _, idx := range c.Holder.Indexes() { fragSources, err := c.fragSources(toCluster, idx) if err != nil { @@ -1002,10 +1007,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err complete.Error = err.Error() } - node := &Node{ - URI: decodeURI(instr.Coordinator), - } - if err := c.Broadcaster.SendTo(node, complete); err != nil { + if err := c.sendTo(decodeURI(instr.Coordinator), complete); err != nil { c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) } }() @@ -1401,6 +1403,10 @@ func (c *Cluster) nodeJoin(uri URI) error { if c.haveTopologyAgreement() && c.allNodesReady() { return c.setStateAndBroadcast(ClusterStateNormal) + } else { + // Send the status to the remote node. This lets the remote node + // know that it can proceed with opening its Holder. + return c.sendTo(uri, c.Status()) } return nil From ccdd6262c59a8c3ad107fea2fc337956044ef4c3 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 28 Nov 2017 12:43:49 -0600 Subject: [PATCH 038/311] add error logging for non-topology nodeJoin --- cluster.go | 4 +++- gossip/gossip.go | 4 ++-- server.go | 21 ++++----------------- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/cluster.go b/cluster.go index 9f30c0f94..6d02d352b 100644 --- a/cluster.go +++ b/cluster.go @@ -1382,7 +1382,9 @@ func (c *Cluster) nodeJoin(uri URI) error { if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsURI(uri) { - return fmt.Errorf("host is not in topology: %v", uri) + err := fmt.Sprintf("host is not in topology: %v", uri) + c.logger().Print(err) + return errors.New(err) } if err := c.AddNode(uri); err != nil { diff --git a/gossip/gossip.go b/gossip/gossip.go index 2c5c85faf..7e89e17c1 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -148,7 +148,7 @@ func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSe g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) g.config.memberlistConfig.AdvertisePort = gossipPort - //g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. // TODO travis: change this from 0 + //g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. g.config.memberlistConfig.Delegate = g g.config.memberlistConfig.SecretKey = secretKey g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) @@ -336,7 +336,7 @@ func (g *GossipEventReceiver) listen() { Event: nodeEventType, URI: *uri, } - g.eventHandler.ReceiveEvent(ne) + _ = g.eventHandler.ReceiveEvent(ne) // TODO: don't swallow this error } } diff --git a/server.go b/server.go index 7328eac80..8a4ad38d5 100644 --- a/server.go +++ b/server.go @@ -432,6 +432,10 @@ func (s *Server) ClusterStatus() (proto.Message, error) { // HandleRemoteStatus receives incoming NodeStatus from remote nodes. func (s *Server) HandleRemoteStatus(pb proto.Message) error { + // Ignore NodeStatus messages until the cluster is in a Normal state. + if s.Cluster.State != ClusterStateNormal { + return nil + } return s.mergeRemoteStatus(pb.(*internal.NodeStatus)) } @@ -441,23 +445,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return nil } - // If this node is still STARTING, don't apply remote status. - // There is an issue where starting up a cluster with existing - // data will error on `flock: resource temporarily unavailable`. - // This is because the ApplySchema creates/opens indexes before - // Holder.Open() has run. When Holder.Open() runs later, the - // fragment files are locked. - // TODO: There is still a race condition where the coordinator - // changes state to NORMAL, broadcasts that to the remote node, - // the remote node receives a `NodeStatus` (with schema) before - // running `Holder.Open()`. In that case, state would be NORMAL, - // meaning this check wouldn't pass, and `Holder.Open()` still - // hasn't run. We may need to track whether `Holder.Open()` has - // run, and use that to determine if we bail here. - if s.Cluster.State == ClusterStateStarting { - return nil - } - // Sync schema. if err := s.Holder.ApplySchema(ns.Schema); err != nil { return err From 8228ef238203eaebc0a41ad62ad38d390bc44deb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 28 Nov 2017 16:40:05 -0600 Subject: [PATCH 039/311] Add a Cluster.Static override for tests to treat static nodes as Coordinator. Refactor the Server.joining channel to be in Cluster instead. --- cluster.go | 32 +++++++++++++++++++++++++++++++- server.go | 21 +-------------------- server/server.go | 14 +++++++++++++- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/cluster.go b/cluster.go index 6d02d352b..50996d6f1 100644 --- a/cluster.go +++ b/cluster.go @@ -166,6 +166,7 @@ type Cluster struct { Topology *Topology // Required for cluster Resize. + Static bool // Static is primarily used for testing in a non-gossip environment. State string Coordinator URI Holder *Holder @@ -173,6 +174,11 @@ type Cluster struct { joiningLeavingNodes chan nodeAction + // joining is held open until this node + // receives ClusterStatus from the coordinator. + joining chan struct{} + joined bool + mu sync.RWMutex jobs map[int64]*ResizeJob currentJob *ResizeJob @@ -197,6 +203,7 @@ func NewCluster() *Cluster { joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel jobs: make(map[int64]*ResizeJob), closing: make(chan struct{}), + joining: make(chan struct{}), LogOutput: os.Stderr, prefect: &NopSecurityManager{}, @@ -210,7 +217,7 @@ func (c *Cluster) logger() *log.Logger { // IsCoordinator is true if this node is the coordinator. func (c *Cluster) IsCoordinator() bool { - return c.Coordinator == c.URI + return c.Static || c.Coordinator == c.URI } // SetCoordinator updates the Coordinator to new if it is @@ -709,6 +716,12 @@ func (c *Cluster) Open() error { return fmt.Errorf("opening MemberSet: %v", err) } + // If not coordinator then wait for ClusterStatus from coordinator. + if !c.IsCoordinator() { + c.logger().Printf("wait for joining to complete") + <-c.joining + } + return nil } @@ -720,15 +733,28 @@ func (c *Cluster) Close() error { return nil } +func (c *Cluster) MarkAsJoined() { + if !c.joined { + c.joined = true + close(c.joining) + } +} + func (c *Cluster) needTopologyAgreement() bool { return c.State == ClusterStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } func (c *Cluster) haveTopologyAgreement() bool { + if c.Static { + return true + } return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } func (c *Cluster) allNodesReady() bool { + if c.Static { + return true + } for _, uri := range c.Topology.NodeSet { if c.Topology.nodeStates[uri] != NodeStateReady { return false @@ -1335,6 +1361,10 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { } func (c *Cluster) considerTopology() error { + if c.Static { + return nil + } + // If there is no .topology file, it's safe to proceed. if len(c.Topology.NodeSet) == 0 { return nil diff --git a/server.go b/server.go index 8a4ad38d5..37270b0f1 100644 --- a/server.go +++ b/server.go @@ -51,11 +51,6 @@ type Server struct { wg sync.WaitGroup closing chan struct{} - // joining is held open until this node - // receives ClusterStatus from the coordinator. - joining chan struct{} - joined bool - // Data storage and HTTP interface. Holder *Holder Handler *Handler @@ -89,7 +84,6 @@ type Server struct { func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - joining: make(chan struct{}), Holder: NewHolder(), Handler: NewHandler(), @@ -198,12 +192,6 @@ func (s *Server) Open() error { return fmt.Errorf("opening Cluster: %v", err) } - // If not coordinator then wait for ClusterStatus from coordinator. - if !s.Cluster.IsCoordinator() { - s.Logger().Printf("wait for joining to complete") - <-s.joining - } - // Open holder. if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) @@ -228,13 +216,6 @@ func (s *Server) Open() error { return nil } -func (s *Server) markAsJoined() { - if !s.joined { - s.joined = true - close(s.joining) - } -} - // Close closes the server and waits for it to shutdown. func (s *Server) Close() error { // Notify goroutines to stop. @@ -370,7 +351,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } - s.markAsJoined() + s.Cluster.MarkAsJoined() case *internal.ResizeInstruction: err := s.Cluster.FollowResizeInstruction(obj) if err != nil { diff --git a/server/server.go b/server/server.go index b4d067683..5237df085 100644 --- a/server/server.go +++ b/server/server.go @@ -63,7 +63,7 @@ type Command struct { // Standard input/output *pilosa.CmdIO - // running will be closed once Command.Run is finished. + // Started will be closed once Command.Run is finished. Started chan struct{} // Done will be closed when Command.Close() is called Done chan struct{} @@ -220,6 +220,18 @@ func (m *Command) SetupServer() error { m.Server.Broadcaster = gossipMemberSet m.Server.BroadcastReceiver = gossipMemberSet case pilosa.ClusterStatic, pilosa.ClusterNone: + + m.Server.Cluster.Static = true + for _, address := range m.Config.Cluster.Hosts { + uri, err := pilosa.NewURIFromAddress(address) + if err != nil { + return err + } + cluster.Nodes = append(cluster.Nodes, &pilosa.Node{ + URI: *uri, + }) + } + m.Server.Broadcaster = pilosa.NopBroadcaster m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver From 91c7abfa9a01a4a2ca71b35fe55eea25318e6703 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 28 Nov 2017 16:47:40 -0600 Subject: [PATCH 040/311] remove outdated TODO --- handler.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/handler.go b/handler.go index 1ef7be7f9..11b1f0cb9 100644 --- a/handler.go +++ b/handler.go @@ -2010,8 +2010,6 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht // TODO: prevent removing the coordinator node // Start the resize process (similar to NodeJoin) - // TODO: this currently blocks. we should leverage listenForJoins() in cluster - // by converted it to a channel of nodeAction {URI, action}. err := h.Cluster.NodeLeave(*removeURI) if err != nil { return err From 59cdd5dde94f8309fa04a3876cdab839bd324ae7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 29 Nov 2017 12:45:27 -0600 Subject: [PATCH 041/311] Add HolderCleaner and view.DeleteFragment to support post-resize cleanups Add tests for view.DeleteFragment and HolderCleaner --- cluster.go | 50 +++++++++++++++++- cluster_test.go | 50 +++++++++++------- holder.go | 62 ++++++++++++++++++++++ holder_test.go | 138 ++++++++++++++++++++++++++++++++++++++++++++++++ view.go | 34 ++++++++++++ view_test.go | 39 ++++++++++++-- 6 files changed, 348 insertions(+), 25 deletions(-) diff --git a/cluster.go b/cluster.go index 50996d6f1..68878f7b6 100644 --- a/cluster.go +++ b/cluster.go @@ -284,6 +284,10 @@ func (c *Cluster) setState(state string) { return } + c.logger().Printf("Change cluster state from %s to %s", c.State, state) + + var doCleanup bool + switch state { case ClusterStateResizing: c.prefect.SetRestricted() @@ -291,10 +295,28 @@ func (c *Cluster) setState(state string) { c.prefect.SetNormal() // Don't change routing for these states: // - ClusterStateStarting + + // If state is RESIZING -> NORMAL then run cleanup. + if c.State == ClusterStateResizing { + doCleanup = true + } } - c.logger().Printf("Change cluster state from %s to %s", c.State, state) c.State = state + + // It's safe to do a cleanup after state changes back to normal. + if doCleanup { + var cleaner HolderCleaner + cleaner.URI = c.URI + cleaner.Holder = c.Holder + cleaner.Cluster = c + cleaner.Closing = c.closing + + // Clean holder. + if err := cleaner.CleanHolder(); err != nil { + c.logger().Printf("holder clean error: err=%s", err) + } + } } func (c *Cluster) SetNodeState(state string) error { @@ -315,11 +337,19 @@ func (c *Cluster) SetNodeState(state string) error { return nil } +// ReceiveNodeState set node state in Topology in order for the +// Coordinator to keep track of, during startup, which nodes have +// finished opening their Holder. func (c *Cluster) ReceiveNodeState(uri URI, state string) error { if !c.IsCoordinator() { return nil } + // This method is really only useful during initial startup. + if c.State != ClusterStateStarting { + return nil + } + c.Topology.nodeStates[uri] = state // Set cluster state to NORMAL. @@ -649,7 +679,7 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// OwnsSlices find the set of slices owned by the node per Index +// OwnsSlices finds the set of slices owned by the node per Index func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { @@ -663,6 +693,22 @@ func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { return slices } +// ContainsSlices is like OwnsSlices, but it includes replicas. +func (c *Cluster) ContainsSlices(index string, maxSlice uint64, uri URI) []uint64 { + var slices []uint64 + for i := uint64(0); i <= maxSlice; i++ { + p := c.Partition(index, i) + // Determine the nodes for partition. + nodes := c.PartitionNodes(p) + for _, node := range nodes { + if node.URI == uri { + slices = append(slices, i) + } + } + } + return slices +} + // Hasher represents an interface to hash integers into buckets. type Hasher interface { // Hashes the key into a number between [0,N). diff --git a/cluster_test.go b/cluster_test.go index b444fbf59..a7fb77c83 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -102,6 +102,17 @@ func TestCluster_OwnsSlices(t *testing.T) { } } +// Ensure ContainsSlices can find the actual slice list for node and index. +func TestCluster_ContainsSlices(t *testing.T) { + c := test.NewCluster(5) + c.ReplicaN = 3 + slices := c.ContainsSlices("test", 10, test.NewURIFromHostPort("host2", 0)) + + if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected slices for node's index: %v", slices) + } +} + func TestCluster_Nodes(t *testing.T) { uri0 := test.NewURIFromHostPort("node0", 0) uri1 := test.NewURIFromHostPort("node1", 0) @@ -407,6 +418,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, with data", func(t *testing.T) { tc := test.NewTestCluster(0) tc.AddNode(false) + node0 := tc.Clusters[0] // Open TestCluster. if err := tc.Open(); err != nil { @@ -441,15 +453,23 @@ func TestCluster_ResizeStates(t *testing.T) { tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) - // AddNode needs to block until the resize process has completed. - if err := tc.AddNode(false); err != nil { - t.Fatal(err) - } + // Before starting the resize, get the CheckSum to use for + // comparison later. + node0Frame := node0.Holder.Frame("i", "f") + node0View := node0Frame.View("standard") + node0Fragment := node0View.Fragment(1) + node0Checksum := node0Fragment.Checksum() - node0 := tc.Clusters[0] + node0Frame = node0.Holder.Frame("i", "fields") + node0View = node0Frame.View("field_fld0") + node0Fragment = node0View.Fragment(1) + node0ChecksumFld := node0Fragment.Checksum() + + // AddNode needs to block until the resize process has completed. + tc.AddNode(false) node1 := tc.Clusters[1] - // Ensure that nodes comes up in state NORMAL. + // Ensure that nodes come up in state NORMAL. if node0.State != pilosa.ClusterStateNormal { t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) } else if node1.State != pilosa.ClusterStateNormal { @@ -469,34 +489,24 @@ func TestCluster_ResizeStates(t *testing.T) { // Bits // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node0Frame := node0.Holder.Frame("i", "f") - node0View := node0Frame.View("standard") - node0Fragment := node0View.Fragment(1) - node1Frame := node1.Holder.Frame("i", "f") node1View := node1Frame.View("standard") node1Fragment := node1View.Fragment(1) // Ensure checksums are the same. - orig := node0Fragment.Checksum() - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, orig) { - t.Fatalf("expected standard view checksum to match: %x - %x", chksum, orig) + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) } // Values // Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0. - node0Frame = node0.Holder.Frame("i", "fields") - node0View = node0Frame.View("field_fld0") - node0Fragment = node0View.Fragment(1) - node1Frame = node1.Holder.Frame("i", "fields") node1View = node1Frame.View("field_fld0") node1Fragment = node1View.Fragment(1) // Ensure checksums are the same. - orig = node0Fragment.Checksum() - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, orig) { - t.Fatalf("expected field view checksum to match: %x - %x", chksum, orig) + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) { + t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld) } // Close TestCluster. diff --git a/holder.go b/holder.go index cd700e586..d5a08ec58 100644 --- a/holder.go +++ b/holder.go @@ -712,3 +712,65 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err return nil } + +// HolderCleaner removes fragments and data files that are no longer used. +type HolderCleaner struct { + URI URI + + Holder *Holder + Cluster *Cluster + + // Signals that the sync should stop. + Closing <-chan struct{} +} + +// IsClosing returns true if the cleaner has been marked to close. +func (c *HolderCleaner) IsClosing() bool { + select { + case <-c.Closing: + return true + default: + return false + } +} + +// CleanHolder compares the holder with the cluster state and removes +// any unnecessary fragments and files. +func (c *HolderCleaner) CleanHolder() error { + for _, index := range c.Holder.Indexes() { + // Verify cleaner has not closed. + if c.IsClosing() { + return nil + } + + // Get the fragments that node is responsible for (based on hash(index, node)). + containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.URI) + + // Get the fragments registered in memory. + for _, frame := range index.Frames() { + for _, view := range frame.Views() { + for _, fragment := range view.Fragments() { + fragSlice := fragment.Slice() + // Ignore fragments that should be present. + if uint64InSlice(fragSlice, containedSlices) { + continue + } + // Delete fragment. + if err := view.DeleteFragment(fragSlice); err != nil { + return err + } + } + } + } + } + return nil +} + +func uint64InSlice(i uint64, s []uint64) bool { + for _, o := range s { + if i == o { + return true + } + } + return false +} diff --git a/holder_test.go b/holder_test.go index f182d3496..5121db568 100644 --- a/holder_test.go +++ b/holder_test.go @@ -506,3 +506,141 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } } } + +// Ensure holder can clean up orphaned fragments. +func TestHolderCleaner_CleanHolder(t *testing.T) { + cluster := test.NewCluster(2) + + // Create a local holder. + hldr0 := test.MustOpenHolder() + defer hldr0.Close() + + // Mock 2-node, fully replicated cluster. + cluster.ReplicaN = 2 + + cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + + // Create frames on nodes. + for _, hldr := range []*test.Holder{hldr0} { + hldr.MustCreateFrameIfNotExists("i", "f") + hldr.MustCreateFrameIfNotExists("i", "f0") + hldr.MustCreateFrameIfNotExists("y", "z") + } + + // Set data on the local holder. + f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) + if _, err := f.SetBit(0, 10); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 4000); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(2, 20); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(3, 10); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(120, 10); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(200, 4); err != nil { + t.Fatal(err) + } + + f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) + if _, err := f.SetBit(9, SliceWidth+5); err != nil { + t.Fatal(err) + } + + f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2) + if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil { + t.Fatal(err) + } + + // Set highest slice. + hldr0.Index("i").SetRemoteMaxSlice(1) + hldr0.Index("y").SetRemoteMaxSlice(2) + + // Keep replication the same and ensure we get the expected results. + cluster.ReplicaN = 2 + + // Set up cleaner for replication 2. + cleaner2 := pilosa.HolderCleaner{ + URI: cluster.Nodes[0].URI, + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner2.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*test.Holder{hldr0} { + f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected bits(%d/0): %+v", i, a) + } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected bits(%d/2): %+v", i, a) + } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/3): %+v", i, a) + } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/120): %+v", i, a) + } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected bits(%d/200): %+v", i, a) + } + + f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + a := f.Row(9).Bits() + if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) + } + if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + } + f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) + if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + } + } + + // Change replication factor to ensure we have fragments to remove. + cluster.ReplicaN = 1 + + // Set up cleaner for replication 1. + cleaner1 := pilosa.HolderCleaner{ + URI: cluster.Nodes[0].URI, + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner1.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*test.Holder{hldr0} { + f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected bits(%d/0): %+v", i, a) + } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected bits(%d/2): %+v", i, a) + } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/3): %+v", i, a) + } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/120): %+v", i, a) + } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected bits(%d/200): %+v", i, a) + } + + f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + if f != nil { + t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) + } + + f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) + if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + } + } +} diff --git a/view.go b/view.go index 5da97d644..c6c478147 100644 --- a/view.go +++ b/view.go @@ -18,6 +18,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "os" "path/filepath" "strconv" @@ -119,6 +120,9 @@ func (v *View) Open() error { return nil } +// logger returns a logger instance for the view. +func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) } + // openFragments opens and initializes the fragments inside the view. func (v *View) openFragments() error { file, err := os.Open(filepath.Join(v.path, "fragments")) @@ -270,6 +274,36 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { return frag } +// DeleteFragment removes the fragment from the view. +func (v *View) DeleteFragment(slice uint64) error { + + fragment := v.fragments[slice] + if fragment == nil { + return ErrFragmentNotFound + } + + v.logger().Printf("delete fragment: %d", slice) + + // Close data files before deletion. + if err := fragment.Close(); err != nil { + return err + } + + // Delete fragment file. + if err := os.Remove(fragment.Path()); err != nil { + return err + } + + // Delete fragment cache file. + if err := os.Remove(fragment.CachePath()); err != nil { + v.logger().Printf("no cache file to delete for slice %d", slice) + } + + delete(v.fragments, slice) + + return nil +} + // SetBit sets a bit within the view. func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth diff --git a/view_test.go b/view_test.go index 5f5ce94e2..64041aea0 100644 --- a/view_test.go +++ b/view_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "io/ioutil" "os" + "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" @@ -30,14 +31,13 @@ type View struct { // NewView returns a new instance of View with a temporary path. func NewView(index, frame, name string) *View { - file, err := ioutil.TempFile("", "pilosa-view-") + path, err := ioutil.TempDir("", "pilosa-view-") if err != nil { panic(err) } - file.Close() v := &View{ - View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize), + View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize), RowAttrStore: test.MustOpenAttrStore(), } v.View.RowAttrStore = v.RowAttrStore.AttrStore @@ -93,3 +93,36 @@ func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { } } } + +// Ensure view can open and retrieve a fragment. +func TestView_DeleteFragment(t *testing.T) { + v := MustOpenView("i", "f", "v") + defer v.Close() + + slice := uint64(9) + + // Create fragment. + fragment, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == nil { + t.Fatal("expected fragment") + } + + err = v.DeleteFragment(slice) + if err != nil { + t.Fatal(err) + } + + if v.Fragment(slice) != nil { + t.Fatal("fragment still exists in view") + } + + // Recreate fragment with same slice, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == fragment2 { + t.Fatal("failed to create new fragment") + } +} From 073848ae3107b26f43a2b823a6dd33362a3d6cf7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 29 Nov 2017 16:34:14 -0600 Subject: [PATCH 042/311] Make sure node gets removed from all nodeSets after resize --- cluster.go | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/cluster.go b/cluster.go index 68878f7b6..c44f35d55 100644 --- a/cluster.go +++ b/cluster.go @@ -304,6 +304,8 @@ func (c *Cluster) setState(state string) { c.State = state + // TODO: consider NOT running cleanup on an active node that has + // been removed. // It's safe to do a cleanup after state changes back to normal. if doCleanup { var cleaner HolderCleaner @@ -529,7 +531,7 @@ func (c *Cluster) diff(other *Cluster) (action string, uri URI, err error) { break } } - } else if len(c.Nodes) > len(other.Nodes) { + } else if lenFrom > lenTo { // Removing a node. if lenFrom-lenTo > 1 { return action, uri, errors.New("removing more than one node at a time is not supported") @@ -1274,6 +1276,16 @@ func (u NodeSet) ToStrings() []string { return other } +// ContainsURI returns true if uri matches one of the nodesets's uris. +func (n NodeSet) ContainsURI(uri URI) bool { + for _, nuri := range n { + if nuri == uri { + return true + } + } + return false +} + // Topology represents the list of hosts in the cluster. type Topology struct { mu sync.RWMutex @@ -1298,12 +1310,7 @@ func (t *Topology) ContainsURI(uri URI) bool { } func (t *Topology) containsURI(uri URI) bool { - for _, turi := range t.NodeSet { - if turi == uri { - return true - } - } - return false + return NodeSet(t.NodeSet).ContainsURI(uri) } func (t *Topology) positionByURI(uri URI) int { @@ -1557,9 +1564,25 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { return nil } - for _, uri := range decodeURIs(cs.NodeSet) { - c.AddNode(uri) + officialURIs := decodeURIs(cs.NodeSet) + + // Add all nodes from the coordinator. + for _, uri := range officialURIs { + if err := c.AddNode(uri); err != nil { + return err + } } + + // Remove any nodes not specified by the coordinator. + for _, uri := range c.NodeSet() { + if NodeSet(officialURIs).ContainsURI(uri) { + continue + } + if err := c.RemoveNode(uri); err != nil { + return err + } + } + c.setState(cs.State) return nil From e6ff67bd8374790eef8c97af24420c2833760f12 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 4 Dec 2017 17:47:16 -0600 Subject: [PATCH 043/311] add index/frame/view info to delete fragment log information --- view.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/view.go b/view.go index c6c478147..d77d9257a 100644 --- a/view.go +++ b/view.go @@ -282,7 +282,7 @@ func (v *View) DeleteFragment(slice uint64) error { return ErrFragmentNotFound } - v.logger().Printf("delete fragment: %d", slice) + v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) // Close data files before deletion. if err := fragment.Close(); err != nil { From f17a399a37fbbc1f4e64788328282db528cd8880 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 7 Dec 2017 08:33:26 -0600 Subject: [PATCH 044/311] adjust to single http client --- cluster.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index c44f35d55..f6966a333 100644 --- a/cluster.go +++ b/cluster.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "log" "math/rand" + "net/http" "os" "path/filepath" "sort" @@ -190,6 +191,9 @@ type Cluster struct { // The writer for any logging. LogOutput io.Writer + + // + RemoteClient *http.Client } // NewCluster returns a new instance of Cluster with defaults. @@ -1031,7 +1035,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.URI, nil) // TODO: ClientOptions + client := NewInternalHTTPClientFromURI(&c.URI, GetHTTPClient(nil)) // TODO: ClientOptions // Request each source file in ResizeSources. for _, src := range instr.Sources { From c14bc2904114f7d0fd622842b442c9e525a1b5f3 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 7 Dec 2017 11:17:37 -0600 Subject: [PATCH 045/311] added logging for node membership --- cluster.go | 11 ++++++++++- holder.go | 2 ++ server/server.go | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index f6966a333..368cddba4 100644 --- a/cluster.go +++ b/cluster.go @@ -336,6 +336,7 @@ func (c *Cluster) SetNodeState(state string) error { State: state, } + c.logger().Printf("Sending State %s (%s)", state, c.Coordinator.String()) if err := c.sendTo(c.Coordinator, ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -347,6 +348,8 @@ func (c *Cluster) SetNodeState(state string) error { // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. func (c *Cluster) ReceiveNodeState(uri URI, state string) error { + + c.logger().Printf("Receiving State %s (%s)", state, uri.String()) if !c.IsCoordinator() { return nil } @@ -357,9 +360,11 @@ func (c *Cluster) ReceiveNodeState(uri URI, state string) error { } c.Topology.nodeStates[uri] = state + c.logger().Printf("Receiving State %s (%s)", state, uri.String()) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { + c.logger().Printf("Broadcasting ClusterStateNormal") return c.setStateAndBroadcast(ClusterStateNormal) } @@ -800,6 +805,8 @@ func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } + c.logger().Printf("haveTopologyAgreement") + c.logger().Printf(" (%v)(%v)", c.Topology.NodeSet, c.NodeSet()) return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } @@ -807,7 +814,9 @@ func (c *Cluster) allNodesReady() bool { if c.Static { return true } + c.logger().Printf("allNodesReady") for _, uri := range c.Topology.NodeSet { + c.logger().Printf("allNodesReady: %s,%s", uri.String(), c.Topology.nodeStates[uri]) if c.Topology.nodeStates[uri] != NodeStateReady { return false } @@ -1035,7 +1044,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.URI, GetHTTPClient(nil)) // TODO: ClientOptions + client := NewInternalHTTPClientFromURI(&c.URI, c.RemoteClient) // TODO: ClientOptions // Request each source file in ResizeSources. for _, src := range instr.Sources { diff --git a/holder.go b/holder.go index 122dafd7a..99d9d079d 100644 --- a/holder.go +++ b/holder.go @@ -133,6 +133,7 @@ func (h *Holder) Open() error { return err } + h.logger().Printf("Holder Start") for _, fi := range fis { if !fi.IsDir() { continue @@ -156,6 +157,7 @@ func (h *Holder) Open() error { } h.indexes[index.Name()] = index } + h.logger().Printf("Holder Complete") // Periodically flush cache. h.wg.Add(1) diff --git a/server/server.go b/server/server.go index 9af9a1947..e7396dfb1 100644 --- a/server/server.go +++ b/server/server.go @@ -181,6 +181,7 @@ func (m *Command) SetupServer() error { c := pilosa.GetHTTPClient(TLSConfig) m.Server.RemoteClient = c m.Server.Handler.RemoteClient = c + m.Server.Cluster.RemoteClient = c // Set the coordinator node. curi, err := pilosa.AddressWithDefaults(m.Config.Cluster.Coordinator) From 9565db43a83f25d53c853e01ede26effe642e6aa Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 7 Dec 2017 14:11:49 -0600 Subject: [PATCH 046/311] add sort order to URI addition --- cluster.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 368cddba4..a17323783 100644 --- a/cluster.go +++ b/cluster.go @@ -805,8 +805,6 @@ func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - c.logger().Printf("haveTopologyAgreement") - c.logger().Printf(" (%v)(%v)", c.Topology.NodeSet, c.NodeSet()) return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } @@ -814,9 +812,7 @@ func (c *Cluster) allNodesReady() bool { if c.Static { return true } - c.logger().Printf("allNodesReady") for _, uri := range c.Topology.NodeSet { - c.logger().Printf("allNodesReady: %s,%s", uri.String(), c.Topology.nodeStates[uri]) if c.Topology.nodeStates[uri] != NodeStateReady { return false } @@ -1343,6 +1339,12 @@ func (t *Topology) AddURI(uri URI) bool { return false } t.NodeSet = append(t.NodeSet, uri) + + sort.Slice(t.NodeSet, + func(i, j int) bool { + return t.NodeSet[i].String() < t.NodeSet[j].String() + }) + return true } @@ -1422,6 +1424,10 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { t := NewTopology() t.NodeSet = decodeURIs(topology.NodeSet) + sort.Slice(t.NodeSet, + func(i, j int) bool { + return t.NodeSet[i].String() < t.NodeSet[j].String() + }) return t, nil } From e6adb361b90681bc766f883bd38d851c28067119 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 7 Dec 2017 15:09:42 -0600 Subject: [PATCH 047/311] cleanup logging --- cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index a17323783..1fddefcf3 100644 --- a/cluster.go +++ b/cluster.go @@ -336,7 +336,7 @@ func (c *Cluster) SetNodeState(state string) error { State: state, } - c.logger().Printf("Sending State %s (%s)", state, c.Coordinator.String()) + c.logger().Printf("Sending State %s (%s)", state, c.Coordinator) if err := c.sendTo(c.Coordinator, ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -360,7 +360,7 @@ func (c *Cluster) ReceiveNodeState(uri URI, state string) error { } c.Topology.nodeStates[uri] = state - c.logger().Printf("Receiving State %s (%s)", state, uri.String()) + c.logger().Printf("Receiving State %s (%s)", state, uri) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { From 0342da92bb886fc1829b7ceb90a59ccf4cea3a99 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 12 Dec 2017 16:27:47 -0600 Subject: [PATCH 048/311] WIP: add tests for cluster resize This commit adds support for allocating a gossip transport and a server listener prior to opening server (and cluster). Doing that allows tests to use a dynamically allocated port by supplying bind port: 0. --- cluster.go | 87 ++++++-- gossip/gossip.go | 211 ++++++++++++-------- handler.go | 2 +- holder.go | 5 +- internal/private.pb.go | 234 +++++++++++++--------- internal/private.proto | 1 + server.go | 87 +++++--- server/cluster_test.go | 438 +++++++++++++++++++++++++++++++++++++++++ server/server.go | 76 +++++-- server/server_test.go | 75 +++++++ 10 files changed, 974 insertions(+), 242 deletions(-) create mode 100644 server/cluster_test.go diff --git a/cluster.go b/cluster.go index 132c94ec7..2fdd73b40 100644 --- a/cluster.go +++ b/cluster.go @@ -31,6 +31,8 @@ import ( "sync" "time" + "golang.org/x/sync/errgroup" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -241,6 +243,7 @@ func (c *Cluster) SetCoordinator(oldURI, newURI URI) bool { // AddNode adds a node to the Cluster and updates and saves the // new topology. func (c *Cluster) AddNode(uri URI) error { + c.logger().Printf("add node %s to cluster on %s", uri, c.URI) // add to cluster _, added := c.addNodeBasicSorted(uri) @@ -292,7 +295,7 @@ func (c *Cluster) setState(state string) { return } - c.logger().Printf("Change cluster state from %s to %s", c.State, state) + c.logger().Printf("change cluster state from %s to %s on %s", c.State, state, c.URI) var doCleanup bool @@ -352,8 +355,6 @@ func (c *Cluster) SetNodeState(state string) error { // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. func (c *Cluster) ReceiveNodeState(uri URI, state string) error { - - c.logger().Printf("Receiving State %s (%s)", state, uri.String()) if !c.IsCoordinator() { return nil } @@ -364,11 +365,10 @@ func (c *Cluster) ReceiveNodeState(uri URI, state string) error { } c.Topology.nodeStates[uri] = state - c.logger().Printf("Receiving State %s (%s)", state, uri) + c.logger().Printf("received state %s (%s)", state, uri) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { - c.logger().Printf("Broadcasting ClusterStateNormal") return c.setStateAndBroadcast(ClusterStateNormal) } @@ -765,7 +765,10 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. + //NEXT + //if c.URI.Port() != 0 { c.AddNode(c.URI) + //} // Start the EventReceiver. if err := c.EventReceiver.Start(c); err != nil { @@ -781,6 +784,7 @@ func (c *Cluster) Open() error { if !c.IsCoordinator() { c.logger().Printf("wait for joining to complete") <-c.joining + c.logger().Printf("joining has completed") } return nil @@ -794,7 +798,8 @@ func (c *Cluster) Close() error { return nil } -func (c *Cluster) MarkAsJoined() { +func (c *Cluster) markAsJoined() { + c.logger().Printf("mark node as joined (received coordinator update)") if !c.joined { c.joined = true close(c.joining) @@ -830,14 +835,24 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { return err } - // Run the job. - err = j.Run() - if err != nil { + // j.Run() runs in a goroutine because in the case where the + // job requires no action, it immediately writes to the j.result + // channel, which is not consumed until the code below. + var eg errgroup.Group + eg.Go(func() error { + return j.Run() + }) + + // Wait for the ResizeJob to finish or be aborted. + c.logger().Printf("wait for jobResult") + jobResult := <-j.result + + // Make sure j.Run() didn't return an error. + if eg.Wait() != nil { return err } - // Wait for the ResizeJob to finish or be aborted. - jobResult := <-j.result + c.logger().Printf("received jobResult: %s", jobResult) switch jobResult { case ResizeJobStateDone: if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { @@ -860,6 +875,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { func (c *Cluster) setStateAndBroadcast(state string) error { c.setState(state) // Broadcast cluster status changes to the cluster. + c.logger().Printf("broadcasting ClusterStatus: %s", state) return c.Broadcaster.SendSync(c.Status()) } @@ -996,11 +1012,12 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, continue } instr := &internal.ResizeInstruction{ - JobID: j.ID, - URI: u.Encode(), - Coordinator: encodeURI(c.Coordinator), - Sources: sources, - Schema: pbSchema, // Include the schema to ensure it's in sync on the receiving node. + JobID: j.ID, + URI: u.Encode(), + Coordinator: encodeURI(c.Coordinator), + Sources: sources, + Schema: pbSchema, // Include the schema to ensure it's in sync on the receiving node. + ClusterStatus: c.Status(), } j.Instructions = append(j.Instructions, instr) } @@ -1023,6 +1040,17 @@ func (c *Cluster) CompleteCurrentJob(state string) error { // FollowResizeInstruction is run by any node that receives a ResizeInstruction. func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + c.logger().Printf("follow resize instruction on %s", c.URI) + // Make sure the cluster status on this node agrees with the Coordinator + // before attempting a resize. + if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { + return err + } + + c.logger().Printf("MergeClusterStatus done, start goroutine") + + // The actual resizing runs in a goroutine because we don't want to block + // the distribution of other ResizeInstructions to the rest of the cluster. go func() { // Make sure the holder has opened. @@ -1039,6 +1067,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err if err := func() error { // Sync the schema received in the resize instruction. + c.logger().Printf("Holder ApplySchema") if err := c.Holder.ApplySchema(instr.Schema); err != nil { return err } @@ -1048,7 +1077,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.logger().Printf("\n**** Get slice %d for index %s from host %s ****\n\n", src.Slice, src.Index, src.URI) + c.logger().Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.URI) srcURI := decodeURI(src.URI) @@ -1071,8 +1100,18 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Stream slice from remote node. + c.logger().Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.URI) rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) if err != nil { + // For now it is an acceptable error if the fragment is not found + // on the remote node. This occurs when a slice has been skipped and + // therefore doesn't contain data. The coordinator correctly determined + // the resize instruction to retrieve the slice, but it doesn't have data. + // TODO: figure out a way to distinguish from "fragment not found" errors + // which are true errors and which simply mean the fragment doesn't have data. + if err == ErrFragmentNotFound { + return nil + } return err } else if rd == nil { return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.URI) @@ -1213,15 +1252,18 @@ func (j *ResizeJob) setState(state string) { // Run distributes ResizeInstructions. func (j *ResizeJob) Run() error { + j.logger().Printf("run ResizeJob") // Set job state to RUNNING. j.SetState(ResizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.urisArePending() { + j.logger().Printf("ResizeJob contains no pending tasks; mark as done") j.result <- ResizeJobStateDone return nil } + j.logger().Printf("distribute tasks for ResizeJob") err := j.distributeResizeInstructions() if err != nil { j.result <- ResizeJobStateAborted @@ -1470,6 +1512,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { switch e.Event { case NodeJoin: + c.logger().Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. if !c.IsCoordinator() { return nil @@ -1582,6 +1625,7 @@ func (c *Cluster) nodeLeave(uri URI) error { } func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { + c.logger().Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). if c.IsCoordinator() { return nil @@ -1596,8 +1640,13 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } } - // Remove any nodes not specified by the coordinator. + // Remove any nodes not specified by the coordinator + // except for self. for _, uri := range c.NodeSet() { + // Don't remove this node. + if uri == c.URI { + continue + } if NodeSet(officialURIs).ContainsURI(uri) { continue } @@ -1608,5 +1657,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { c.setState(cs.State) + c.markAsJoined() + return nil } diff --git a/gossip/gossip.go b/gossip/gossip.go index 28b530e93..9d5845315 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -142,81 +142,26 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } -// newTransport returns a NetTransport based on the memberlist configuration. -// It will dynamically bind to a port if conf.BindPort is 0. -// This is useful for test cases where specifiying a port is not reasonable. -func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { - if conf.LogOutput != nil && conf.Logger != nil { - return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") - } +// NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport. +func NewGossipMemberSetWithTransport(name string, gossipHost string, transport *Transport, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipMemberSet, error) { + port := transport.Net.GetAutoBindPort() - logDest := conf.LogOutput - if logDest == nil { - logDest = os.Stderr - } - - logger := conf.Logger - if logger == nil { - logger = log.New(logDest, "", log.LstdFlags) - } - - nc := &memberlist.NetTransportConfig{ - BindAddrs: []string{conf.BindAddr}, - BindPort: conf.BindPort, - Logger: logger, - } - - // See comment below for details about the retry in here. - makeNetRetry := func(limit int) (*memberlist.NetTransport, error) { - var err error - for try := 0; try < limit; try++ { - var nt *memberlist.NetTransport - if nt, err = memberlist.NewNetTransport(nc); err == nil { - return nt, nil - } - if strings.Contains(err.Error(), "address already in use") { - logger.Printf("[DEBUG] Got bind error: %v", err) - continue - } - } - - return nil, fmt.Errorf("failed to obtain an address: %v", err) - } - - // The dynamic bind port operation is inherently racy because - // even though we are using the kernel to find a port for us, we - // are attempting to bind multiple protocols (and potentially - // multiple addresses) with the same port number. We build in a - // few retries here since this often gets transient errors in - // busy unit tests. - limit := 1 - if conf.BindPort == 0 { - limit = 10 - } - - nt, err := makeNetRetry(limit) - if err != nil { - return nil, fmt.Errorf("Could not set up network transport: %v", err) - } - if conf.BindPort == 0 { - port := nt.GetAutoBindPort() - conf.BindPort = port - conf.AdvertisePort = port - logger.Printf("[DEBUG] Using dynamic bind port %d", port) - } - - return nt, nil -} - -// NewGossipMemberSet returns a new instance of GossipMemberSet. -func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipMemberSet, error) { g := &GossipMemberSet{ LogOutput: server.LogOutput, } + // memberlist config conf := memberlist.DefaultLocalConfig() - conf.BindPort = gossipPort - conf.AdvertisePort = gossipPort + conf.Transport = transport.Net + conf.BindPort = port + conf.AdvertisePort = port + conf.Name = name + conf.BindAddr = gossipHost + conf.AdvertiseAddr = pilosa.HostToIP(gossipHost) + //conf.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. + conf.Delegate = g + conf.SecretKey = secretKey + conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) //TODO: pull memberlist config from pilosa.cfg file g.config = &gossipConfig{ @@ -224,32 +169,27 @@ func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSe gossipSeed: gossipSeed, } - g.config.memberlistConfig.Name = name - g.config.memberlistConfig.BindAddr = gossipHost - g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) - g.config.memberlistConfig.AdvertisePort = gossipPort - //g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. - g.config.memberlistConfig.Delegate = g - g.config.memberlistConfig.SecretKey = secretKey - g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) - g.statusHandler = server - // set up the transport - transport, err := newTransport(g.config.memberlistConfig) - if err != nil { - return nil, err - } - g.config.memberlistConfig.Transport = transport - // If no gossipSeed is provided, use local host:port. if gossipSeed == "" { - g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, g.config.memberlistConfig.BindPort) + g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, port) } return g, nil } +// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port. +func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipMemberSet, error) { + // set up the transport + transport, err := NewTransport(gossipHost, gossipPort) + if err != nil { + return nil, err + } + + return NewGossipMemberSetWithTransport(name, gossipHost, transport, gossipSeed, server, secretKey) +} + // SendSync implementation of the Broadcaster interface. func (g *GossipMemberSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) @@ -432,3 +372,102 @@ func (b *broadcast) Finished() { close(b.notify) } } + +// Transport is a gossip transport for binding to a port. +type Transport struct { + //memberlist.Transport + Net *memberlist.NetTransport + URI *pilosa.URI +} + +// NewTransport returns a NetTransport based on the given host and port. +// It will dynamically bind to a port if port is 0. +// This is useful for test cases where specifiying a port is not reasonable. +//func NewTransport(host string, port int) (*memberlist.NetTransport, error) { +func NewTransport(host string, port int) (*Transport, error) { + // memberlist config + conf := memberlist.DefaultLocalConfig() + conf.BindAddr = host + conf.BindPort = port + conf.AdvertisePort = port + + net, err := newTransport(conf) + if err != nil { + return nil, err + } + + uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort())) + if err != nil { + return nil, err + } + + return &Transport{ + Net: net, + URI: uri, + }, nil +} + +// newTransport returns a NetTransport based on the memberlist configuration. +// It will dynamically bind to a port if conf.BindPort is 0. +func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { + if conf.LogOutput != nil && conf.Logger != nil { + return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") + } + + logDest := conf.LogOutput + if logDest == nil { + logDest = os.Stderr + } + + logger := conf.Logger + if logger == nil { + logger = log.New(logDest, "", log.LstdFlags) + } + + nc := &memberlist.NetTransportConfig{ + BindAddrs: []string{conf.BindAddr}, + BindPort: conf.BindPort, + Logger: logger, + } + + // See comment below for details about the retry in here. + makeNetRetry := func(limit int) (*memberlist.NetTransport, error) { + var err error + for try := 0; try < limit; try++ { + var nt *memberlist.NetTransport + if nt, err = memberlist.NewNetTransport(nc); err == nil { + return nt, nil + } + if strings.Contains(err.Error(), "address already in use") { + logger.Printf("[DEBUG] Got bind error: %v", err) + continue + } + } + + return nil, fmt.Errorf("failed to obtain an address: %v", err) + } + + // The dynamic bind port operation is inherently racy because + // even though we are using the kernel to find a port for us, we + // are attempting to bind multiple protocols (and potentially + // multiple addresses) with the same port number. We build in a + // few retries here since this often gets transient errors in + // busy unit tests. + limit := 1 + if conf.BindPort == 0 { + limit = 10 + } + + nt, err := makeNetRetry(limit) + if err != nil { + return nil, fmt.Errorf("Could not set up network transport: %v", err) + } + if conf.BindPort == 0 { + port := nt.GetAutoBindPort() + conf.BindPort = port + conf.AdvertisePort = port + logger.Printf("[DEBUG] Using dynamic bind port %d", port) + } + + return nt, nil +} diff --git a/handler.go b/handler.go index 36e6adb34..f640995a9 100644 --- a/handler.go +++ b/handler.go @@ -124,6 +124,7 @@ func (h *Handler) SetRestricted() { } func loadCommon(router *mux.Router, handler *Handler) { + router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") @@ -175,7 +176,6 @@ func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: diff --git a/holder.go b/holder.go index 99d9d079d..4c3d476d2 100644 --- a/holder.go +++ b/holder.go @@ -88,6 +88,7 @@ func NewHolder() *Holder { // without actually loading any data into memory. // HasData is returned, and h.hasData is set. func (h *Holder) Peek() bool { + h.logger().Printf("peek at holder path: %s", h.Path) h.hasData = false // Open path to read all index directories. @@ -117,6 +118,7 @@ func (h *Holder) Peek() bool { func (h *Holder) Open() error { h.setFileLimit() + h.logger().Printf("open holder path: %s", h.Path) if err := os.MkdirAll(h.Path, 0777); err != nil { return err } @@ -133,7 +135,6 @@ func (h *Holder) Open() error { return err } - h.logger().Printf("Holder Start") for _, fi := range fis { if !fi.IsDir() { continue @@ -157,7 +158,7 @@ func (h *Holder) Open() error { } h.indexes[index.Name()] = index } - h.logger().Printf("Holder Complete") + h.logger().Printf("open holder: complete") // Periodically flush cache. h.wg.Add(1) diff --git a/internal/private.pb.go b/internal/private.pb.go index 0b43b5cc9..d8a216b03 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -824,11 +824,12 @@ func (m *DeleteViewMessage) GetView() string { } type ResizeInstruction struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` - Coordinator *URI `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` - Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` - Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + Coordinator *URI `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` + Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` + ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` } func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } @@ -871,6 +872,13 @@ func (m *ResizeInstruction) GetSchema() *Schema { return nil } +func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { + if m != nil { + return m.ClusterStatus + } + return nil +} + type ResizeSource struct { URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -2130,6 +2138,16 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { } i += n17 } + if m.ClusterStatus != nil { + dAtA[i] = 0x32 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n18 + } return i, nil } @@ -2152,11 +2170,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n18, err := m.URI.MarshalTo(dAtA[i:]) + n19, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n19 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2208,11 +2226,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n19, err := m.URI.MarshalTo(dAtA[i:]) + n20, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n20 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2242,21 +2260,21 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Old.Size())) - n20, err := m.Old.MarshalTo(dAtA[i:]) + n21, err := m.Old.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n21 } if m.New != nil { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n21, err := m.New.MarshalTo(dAtA[i:]) + n22, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n22 } return i, nil } @@ -2783,6 +2801,10 @@ func (m *ResizeInstruction) Size() (n int) { l = m.Schema.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.ClusterStatus != nil { + l = m.ClusterStatus.Size() + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -6611,6 +6633,39 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterStatus", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ClusterStatus == nil { + m.ClusterStatus = &ClusterStatus{} + } + if err := m.ClusterStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7257,80 +7312,81 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1192 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0x23, 0x45, - 0x13, 0x7e, 0xc7, 0x63, 0x3b, 0x76, 0x79, 0xbd, 0xeb, 0xf4, 0x9b, 0x8d, 0x9c, 0x28, 0xf2, 0x9a, - 0x3e, 0x90, 0xb0, 0x12, 0x01, 0x12, 0x09, 0x41, 0x10, 0x12, 0x6c, 0xec, 0xd5, 0x0e, 0x90, 0x64, - 0x69, 0x67, 0x17, 0xc1, 0x01, 0xa9, 0x63, 0x37, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xd3, 0x4e, 0xe2, - 0x3d, 0x70, 0x83, 0x03, 0xfc, 0x01, 0xee, 0xfc, 0x19, 0x8e, 0xdc, 0xb8, 0xa2, 0xf0, 0x23, 0x90, - 0xb8, 0x80, 0xba, 0xa6, 0x7b, 0x66, 0xfc, 0x15, 0x93, 0xdc, 0xa6, 0xaa, 0xab, 0xaa, 0x9f, 0x7e, - 0xea, 0xa3, 0x7b, 0xa0, 0x3a, 0x08, 0xdd, 0x0b, 0x2e, 0xc5, 0xf6, 0x20, 0x0c, 0x64, 0x40, 0x4a, - 0xae, 0x2f, 0x45, 0xe8, 0x73, 0x8f, 0x1e, 0x41, 0xd9, 0xf1, 0x7b, 0xe2, 0xea, 0x40, 0x48, 0x4e, - 0x9a, 0x50, 0xd9, 0x0f, 0xbc, 0x61, 0xdf, 0xff, 0x8c, 0x9f, 0x08, 0xaf, 0x6e, 0x35, 0xad, 0xad, - 0x32, 0xcb, 0xaa, 0x94, 0xc5, 0xb1, 0xdb, 0x17, 0x9f, 0x0f, 0xb9, 0x2f, 0x87, 0xfd, 0x7a, 0x2e, - 0xb6, 0xc8, 0xa8, 0xe8, 0xdf, 0x16, 0x94, 0x9f, 0x86, 0xbc, 0x2f, 0x30, 0xe2, 0x3a, 0x94, 0x58, - 0x70, 0x99, 0x0d, 0x97, 0xc8, 0xe4, 0x75, 0xb8, 0xef, 0xf8, 0x17, 0x22, 0x8c, 0x44, 0xdb, 0xe7, - 0x27, 0x9e, 0xe8, 0x61, 0xb8, 0x12, 0x9b, 0xd0, 0x92, 0x0d, 0x28, 0xef, 0xf3, 0xee, 0x99, 0x38, - 0x1e, 0x0d, 0x44, 0xdd, 0xc6, 0x20, 0xa9, 0x22, 0x59, 0xed, 0xb8, 0xaf, 0x44, 0x3d, 0xdf, 0xb4, - 0xb6, 0xaa, 0x2c, 0x55, 0x4c, 0xe2, 0x2d, 0x4c, 0xe1, 0x25, 0x14, 0xee, 0x31, 0xee, 0x9f, 0x26, - 0x18, 0x8a, 0x88, 0x61, 0x4c, 0x47, 0x36, 0xa1, 0xf8, 0xd4, 0x15, 0x5e, 0x2f, 0xaa, 0x2f, 0x35, - 0xed, 0xad, 0xca, 0xce, 0x83, 0x6d, 0xc3, 0xdf, 0x36, 0xea, 0x99, 0x5e, 0xa6, 0x14, 0xee, 0x3b, - 0xfd, 0x41, 0x10, 0x4a, 0x26, 0xa2, 0x41, 0xe0, 0x47, 0x82, 0xd4, 0xc0, 0x6e, 0x87, 0xa1, 0x3e, - 0xbb, 0xfa, 0xa4, 0xdf, 0x41, 0xed, 0x89, 0x17, 0x74, 0xcf, 0x5b, 0x5c, 0x72, 0x26, 0xbe, 0x1d, - 0x8a, 0x48, 0x92, 0x15, 0x28, 0x60, 0x16, 0xb4, 0x5d, 0x2c, 0x28, 0x2d, 0x32, 0xa9, 0x69, 0x8e, - 0x05, 0xa5, 0x45, 0x7f, 0xa4, 0x22, 0xcf, 0x62, 0x41, 0x69, 0x3b, 0x9e, 0xdb, 0x8d, 0x29, 0xc8, - 0xb3, 0x58, 0x20, 0x04, 0xf2, 0x2f, 0x5d, 0x71, 0xa9, 0xcf, 0x8d, 0xdf, 0xd4, 0x81, 0xe5, 0xcc, - 0xfe, 0x1a, 0xe6, 0x2a, 0x14, 0x59, 0x70, 0xe9, 0xb4, 0xa2, 0xba, 0xd5, 0xb4, 0xb7, 0xf2, 0x4c, - 0x4b, 0xc8, 0x2e, 0xa6, 0x5f, 0x2d, 0xe5, 0x70, 0x29, 0x55, 0xd0, 0x35, 0x28, 0x20, 0xd5, 0xea, - 0x94, 0xa9, 0xaf, 0xfa, 0xa4, 0xff, 0x58, 0x50, 0x3e, 0xe0, 0x57, 0x08, 0x23, 0x22, 0x1f, 0x42, - 0xa9, 0x23, 0xb9, 0xdf, 0xe3, 0x61, 0x0f, 0x8d, 0x2a, 0x3b, 0xaf, 0xa5, 0x14, 0x26, 0x66, 0xdb, - 0xc6, 0xa6, 0xed, 0xcb, 0x70, 0xc4, 0x12, 0x17, 0xb2, 0x07, 0x4b, 0xba, 0x26, 0x10, 0x43, 0x65, - 0xa7, 0x39, 0xcb, 0x3b, 0x29, 0x1b, 0xe5, 0x6c, 0x1c, 0xd6, 0x3f, 0x80, 0xea, 0x58, 0x58, 0x85, - 0xf5, 0x5c, 0x8c, 0x4c, 0x46, 0xce, 0xc5, 0x48, 0x71, 0x77, 0xc1, 0xbd, 0x61, 0xcc, 0x73, 0x9e, - 0xc5, 0xc2, 0x5e, 0xee, 0x3d, 0x6b, 0x7d, 0x0f, 0xee, 0x65, 0xa3, 0xde, 0xc6, 0x97, 0x7e, 0x0d, - 0x64, 0x3f, 0x14, 0x5c, 0x0a, 0x84, 0x77, 0x20, 0xa2, 0x88, 0x9f, 0x8a, 0xf9, 0x99, 0x8e, 0xb3, - 0x97, 0xcb, 0x66, 0x6f, 0x03, 0xca, 0x4e, 0x64, 0x0e, 0x6e, 0x63, 0x5d, 0xa6, 0x0a, 0xfa, 0x18, - 0x48, 0x4b, 0x78, 0x42, 0x0a, 0xdd, 0xbf, 0x37, 0xc4, 0xa7, 0x1d, 0x83, 0x65, 0xb1, 0x2d, 0xd9, - 0x84, 0xbc, 0x6a, 0x5d, 0x84, 0x52, 0xd9, 0xf9, 0x7f, 0xca, 0x74, 0x32, 0x27, 0x18, 0x1a, 0x50, - 0xd7, 0x04, 0xd5, 0xed, 0xbe, 0xe0, 0x80, 0x33, 0x4a, 0xd9, 0x6c, 0x65, 0x4f, 0x6e, 0x95, 0x0c, - 0x10, 0xbd, 0xd5, 0x47, 0xe6, 0xac, 0x77, 0xdd, 0x8a, 0x7e, 0xa5, 0xb5, 0xaa, 0x25, 0x0e, 0xd5, - 0x6a, 0xec, 0x83, 0xdf, 0xf3, 0x8f, 0x3c, 0x81, 0x43, 0xc5, 0x56, 0x3d, 0x14, 0xd5, 0xed, 0xa6, - 0xad, 0x62, 0xa3, 0x40, 0x77, 0xa1, 0xd8, 0xe9, 0x9e, 0x89, 0x3e, 0x27, 0x6f, 0xa8, 0x42, 0xed, - 0x89, 0x2b, 0x11, 0xe9, 0x32, 0x7f, 0x30, 0x41, 0x1f, 0x33, 0xeb, 0xf4, 0x27, 0x4b, 0xa3, 0x9f, - 0x83, 0xa8, 0x88, 0x7b, 0x47, 0xf5, 0xfc, 0xd4, 0xc4, 0x51, 0x7a, 0xa6, 0x97, 0x49, 0x1b, 0x6a, - 0x8e, 0x3f, 0x18, 0xca, 0x96, 0xf8, 0xc6, 0xf5, 0x5d, 0xe9, 0x06, 0x7e, 0x54, 0x2f, 0xa2, 0xcb, - 0x5a, 0x76, 0xeb, 0x31, 0x0b, 0x36, 0xe5, 0x42, 0x7f, 0xb0, 0xe0, 0xc1, 0x84, 0x72, 0x01, 0xae, - 0xdc, 0xcd, 0xb8, 0xde, 0x4d, 0x46, 0xa6, 0x8d, 0x86, 0x8d, 0xb9, 0x68, 0xc6, 0x27, 0xe8, 0x2f, - 0x16, 0xac, 0xcc, 0x32, 0x98, 0x89, 0xa6, 0x01, 0xf0, 0x3c, 0x74, 0xfb, 0x3c, 0x1c, 0x7d, 0x2a, - 0x46, 0xfa, 0xf6, 0xc8, 0x68, 0xc8, 0x17, 0xb0, 0x3a, 0x11, 0xeb, 0xe3, 0x6e, 0x4c, 0x51, 0x0c, - 0xea, 0xd1, 0x5c, 0x50, 0xb1, 0x1d, 0x9b, 0xe3, 0x4e, 0xff, 0xb2, 0xe0, 0xe1, 0xcc, 0xa5, 0xb4, - 0xfa, 0xac, 0x6c, 0xa1, 0x3f, 0x86, 0xda, 0x4b, 0x35, 0x18, 0x5a, 0x22, 0x92, 0xae, 0xcf, 0x95, - 0xa5, 0x2e, 0xcf, 0x29, 0x3d, 0x71, 0xa0, 0x84, 0xba, 0x03, 0x3e, 0xd0, 0x30, 0xdf, 0x5c, 0x00, - 0x73, 0xdb, 0xd8, 0xeb, 0xb9, 0x69, 0x44, 0x05, 0x06, 0xe7, 0xb8, 0xb9, 0x14, 0x50, 0x50, 0x13, - 0x71, 0xcc, 0xe1, 0x56, 0x53, 0x2d, 0x80, 0x0d, 0x33, 0x49, 0xc6, 0x90, 0xdc, 0xdc, 0x93, 0xef, - 0x03, 0xa4, 0xa6, 0xba, 0xdd, 0x6f, 0xa8, 0xcf, 0x8c, 0x31, 0x7d, 0x06, 0x1b, 0x66, 0xcc, 0xdd, - 0x62, 0x43, 0x53, 0x2d, 0xb9, 0xb4, 0x5a, 0x68, 0x1b, 0xec, 0x17, 0xcc, 0x51, 0x57, 0x1d, 0x76, - 0xab, 0x49, 0x91, 0x96, 0x94, 0xcb, 0xb3, 0x20, 0x92, 0xc6, 0x45, 0x7d, 0x2b, 0xdd, 0xf3, 0x20, - 0x94, 0x88, 0xb8, 0xca, 0xf0, 0x9b, 0x3a, 0x50, 0x3b, 0x0c, 0x7a, 0xa2, 0x23, 0xb9, 0x4c, 0x26, - 0xd1, 0x23, 0x0c, 0x8d, 0x01, 0x2b, 0x3b, 0xd5, 0xf4, 0x60, 0x2f, 0x98, 0xc3, 0x70, 0x53, 0x35, - 0xe0, 0x95, 0x83, 0x19, 0x4a, 0x28, 0xd0, 0x1f, 0x2d, 0x00, 0x13, 0x6b, 0x18, 0x2d, 0x8e, 0xf2, - 0x4e, 0xe6, 0x4e, 0x9d, 0x1e, 0x56, 0xc9, 0x12, 0xcb, 0xdc, 0xbc, 0x5b, 0x66, 0x36, 0x69, 0xd6, - 0x6b, 0xa9, 0x7d, 0xac, 0xd7, 0xe7, 0xe7, 0xf4, 0x10, 0xaa, 0xfb, 0xde, 0x30, 0x92, 0x22, 0xd4, - 0x70, 0x12, 0xcc, 0x56, 0x06, 0x33, 0xd9, 0x84, 0x25, 0x84, 0x2c, 0xa4, 0x1e, 0x01, 0x13, 0x40, - 0xcd, 0x2a, 0xed, 0x40, 0x61, 0x7e, 0xe7, 0x12, 0xc8, 0xe3, 0x73, 0x4e, 0x93, 0x8d, 0x2f, 0xb9, - 0x1a, 0xd8, 0x07, 0x6e, 0x5c, 0x1d, 0x36, 0x53, 0x9f, 0xa8, 0xe1, 0x57, 0x58, 0xbd, 0x4a, 0xc3, - 0xd5, 0x45, 0xb6, 0x1c, 0x57, 0x83, 0x9a, 0xbc, 0x77, 0xb9, 0x72, 0xcc, 0x8b, 0xc8, 0xce, 0xbc, - 0x88, 0x7e, 0xb7, 0x60, 0x99, 0x89, 0xc8, 0x7d, 0x25, 0x1c, 0x3f, 0x92, 0xe1, 0x30, 0xe9, 0xe4, - 0x4f, 0x82, 0x13, 0xa7, 0x85, 0x51, 0x6d, 0x16, 0x0b, 0x26, 0x47, 0xb9, 0xb9, 0x39, 0x7a, 0x4b, - 0xbd, 0xa1, 0x83, 0xb0, 0xa7, 0xda, 0x39, 0x08, 0x35, 0xeb, 0x13, 0x86, 0x59, 0x0b, 0xf2, 0x36, - 0x2c, 0x75, 0x82, 0x61, 0xd8, 0x4d, 0x66, 0xfd, 0x6a, 0x6a, 0x1c, 0xa3, 0x8a, 0x97, 0x99, 0x31, - 0xcb, 0xe4, 0xb4, 0xb0, 0x20, 0xa7, 0xdf, 0x5b, 0x70, 0x2f, 0x1b, 0xe3, 0x3f, 0x15, 0x6a, 0xcc, - 0x65, 0x6e, 0x26, 0x97, 0xf6, 0x2c, 0x2e, 0xf3, 0x29, 0x97, 0xe9, 0x4b, 0xa6, 0x90, 0x79, 0xc9, - 0xd0, 0x33, 0x58, 0x9b, 0x22, 0x78, 0x3f, 0xe8, 0x0f, 0x54, 0x26, 0xef, 0x4a, 0xf4, 0x0a, 0x14, - 0xda, 0x61, 0xa8, 0x29, 0x2e, 0xb3, 0x58, 0xa0, 0x5f, 0xc2, 0xc3, 0x8e, 0x90, 0x19, 0x7e, 0x33, - 0x2d, 0x7a, 0xe4, 0xf5, 0xe6, 0x9c, 0xfc, 0xc8, 0xeb, 0x29, 0x83, 0x43, 0x71, 0x39, 0x67, 0xc3, - 0x43, 0x71, 0x49, 0x77, 0xa1, 0x74, 0x1c, 0x0c, 0x02, 0x2f, 0x38, 0x1d, 0x65, 0xbb, 0xc0, 0xba, - 0xa9, 0x0b, 0x9e, 0xd4, 0x7e, 0xbd, 0x6e, 0x58, 0xbf, 0x5d, 0x37, 0xac, 0x3f, 0xae, 0x1b, 0xd6, - 0xcf, 0x7f, 0x36, 0xfe, 0x77, 0x52, 0xc4, 0x5f, 0xb0, 0xdd, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, - 0x50, 0x2b, 0xa7, 0xda, 0x93, 0x0d, 0x00, 0x00, + // 1208 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcf, 0x6f, 0x1b, 0xc5, + 0x17, 0xff, 0xae, 0xd7, 0x76, 0xec, 0xe7, 0xba, 0x75, 0xe7, 0xdb, 0x16, 0x27, 0x8a, 0x5c, 0x33, + 0x07, 0x12, 0x2a, 0x11, 0xc0, 0x91, 0x10, 0x04, 0x55, 0x82, 0xc6, 0xae, 0xba, 0x40, 0x92, 0x32, + 0x4e, 0x8b, 0xe0, 0x80, 0x34, 0xb1, 0x87, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x1d, 0x27, 0x71, 0x0f, + 0xdc, 0xe0, 0x00, 0x77, 0xc4, 0x9d, 0x7f, 0x86, 0x23, 0x7f, 0x02, 0x0a, 0x7f, 0x04, 0x12, 0x17, + 0xd0, 0xbc, 0x9d, 0xd9, 0x5d, 0xff, 0x0c, 0xc9, 0x6d, 0xdf, 0x9b, 0xcf, 0x7b, 0xf3, 0x99, 0xf7, + 0x6b, 0x66, 0xa1, 0x3a, 0x0c, 0xdd, 0x33, 0x2e, 0xc5, 0xd6, 0x30, 0x0c, 0x64, 0x40, 0x4a, 0xae, + 0x2f, 0x45, 0xe8, 0x73, 0x8f, 0x1e, 0x40, 0xd9, 0xf1, 0xfb, 0xe2, 0x62, 0x4f, 0x48, 0x4e, 0x9a, + 0x50, 0xd9, 0x0d, 0xbc, 0xd1, 0xc0, 0xff, 0x8c, 0x1f, 0x09, 0xaf, 0x6e, 0x35, 0xad, 0xcd, 0x32, + 0xcb, 0xaa, 0x14, 0xe2, 0xd0, 0x1d, 0x88, 0xcf, 0x47, 0xdc, 0x97, 0xa3, 0x41, 0x3d, 0x17, 0x23, + 0x32, 0x2a, 0xfa, 0xb7, 0x05, 0xe5, 0xa7, 0x21, 0x1f, 0x08, 0xf4, 0xb8, 0x06, 0x25, 0x16, 0x9c, + 0x67, 0xdd, 0x25, 0x32, 0x79, 0x03, 0x6e, 0x3b, 0xfe, 0x99, 0x08, 0x23, 0xd1, 0xf1, 0xf9, 0x91, + 0x27, 0xfa, 0xe8, 0xae, 0xc4, 0xa6, 0xb4, 0x64, 0x1d, 0xca, 0xbb, 0xbc, 0x77, 0x22, 0x0e, 0xc7, + 0x43, 0x51, 0xb7, 0xd1, 0x49, 0xaa, 0x48, 0x56, 0xbb, 0xee, 0x2b, 0x51, 0xcf, 0x37, 0xad, 0xcd, + 0x2a, 0x4b, 0x15, 0xd3, 0x7c, 0x0b, 0x33, 0x7c, 0x09, 0x85, 0x5b, 0x8c, 0xfb, 0xc7, 0x09, 0x87, + 0x22, 0x72, 0x98, 0xd0, 0x91, 0x0d, 0x28, 0x3e, 0x75, 0x85, 0xd7, 0x8f, 0xea, 0x2b, 0x4d, 0x7b, + 0xb3, 0xd2, 0xba, 0xb3, 0x65, 0xe2, 0xb7, 0x85, 0x7a, 0xa6, 0x97, 0x29, 0x85, 0xdb, 0xce, 0x60, + 0x18, 0x84, 0x92, 0x89, 0x68, 0x18, 0xf8, 0x91, 0x20, 0x35, 0xb0, 0x3b, 0x61, 0xa8, 0xcf, 0xae, + 0x3e, 0xe9, 0x77, 0x50, 0x7b, 0xe2, 0x05, 0xbd, 0xd3, 0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, + 0x92, 0xe4, 0x1e, 0x14, 0x30, 0x0b, 0x1a, 0x17, 0x0b, 0x4a, 0x8b, 0x91, 0xd4, 0x61, 0x8e, 0x05, + 0xa5, 0x45, 0x7b, 0x0c, 0x45, 0x9e, 0xc5, 0x82, 0xd2, 0x76, 0x3d, 0xb7, 0x17, 0x87, 0x20, 0xcf, + 0x62, 0x81, 0x10, 0xc8, 0xbf, 0x74, 0xc5, 0xb9, 0x3e, 0x37, 0x7e, 0x53, 0x07, 0xee, 0x66, 0xf6, + 0xd7, 0x34, 0x1f, 0x40, 0x91, 0x05, 0xe7, 0x4e, 0x3b, 0xaa, 0x5b, 0x4d, 0x7b, 0x33, 0xcf, 0xb4, + 0x84, 0xd1, 0xc5, 0xf4, 0xab, 0xa5, 0x1c, 0x2e, 0xa5, 0x0a, 0xba, 0x0a, 0x05, 0x0c, 0xb5, 0x3a, + 0x65, 0x6a, 0xab, 0x3e, 0xe9, 0x3f, 0x16, 0x94, 0xf7, 0xf8, 0x05, 0xd2, 0x88, 0xc8, 0x63, 0x28, + 0x75, 0x25, 0xf7, 0xfb, 0x3c, 0xec, 0x23, 0xa8, 0xd2, 0x7a, 0x3d, 0x0d, 0x61, 0x02, 0xdb, 0x32, + 0x98, 0x8e, 0x2f, 0xc3, 0x31, 0x4b, 0x4c, 0xc8, 0x0e, 0xac, 0xe8, 0x9a, 0x40, 0x0e, 0x95, 0x56, + 0x73, 0x9e, 0x75, 0x52, 0x36, 0xca, 0xd8, 0x18, 0xac, 0x7d, 0x08, 0xd5, 0x09, 0xb7, 0x8a, 0xeb, + 0xa9, 0x18, 0x9b, 0x8c, 0x9c, 0x8a, 0xb1, 0x8a, 0xdd, 0x19, 0xf7, 0x46, 0x71, 0x9c, 0xf3, 0x2c, + 0x16, 0x76, 0x72, 0xef, 0x5b, 0x6b, 0x3b, 0x70, 0x2b, 0xeb, 0xf5, 0x3a, 0xb6, 0xf4, 0x6b, 0x20, + 0xbb, 0xa1, 0xe0, 0x52, 0x20, 0xbd, 0x3d, 0x11, 0x45, 0xfc, 0x58, 0x2c, 0xce, 0x74, 0x9c, 0xbd, + 0x5c, 0x36, 0x7b, 0xeb, 0x50, 0x76, 0x22, 0x73, 0x70, 0x1b, 0xeb, 0x32, 0x55, 0xd0, 0x47, 0x40, + 0xda, 0xc2, 0x13, 0x52, 0xe8, 0xfe, 0x5d, 0xe2, 0x9f, 0x76, 0x0d, 0x97, 0xab, 0xb1, 0x64, 0x03, + 0xf2, 0xaa, 0x75, 0x91, 0x4a, 0xa5, 0xf5, 0xff, 0x34, 0xd2, 0xc9, 0x9c, 0x60, 0x08, 0xa0, 0xae, + 0x71, 0xaa, 0xdb, 0xfd, 0x8a, 0x03, 0xce, 0x29, 0x65, 0xb3, 0x95, 0x3d, 0xbd, 0x55, 0x32, 0x40, + 0xf4, 0x56, 0x1f, 0x99, 0xb3, 0xde, 0x74, 0x2b, 0xfa, 0x95, 0xd6, 0xaa, 0x96, 0xd8, 0x57, 0xab, + 0xb1, 0x0d, 0x7e, 0x2f, 0x3e, 0xf2, 0x14, 0x0f, 0xe5, 0x5b, 0xf5, 0x50, 0x54, 0xb7, 0x9b, 0xb6, + 0xf2, 0x8d, 0x02, 0xdd, 0x86, 0x62, 0xb7, 0x77, 0x22, 0x06, 0x9c, 0xbc, 0xa9, 0x0a, 0xb5, 0x2f, + 0x2e, 0x44, 0xa4, 0xcb, 0xfc, 0xce, 0x54, 0xf8, 0x98, 0x59, 0xa7, 0x3f, 0x59, 0x9a, 0xfd, 0x02, + 0x46, 0x45, 0xdc, 0x3b, 0xaa, 0xe7, 0x67, 0x26, 0x8e, 0xd2, 0x33, 0xbd, 0x4c, 0x3a, 0x50, 0x73, + 0xfc, 0xe1, 0x48, 0xb6, 0xc5, 0x37, 0xae, 0xef, 0x4a, 0x37, 0xf0, 0xa3, 0x7a, 0x11, 0x4d, 0x56, + 0xb3, 0x5b, 0x4f, 0x20, 0xd8, 0x8c, 0x09, 0xfd, 0xc1, 0x82, 0x3b, 0x53, 0xca, 0x2b, 0x78, 0xe5, + 0x96, 0xf3, 0x7a, 0x2f, 0x19, 0x99, 0x36, 0x02, 0x1b, 0x0b, 0xd9, 0x4c, 0x4e, 0xd0, 0x5f, 0x2d, + 0xb8, 0x37, 0x0f, 0x30, 0x97, 0x4d, 0x03, 0xe0, 0x79, 0xe8, 0x0e, 0x78, 0x38, 0xfe, 0x54, 0x8c, + 0xf5, 0xed, 0x91, 0xd1, 0x90, 0x2f, 0xe0, 0xc1, 0x94, 0xaf, 0x8f, 0x7b, 0x71, 0x88, 0x62, 0x52, + 0x0f, 0x17, 0x92, 0x8a, 0x71, 0x6c, 0x81, 0x39, 0xfd, 0xcb, 0x82, 0xfb, 0x73, 0x97, 0xd2, 0xea, + 0xb3, 0xb2, 0x85, 0xfe, 0x08, 0x6a, 0x2f, 0xd5, 0x60, 0x68, 0x8b, 0x48, 0xba, 0x3e, 0x57, 0x48, + 0x5d, 0x9e, 0x33, 0x7a, 0xe2, 0x40, 0x09, 0x75, 0x7b, 0x7c, 0xa8, 0x69, 0xbe, 0x75, 0x05, 0xcd, + 0x2d, 0x83, 0xd7, 0x73, 0xd3, 0x88, 0x8a, 0x0c, 0xce, 0x71, 0x73, 0x29, 0xa0, 0xa0, 0x26, 0xe2, + 0x84, 0xc1, 0xb5, 0xa6, 0x5a, 0x00, 0xeb, 0x66, 0x92, 0x4c, 0x30, 0x59, 0xde, 0x93, 0x1f, 0x00, + 0xa4, 0x50, 0xdd, 0xee, 0x4b, 0xea, 0x33, 0x03, 0xa6, 0xcf, 0x60, 0xdd, 0x8c, 0xb9, 0x6b, 0x6c, + 0x68, 0xaa, 0x25, 0x97, 0x56, 0x0b, 0xed, 0x80, 0xfd, 0x82, 0x39, 0xea, 0xaa, 0xc3, 0x6e, 0x35, + 0x29, 0xd2, 0x92, 0x32, 0x79, 0x16, 0x44, 0xd2, 0x98, 0xa8, 0x6f, 0xa5, 0x7b, 0x1e, 0x84, 0x12, + 0x19, 0x57, 0x19, 0x7e, 0x53, 0x07, 0x6a, 0xfb, 0x41, 0x5f, 0x74, 0x25, 0x97, 0xc9, 0x24, 0x7a, + 0x88, 0xae, 0xd1, 0x61, 0xa5, 0x55, 0x4d, 0x0f, 0xf6, 0x82, 0x39, 0x0c, 0x37, 0x55, 0x03, 0x5e, + 0x19, 0x98, 0xa1, 0x84, 0x02, 0xfd, 0xd1, 0x02, 0x30, 0xbe, 0x46, 0xd1, 0xd5, 0x5e, 0xde, 0xcd, + 0xdc, 0xa9, 0xb3, 0xc3, 0x2a, 0x59, 0x62, 0x99, 0x9b, 0x77, 0xd3, 0xcc, 0x26, 0x1d, 0xf5, 0x5a, + 0x8a, 0x8f, 0xf5, 0xfa, 0xfc, 0x9c, 0xee, 0x43, 0x75, 0xd7, 0x1b, 0x45, 0x52, 0x84, 0x9a, 0x4e, + 0xc2, 0xd9, 0xca, 0x70, 0x26, 0x1b, 0xb0, 0x82, 0x94, 0x85, 0xd4, 0x23, 0x60, 0x8a, 0xa8, 0x59, + 0xa5, 0x5d, 0x28, 0x2c, 0xee, 0x5c, 0x02, 0x79, 0x7c, 0xce, 0xe9, 0x60, 0xe3, 0x4b, 0xae, 0x06, + 0xf6, 0x9e, 0x1b, 0x57, 0x87, 0xcd, 0xd4, 0x27, 0x6a, 0xf8, 0x05, 0x56, 0xaf, 0xd2, 0x70, 0x75, + 0x91, 0xdd, 0x8d, 0xab, 0x41, 0x4d, 0xde, 0x9b, 0x5c, 0x39, 0xe6, 0x45, 0x64, 0x67, 0x5e, 0x44, + 0x3f, 0xe7, 0xe0, 0x2e, 0x13, 0x91, 0xfb, 0x4a, 0x38, 0x7e, 0x24, 0xc3, 0x51, 0xd2, 0xc9, 0x9f, + 0x04, 0x47, 0x4e, 0x1b, 0xbd, 0xda, 0x2c, 0x16, 0x4c, 0x8e, 0x72, 0x0b, 0x73, 0xf4, 0xb6, 0x7a, + 0x43, 0x07, 0x61, 0x5f, 0xb5, 0x73, 0x10, 0xea, 0xa8, 0x4f, 0x01, 0xb3, 0x08, 0xf2, 0x0e, 0xac, + 0x74, 0x83, 0x51, 0xd8, 0x4b, 0x66, 0xfd, 0x83, 0x14, 0x1c, 0xb3, 0x8a, 0x97, 0x99, 0x81, 0x65, + 0x72, 0x5a, 0x58, 0x9e, 0x53, 0xf2, 0x78, 0x2a, 0xa7, 0xf8, 0xba, 0xad, 0xb4, 0x5e, 0x4b, 0x0d, + 0x26, 0x96, 0xd9, 0x24, 0x9a, 0x7e, 0x6f, 0xc1, 0xad, 0x2c, 0x85, 0xff, 0x54, 0xe7, 0x71, 0x2a, + 0x72, 0x73, 0x53, 0x61, 0xcf, 0x4b, 0x45, 0x3e, 0x4d, 0x45, 0xfa, 0x10, 0x2a, 0x64, 0x1e, 0x42, + 0xf4, 0x04, 0x56, 0x67, 0xf2, 0xb3, 0x1b, 0x0c, 0x86, 0xaa, 0x10, 0x6e, 0x9a, 0xa7, 0x7b, 0x50, + 0xe8, 0x84, 0xa1, 0xce, 0x50, 0x99, 0xc5, 0x02, 0xfd, 0x12, 0xee, 0x77, 0x85, 0xcc, 0xa4, 0x27, + 0xd3, 0xe1, 0x07, 0x5e, 0x7f, 0xc1, 0xc9, 0x0f, 0xbc, 0xbe, 0x02, 0xec, 0x8b, 0xf3, 0x05, 0x1b, + 0xee, 0x8b, 0x73, 0xba, 0x0d, 0xa5, 0xc3, 0x60, 0x18, 0x78, 0xc1, 0xf1, 0x38, 0xdb, 0x44, 0xd6, + 0xb2, 0x26, 0x7a, 0x52, 0xfb, 0xed, 0xb2, 0x61, 0xfd, 0x7e, 0xd9, 0xb0, 0xfe, 0xb8, 0x6c, 0x58, + 0xbf, 0xfc, 0xd9, 0xf8, 0xdf, 0x51, 0x11, 0xff, 0xe0, 0xb6, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, + 0xee, 0xaa, 0xfe, 0xbd, 0xd2, 0x0d, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index bb1cadfea..66bacfd97 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -156,6 +156,7 @@ message ResizeInstruction { URI Coordinator = 3; repeated ResizeSource Sources = 4; Schema Schema = 5; + ClusterStatus ClusterStatus = 6; } message ResizeSource { diff --git a/server.go b/server.go index 94c91f2ae..1d03bd78b 100644 --- a/server.go +++ b/server.go @@ -59,6 +59,9 @@ type Server struct { wg sync.WaitGroup closing chan struct{} + // Unique name identifying the server. + Name string + // Data storage and HTTP interface. Holder *Holder Handler *Handler @@ -117,31 +120,12 @@ func NewServer() *Server { // Open opens and initializes the server. func (s *Server) Open() error { - var ln net.Listener - var err error - - // If bind URI has the https scheme, enable TLS - if s.URI.Scheme() == "https" && s.TLS != nil { - ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS) - if err != nil { + s.Logger().Printf("open server") + // s.ln can be configured prior to Open() via s.OpenListener(). + if s.ln == nil { + if err := s.OpenListener(); err != nil { return err } - } else if s.URI.Scheme() == "http" { - // Open HTTP listener to determine port (if specified as :0). - ln, err = net.Listen(s.Network, s.URI.HostPort()) - if err != nil { - return fmt.Errorf("net.Listen: %v", err) - } - } else { - return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme()) - } - - s.ln = ln - - if s.URI.Port() == 0 { - // If the port is 0, it is set automatically. - // Find out automatically set port and update the host. - s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) } // Set Cluster URI. @@ -170,6 +154,9 @@ func (s *Server) Open() error { e.URI = s.URI e.Cluster = s.Cluster e.MaxWritesPerRequest = s.MaxWritesPerRequest + + // Cluster settings. + s.Cluster.Broadcaster = s.Broadcaster s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest // Initialize HTTP handler. @@ -188,7 +175,7 @@ func (s *Server) Open() error { // Serve HTTP. go func() { - err := http.Serve(ln, s.Handler) + err := http.Serve(s.ln, s.Handler) if err != nil { s.Logger().Printf("HTTP handler terminated with error: %s\n", err) } @@ -199,6 +186,11 @@ func (s *Server) Open() error { return fmt.Errorf("starting BroadcastReceiver: %v", err) } + // If a Coordinator is not specified, then default to s.URI. + if s.Cluster.Coordinator.Port() == 0 { + s.Cluster.Coordinator = s.URI + } + // Open Cluster management. if err := s.Cluster.Open(); err != nil { return fmt.Errorf("opening Cluster: %v", err) @@ -228,6 +220,48 @@ func (s *Server) Open() error { return nil } +// OpenListener opens a listener for the Server. +func (s *Server) OpenListener() error { + s.Logger().Printf("open server listener: %s", s.URI) + if s.ln != nil { + return fmt.Errorf("a listener already exists for server: %s", s.URI) + } + + var ln net.Listener + var err error + + // If bind URI has the https scheme, enable TLS + if s.URI.Scheme() == "https" && s.TLS != nil { + ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS) + if err != nil { + return err + } + } else if s.URI.Scheme() == "http" { + // Open HTTP listener to determine port (if specified as :0). + ln, err = net.Listen(s.Network, s.URI.HostPort()) + if err != nil { + return fmt.Errorf("net.Listen: %v", err) + } + } else { + return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme()) + } + + s.ln = ln + + if s.URI.Port() == 0 { + // If the port is 0, it is set automatically. + // Find out automatically set port and update the host. + s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) + } + + // If name is not provided in the config, default to the URI. + if s.Name == "" { + s.Name = s.URI.String() + } + + return nil +} + // Close closes the server and waits for it to shutdown. func (s *Server) Close() error { // Notify goroutines to stop. @@ -382,7 +416,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } - s.Cluster.MarkAsJoined() case *internal.ResizeInstruction: err := s.Cluster.FollowResizeInstruction(obj) if err != nil { @@ -409,6 +442,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { + s.Logger().Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.URI == node.URI { continue @@ -430,7 +464,8 @@ func (s *Server) SendAsync(pb proto.Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, pb proto.Message) error { - ctx := context.WithValue(context.Background(), "uri", to.URI) + s.Logger().Printf("SendTo: %s", to.URI) + ctx := context.WithValue(context.Background(), "uri", &to.URI) return s.defaultClient.SendMessage(ctx, pb) } diff --git a/server/cluster_test.go b/server/cluster_test.go new file mode 100644 index 000000000..bc7b1cd91 --- /dev/null +++ b/server/cluster_test.go @@ -0,0 +1,438 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server_test + +import ( + "context" + "reflect" + "testing" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gossip" +) + +// Ensure program can send/receive broadcast messages. +func TestMain_XSendReceiveMessage(t *testing.T) { + + m0 := MustRunMain() + defer m0.Close() + + m1 := MustRunMain() + defer m1.Close() + + // Update cluster config + m0.Server.Cluster.Nodes = []*pilosa.Node{ + {URI: m0.Server.URI}, + {URI: m1.Server.URI}, + } + m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes + + // Configure node0 + + // get the host portion of addr to use for binding + gossipHost := m0.Server.URI.Host() + gossipPort := 0 + gossipSeed := "" + + m0.Server.Cluster.Coordinator = m0.Server.URI + m0.Server.Cluster.Topology = &pilosa.Topology{NodeSet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} + m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m0.Server, nil) + if err != nil { + t.Fatal(err) + } + m0.Server.Cluster.MemberSet = gossipMemberSet0 + m0.Server.Broadcaster = m0.Server + m0.Server.Gossiper = gossipMemberSet0 + m0.Server.Handler.Broadcaster = m0.Server.Broadcaster + m0.Server.Holder.Broadcaster = m0.Server.Broadcaster + m0.Server.BroadcastReceiver = gossipMemberSet0 + + if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { + t.Fatal(err) + } + // Open Cluster management. + if err := m0.Server.Cluster.Open(); err != nil { + t.Fatal(err) + } + + // Configure node1 + + // get the host portion of addr to use for binding + gossipHost = m1.Server.URI.Host() + gossipPort = 0 + gossipSeed = gossipMemberSet0.Seed() + + m1.Server.Cluster.Coordinator = m0.Server.URI + m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) + if err != nil { + t.Fatal(err) + } + m1.Server.Cluster.MemberSet = gossipMemberSet1 + m1.Server.Broadcaster = m1.Server + m1.Server.Gossiper = gossipMemberSet1 + m1.Server.Handler.Broadcaster = m1.Server.Broadcaster + m1.Server.Holder.Broadcaster = m1.Server.Broadcaster + m1.Server.BroadcastReceiver = gossipMemberSet1 + + if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { + t.Fatal(err) + } + // Open Cluster management. + if err := m1.Server.Cluster.Open(); err != nil { + t.Fatal(err) + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + // Expected indexes and Frames + expected := map[string][]string{ + "i": []string{"f"}, + } + + // Create a client for each node. + client0 := m0.Client() + client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Make sure node0 knows about the index and frame created. + schema0, err := client0.Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + received0 := map[string][]string{} + for _, idx := range schema0 { + received0[idx.Name] = []string{} + for _, frame := range idx.Frames { + received0[idx.Name] = append(received0[idx.Name], frame.Name) + } + } + if !reflect.DeepEqual(received0, expected) { + t.Fatalf("unexpected schema on node0: %s", received0) + } + + // Make sure node1 knows about the index and frame created. + schema1, err := client1.Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + received1 := map[string][]string{} + for _, idx := range schema1 { + received1[idx.Name] = []string{} + for _, frame := range idx.Frames { + received1[idx.Name] = append(received1[idx.Name], frame.Name) + } + } + if !reflect.DeepEqual(received1, expected) { + t.Fatalf("unexpected schema on node1: %s", received1) + } + + // Write data on first node. + if _, err := m0.Query("i", "", ` + SetBit(rowID=1, frame="f", columnID=1) + SetBit(rowID=1, frame="f", columnID=2400000) + `); err != nil { + t.Fatal(err) + } + + // We have to wait for the broadcast message to be sent before checking state. + time.Sleep(1 * time.Second) + + // Make sure node0 knows about the latest MaxSlice. + maxSlices0, err := client0.MaxSliceByIndex(context.Background()) + if err != nil { + t.Fatal(err) + } + if maxSlices0["i"] != 2 { + t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) + } + + // Make sure node1 knows about the latest MaxSlice. + maxSlices1, err := client1.MaxSliceByIndex(context.Background()) + if err != nil { + t.Fatal(err) + } + if maxSlices1["i"] != 2 { + t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) + } + + // Write input definition to the first node. + if _, err := m0.CreateDefinition("i", "test", `{ + "frames": [{"name": "event-time", + "options": { + "cacheType": "ranked", + "timeQuantum": "YMD" + }}], + "fields": [{"name": "columnID", + "primaryKey": true + }]} + `); err != nil { + t.Fatal(err) + } + + // We have to wait for the broadcast message to be sent before checking state. + time.Sleep(1 * time.Second) + + frame0 := m0.Server.Holder.Frame("i", "event-time") + if frame0 == nil { + t.Fatal("frame not found") + } + frame1 := m1.Server.Holder.Frame("i", "event-time") + if frame1 == nil { + t.Fatal("frame not found") + } +} + +// Ensure that an empty node comes up in a NORMAL state. +func TestClusterResize_EmptyNode(t *testing.T) { + m0 := MustRunMain() + defer m0.Close() + + if m0.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State) + } +} + +// Ensure that a cluster of empty nodes comes up in a NORMAL state. +func TestClusterResize_EmptyNodes(t *testing.T) { + // Configure node0 + m0 := NewMain() + defer m0.Close() + + gossipHost := "localhost" + gossipPort := 0 + seed, coord, err := m0.RunWithTransport(gossipHost, gossipPort, "", nil) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := NewMain() + defer m1.Close() + + seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, seed, &coord) + if err != nil { + t.Fatal(err) + } + + if m0.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) + } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + } +} + +// Ensure that adding a node correctly resizes the cluster. +func TestClusterResize_AddNode(t *testing.T) { + t.Run("NoData", func(t *testing.T) { + // Configure node0 + m0 := NewMain() + defer m0.Close() + + seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := NewMain() + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) + } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + } + }) + t.Run("WithIndex", func(t *testing.T) { + // Configure node0 + m0 := NewMain() + defer m0.Close() + + seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + if err != nil { + t.Fatal(err) + } + + // Create a client for each node. + client0 := m0.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := NewMain() + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) + } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + } + }) + t.Run("ContinuousSlices", func(t *testing.T) { + + // Configure node0 + m0 := NewMain() + defer m0.Close() + + seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + if err != nil { + t.Fatal(err) + } + + // Create a client for each node. + client0 := m0.Client() + //client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Write data on first node. + if _, err := m0.Query("i", "", ` + SetBit(rowID=1, frame="f", columnID=1) + SetBit(rowID=1, frame="f", columnID=1300000) + `); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := NewMain() + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) + } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + } + }) + t.Run("SkippedSlice", func(t *testing.T) { + + // Configure node0 + m0 := NewMain() + defer m0.Close() + + seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + if err != nil { + t.Fatal(err) + } + + // Create a client for each node. + client0 := m0.Client() + //client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Write data on first node. Note that no data is placed on slice 1. + if _, err := m0.Query("i", "", ` + SetBit(rowID=1, frame="f", columnID=1) + SetBit(rowID=1, frame="f", columnID=2400000) + `); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := NewMain() + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) + } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + } + }) +} diff --git a/server/server.go b/server/server.go index dbc85e87a..045079229 100644 --- a/server/server.go +++ b/server/server.go @@ -60,6 +60,9 @@ type Command struct { CPUProfile string CPUTime time.Duration + // Gossip transport + GossipTransport *gossip.Transport + // Standard input/output *pilosa.CmdIO @@ -100,6 +103,12 @@ func (m *Command) Run(args ...string) (err error) { return err } + // SetupNetworking + err = m.SetupNetworking() + if err != nil { + return err + } + // Initialize server. if err = m.Server.Open(); err != nil { return fmt.Errorf("server.Open: %v", err) @@ -123,6 +132,11 @@ func (m *Command) SetupServer() error { } m.Server.URI = *uri + // If using a dynamically allocated port, server.Name will get set later. + if m.Config.Bind != "localhost:0" { + m.Server.Name = m.Server.URI.String() + } + cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN cluster.Holder = m.Server.Holder @@ -183,24 +197,41 @@ func (m *Command) SetupServer() error { m.Server.Handler.RemoteClient = c m.Server.Cluster.RemoteClient = c + // Default coordintor to port 0 when not specified so that coordinator + // can be set to the value of server.URI after server binds to a port. + // This would only be useful in a one-node cluster. + coord := m.Config.Cluster.Coordinator + if coord == "" { + coord = ":0" + } + // Set the coordinator node. - curi, err := pilosa.AddressWithDefaults(m.Config.Cluster.Coordinator) + curi, err := pilosa.AddressWithDefaults(coord) if err != nil { return err } m.Server.Cluster.Coordinator = *curi - // Set internal port (string). - gossipPortStr := pilosa.DefaultGossipPort - // Config.GossipPort is deprecated, so Config.Gossip.Port has priority - if m.Config.Gossip.Port != "" { - gossipPortStr = m.Config.Gossip.Port - } else if m.Config.GossipPort != "" { - gossipPortStr = m.Config.GossipPort - } + // Set configuration options. + m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) + m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) + return nil +} +// SetupNetworking sets up internode communication based on the configuration. +func (m *Command) SetupNetworking() error { switch m.Config.Cluster.Type { case pilosa.ClusterGossip: + + // Set internal port (string). + gossipPortStr := pilosa.DefaultGossipPort + // Config.GossipPort is deprecated, so Config.Gossip.Port has priority + if m.Config.Gossip.Port != "" { + gossipPortStr = m.Config.Gossip.Port + } else if m.Config.GossipPort != "" { + gossipPortStr = m.Config.GossipPort + } + gossipPort, err := strconv.Atoi(gossipPortStr) if err != nil { return err @@ -222,9 +253,22 @@ func (m *Command) SetupServer() error { } // get the host portion of addr to use for binding - gossipHost := uri.Host() + gossipHost := m.Server.URI.Host() + var transport *gossip.Transport + if m.GossipTransport != nil { + transport = m.GossipTransport + } else { + transport, err = gossip.NewTransport(gossipHost, gossipPort) + if err != nil { + return err + } + } + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipMemberSet, err := gossip.NewGossipMemberSet(uri.String(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) + if m.Server.Name == "" { + return fmt.Errorf("must provide a valid name for gossip membership") + } + gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.Name, gossipHost, transport, gossipSeed, m.Server, gossipKey) if err != nil { return err } @@ -233,14 +277,13 @@ func (m *Command) SetupServer() error { m.Server.BroadcastReceiver = gossipMemberSet m.Server.Gossiper = gossipMemberSet case pilosa.ClusterStatic, pilosa.ClusterNone: - m.Server.Cluster.Static = true for _, address := range m.Config.Cluster.Hosts { uri, err := pilosa.NewURIFromAddress(address) if err != nil { return err } - cluster.Nodes = append(cluster.Nodes, &pilosa.Node{ + m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{ URI: *uri, }) } @@ -256,13 +299,6 @@ func (m *Command) SetupServer() error { default: return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type) } - - // Cluster management needs. - m.Server.Cluster.Broadcaster = m.Server.Broadcaster - - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) return nil } diff --git a/server/server_test.go b/server/server_test.go index 9992dab6f..4c9eb52bf 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -636,6 +636,81 @@ func (m *Main) Reopen() error { return nil } +// RunWithTransport runs Main and returns the dynamically allocated gossip port. +func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) { + defer close(m.Started) + + m.Config.Cluster.Type = "gossip" + + /* + TEST: + - SetupServer (just static settings from config) + - OpenListener (sets Server.Name to use in gossip) + - NewTransport (gossip) + - SetupNetworking (does the gossip or static stuff) - uses Server.Name + - Open server + + PRODUCTION: + - SetupServer (just static settings from config) + - SetupNetworking (does the gossip or static stuff) - calls NewTransport + - Open server - calls OpenListener + */ + + // SetupServer + err = m.SetupServer() + if err != nil { + return seed, coord, err + } + + // Open server listener. + // This is used to set Server.Name, which is used as the node + // name for identifying a memberlist node. + err = m.Server.OpenListener() + if err != nil { + return seed, coord, err + } + + // Open gossip transport to use in SetupServer. + transport, err := gossip.NewTransport(host, bindPort) + if err != nil { + return seed, coord, err + } + m.GossipTransport = transport + + if joinSeed != "" { + m.Config.Gossip.Seed = joinSeed + } else { + m.Config.Gossip.Seed = transport.URI.String() + } + seed = m.Config.Gossip.Seed + + // SetupNetworking + err = m.SetupNetworking() + if err != nil { + return seed, coord, err + } + + if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { + return seed, coord, err + } + + if coordinator != nil { + coord = *coordinator + } else { + coord = m.Server.URI + } + m.Server.Cluster.Coordinator = coord + m.Server.Cluster.Static = false + + // Initialize server. + err = m.Server.Open() + if err != nil { + return seed, coord, err + } + + return seed, coord, nil +} + // URL returns the base URL string for accessing the running program. func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } From 2afbdba1c537a7c0872271a119bb35d8724e7c12 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 13 Dec 2017 08:22:58 -0600 Subject: [PATCH 049/311] add lock protection around gossip.memberlist and holder.indexes --- gossip/gossip.go | 11 +++++++++++ holder.go | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/gossip/gossip.go b/gossip/gossip.go index 9d5845315..35f0056c8 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -20,6 +20,7 @@ import ( "log" "os" "strings" + "sync" "time" "golang.org/x/sync/errgroup" @@ -37,6 +38,7 @@ var _ memberlist.Delegate = &GossipMemberSet{} // GossipMemberSet represents a gossip implementation of MemberSet using memberlist. type GossipMemberSet struct { + mu sync.RWMutex memberlist *memberlist.Memberlist handler pilosa.BroadcastHandler @@ -51,6 +53,9 @@ type GossipMemberSet struct { // Nodes implements the MemberSet interface and returns a list of nodes in the cluster. func (g *GossipMemberSet) Nodes() []*pilosa.Node { + g.mu.RLock() + defer g.mu.RUnlock() + a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { uri, _ := pilosa.NewURIFromAddress(n.Name) @@ -78,13 +83,17 @@ func (g *GossipMemberSet) Open() error { } err := error(nil) + g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) + g.mu.Unlock() if err != nil { return err } g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { + g.mu.RLock() + defer g.mu.RUnlock() return g.memberlist.NumMembers() }, RetransmitMult: 3, @@ -98,7 +107,9 @@ func (g *GossipMemberSet) Open() error { // attach to gossip seed node nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds + g.mu.RLock() err = g.joinWithRetry(pilosa.NodeSet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) + g.mu.RUnlock() if err != nil { return err } diff --git a/holder.go b/holder.go index 4c3d476d2..9a1d793d5 100644 --- a/holder.go +++ b/holder.go @@ -156,7 +156,9 @@ func (h *Holder) Open() error { } return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) } + h.mu.Lock() h.indexes[index.Name()] = index + h.mu.Unlock() } h.logger().Printf("open holder: complete") @@ -190,6 +192,8 @@ func (h *Holder) Close() error { // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. func (h *Holder) HasData() bool { + h.mu.RLock() + defer h.mu.RUnlock() return h.hasData || len(h.indexes) > 0 } From 78cddbd0c7f413df919ae8c59dadcea082b24ece Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 15 Dec 2017 12:37:51 -0600 Subject: [PATCH 050/311] Add the rest of the available memberlist configuration options into pilosa.Config.Gossip. --- config.go | 94 +++++++++++++++++++-- ctl/server.go | 11 +++ gossip/gossip.go | 70 +++++++++++----- server/cluster_test.go | 16 ++-- server/server.go | 18 +--- server/server_test.go | 184 ----------------------------------------- 6 files changed, 156 insertions(+), 237 deletions(-) diff --git a/config.go b/config.go index 1b4a4485a..9dfbb53ac 100644 --- a/config.go +++ b/config.go @@ -35,14 +35,77 @@ const ( // DefaultClusterType sets the node intercommunication method. DefaultClusterType = ClusterGossip - // DefaultGossipPort indicates the port to which pilosa should bind for internal state sharing. - DefaultGossipPort = "14000" - // DefaultMetrics sets the internal metrics to no-op. DefaultMetrics = "nop" // DefaultMaxWritesPerRequest is the default number of writes per request. DefaultMaxWritesPerRequest = 5000 + + // Gossip config based on memberlist.Config. + + // Port indicates the port to which pilosa should bind for internal state sharing. + DefaultGossipPort = "14000" + + // StreamTimeout is the timeout for establishing a stream connection with + // a remote node for a full state sync, and for stream read and write + // operations. Maps to memberlist TCPTimeout. + DefaultGossipStreamTimeout = 10 * time.Second + + // SuspicionMult is the multiplier for determining the time an + // inaccessible node is considered suspect before declaring it dead. + // The actual timeout is calculated using the formula: + // + // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval + // + // This allows the timeout to scale properly with expected propagation + // delay with a larger cluster size. The higher the multiplier, the longer + // an inaccessible node is considered part of the cluster before declaring + // it dead, giving that suspect node more time to refute if it is indeed + // still alive. + DefaultGossipSuspicionMult = 4 + + // PushPullInterval is the interval between complete state syncs. + // Complete state syncs are done with a single node over TCP and are + // quite expensive relative to standard gossiped messages. Setting this + // to zero will disable state push/pull syncs completely. + // + // Setting this interval lower (more frequent) will increase convergence + // speeds across larger clusters at the expense of increased bandwidth + // usage. + DefaultGossipPushPullInterval = 30 * time.Second + + // ProbeInterval and ProbeTimeout are used to configure probing behavior + // for memberlist. + // + // ProbeInterval is the interval between random node probes. Setting + // this lower (more frequent) will cause the memberlist cluster to detect + // failed nodes more quickly at the expense of increased bandwidth usage. + // + // ProbeTimeout is the timeout to wait for an ack from a probed node + // before assuming it is unhealthy. This should be set to 99-percentile + // of RTT (round-trip time) on your network. + DefaultGossipProbeInterval = 1 * time.Second + DefaultGossipProbeTimeout = 500 * time.Millisecond + + // GossipInterval and GossipNodes are used to configure the gossip + // behavior of memberlist. + // + // GossipInterval is the interval between sending messages that need + // to be gossiped that haven't been able to piggyback on probing messages. + // If this is set to zero, non-piggyback gossip is disabled. By lowering + // this value (more frequent) gossip messages are propagated across + // the cluster more quickly at the expense of increased bandwidth. + // + // GossipNodes is the number of random nodes to send gossip messages to + // per GossipInterval. Increasing this number causes the gossip messages + // to propagate across the cluster more quickly at the expense of + // increased bandwidth. + // + // GossipToTheDeadTime is the interval after which a node has died that + // we will still try to gossip to it. This gives it a chance to refute. + DefaultGossipGossipInterval = 200 * time.Millisecond + DefaultGossipGossipNodes = 3 + DefaultGossipGossipToTheDeadTime = 30 * time.Second ) // ClusterTypes set of cluster types. @@ -68,9 +131,17 @@ type Config struct { GossipSeed string `toml:"gossip-seed"` Gossip struct { - Port string `toml:"port"` - Seed string `toml:"seed"` - Key string `toml:"key"` + Port string `toml:"port"` + Seed string `toml:"seed"` + Key string `toml:"key"` + StreamTimeout Duration `toml:"stream-timeout"` + SuspicionMult int `toml:"suspicion-mult"` + PushPullInterval Duration `toml:"push-pull-interval"` + ProbeTimeout Duration `toml:"probe-timeout"` + ProbeInterval Duration `toml:"probe-interval"` + GossipNodes int `toml:"gossip-nodes"` + GossipInterval Duration `toml:"gossip-interval"` + GossipToTheDeadTime Duration `toml:"gossip-to-the-dead-time"` } `toml:"gossip"` Cluster struct { @@ -115,6 +186,17 @@ func NewConfig() *Config { c.Metric.Service = DefaultMetrics c.Metric.Diagnostics = true c.TLS = TLSConfig{} + + // Gossip related config. + c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) + c.Gossip.SuspicionMult = DefaultGossipSuspicionMult + c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) + c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout) + c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval) + c.Gossip.GossipNodes = DefaultGossipGossipNodes + c.Gossip.GossipInterval = Duration(DefaultGossipGossipInterval) + c.Gossip.GossipToTheDeadTime = Duration(DefaultGossipGossipToTheDeadTime) + return c } diff --git a/ctl/server.go b/ctl/server.go index 07837caa0..ba49aded1 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -28,9 +28,20 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.") + // gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") + + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", 10*time.Second, "Timeout for establishing a stream connection with a remote node for a full state sync.") + flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", 4, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", 30*time.Second, "Interval between complete state syncs.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", 500*time.Millisecond, "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", 1*time.Second, "Interval between random node probes.") + flags.IntVarP(&srv.Config.Gossip.GossipNodes, "gossip.gossip-nodes", "", 3, "Number of random nodes to send gossip messages to per GossipInterval.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipInterval), "gossip.gossip-interval", "", 200*time.Millisecond, "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipToTheDeadTime), "gossip.gossip-to-the-dead-time", "", 30*time.Second, "Interval after which a node has died that we will still try to gossip to it.") + flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") diff --git a/gossip/gossip.go b/gossip/gossip.go index 35f0056c8..1ee93263a 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -17,8 +17,11 @@ package gossip import ( "fmt" "io" + "io/ioutil" "log" + "net" "os" + "strconv" "strings" "sync" "time" @@ -154,51 +157,82 @@ type gossipConfig struct { } // NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport. -func NewGossipMemberSetWithTransport(name string, gossipHost string, transport *Transport, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipMemberSet, error) { - port := transport.Net.GetAutoBindPort() +func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport *Transport, server *pilosa.Server) (*GossipMemberSet, error) { g := &GossipMemberSet{ LogOutput: server.LogOutput, } + port := transport.Net.GetAutoBindPort() + host, _, err := net.SplitHostPort(cfg.Bind) + if err != nil { + return nil, err + } + + var gossipKey []byte + if cfg.Gossip.Key != "" { + gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key) + if err != nil { + return nil, err + } + } + // memberlist config conf := memberlist.DefaultLocalConfig() conf.Transport = transport.Net + conf.Name = name + conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port - conf.Name = name - conf.BindAddr = gossipHost - conf.AdvertiseAddr = pilosa.HostToIP(gossipHost) - //conf.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. + conf.AdvertiseAddr = pilosa.HostToIP(host) + // + conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout) + conf.SuspicionMult = cfg.Gossip.SuspicionMult + conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval) + conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout) + conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval) + conf.GossipNodes = cfg.Gossip.GossipNodes + conf.GossipInterval = time.Duration(cfg.Gossip.GossipInterval) + conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.GossipToTheDeadTime) + // conf.Delegate = g - conf.SecretKey = secretKey + conf.SecretKey = gossipKey conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) - //TODO: pull memberlist config from pilosa.cfg file g.config = &gossipConfig{ memberlistConfig: conf, - gossipSeed: gossipSeed, + gossipSeed: cfg.Gossip.Seed, } g.statusHandler = server // If no gossipSeed is provided, use local host:port. - if gossipSeed == "" { - g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, port) + if cfg.Gossip.Seed == "" { + g.config.gossipSeed = fmt.Sprintf("%s:%d", host, port) } return g, nil } // NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port. -func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipMemberSet, error) { - // set up the transport - transport, err := NewTransport(gossipHost, gossipPort) +func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) { + + port, err := strconv.Atoi(cfg.Gossip.Port) + if err != nil { + return nil, err + } + host, _, err := net.SplitHostPort(cfg.Bind) if err != nil { return nil, err } - return NewGossipMemberSetWithTransport(name, gossipHost, transport, gossipSeed, server, secretKey) + // Set up the transport. + transport, err := NewTransport(host, port) + if err != nil { + return nil, err + } + + return NewGossipMemberSetWithTransport(name, cfg, transport, server) } // SendSync implementation of the Broadcaster interface. @@ -473,12 +507,6 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { if err != nil { return nil, fmt.Errorf("Could not set up network transport: %v", err) } - if conf.BindPort == 0 { - port := nt.GetAutoBindPort() - conf.BindPort = port - conf.AdvertisePort = port - logger.Printf("[DEBUG] Using dynamic bind port %d", port) - } return nt, nil } diff --git a/server/cluster_test.go b/server/cluster_test.go index bc7b1cd91..fdf6fdc88 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -27,7 +27,7 @@ import ( ) // Ensure program can send/receive broadcast messages. -func TestMain_XSendReceiveMessage(t *testing.T) { +func TestMain_SendReceiveMessage(t *testing.T) { m0 := MustRunMain() defer m0.Close() @@ -45,14 +45,13 @@ func TestMain_XSendReceiveMessage(t *testing.T) { // Configure node0 // get the host portion of addr to use for binding - gossipHost := m0.Server.URI.Host() - gossipPort := 0 - gossipSeed := "" + m0.Config.Gossip.Port = "0" + m0.Config.Gossip.Seed = "" m0.Server.Cluster.Coordinator = m0.Server.URI m0.Server.Cluster.Topology = &pilosa.Topology{NodeSet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m0.Server, nil) + gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) if err != nil { t.Fatal(err) } @@ -74,13 +73,12 @@ func TestMain_XSendReceiveMessage(t *testing.T) { // Configure node1 // get the host portion of addr to use for binding - gossipHost = m1.Server.URI.Host() - gossipPort = 0 - gossipSeed = gossipMemberSet0.Seed() + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seed = gossipMemberSet0.Seed() m1.Server.Cluster.Coordinator = m0.Server.URI m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) + gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) if err != nil { t.Fatal(err) } diff --git a/server/server.go b/server/server.go index 045079229..27c3c2e4c 100644 --- a/server/server.go +++ b/server/server.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -236,21 +235,6 @@ func (m *Command) SetupNetworking() error { if err != nil { return err } - gossipSeed := pilosa.DefaultHost + ":" + pilosa.DefaultGossipPort - // Config.GossipSeed is deprecated, so Config.Gossip.Seed has priority - if m.Config.Gossip.Seed != "" { - gossipSeed = m.Config.Gossip.Seed - } else if m.Config.GossipSeed != "" { - gossipSeed = m.Config.GossipSeed - } - - var gossipKey []byte - if m.Config.Gossip.Key != "" { - gossipKey, err = ioutil.ReadFile(m.Config.Gossip.Key) - if err != nil { - return err - } - } // get the host portion of addr to use for binding gossipHost := m.Server.URI.Host() @@ -268,7 +252,7 @@ func (m *Command) SetupNetworking() error { if m.Server.Name == "" { return fmt.Errorf("must provide a valid name for gossip membership") } - gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.Name, gossipHost, transport, gossipSeed, m.Server, gossipKey) + gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.Name, m.Config, transport, m.Server) if err != nil { return err } diff --git a/server/server_test.go b/server/server_test.go index 4c9eb52bf..b96dbec26 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,7 +30,6 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" @@ -384,189 +383,6 @@ func TestCountOpenFiles(t *testing.T) { } } -// Ensure program can send/receive broadcast messages. -func TestMain_SendReceiveMessage(t *testing.T) { - - m0 := MustRunMain() - defer m0.Close() - - m1 := MustRunMain() - defer m1.Close() - - // Update cluster config - m0.Server.Cluster.Nodes = []*pilosa.Node{ - {URI: m0.Server.URI}, - {URI: m1.Server.URI}, - } - m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes - - // Configure node0 - - // get the host portion of addr to use for binding - gossipHost := m0.Server.URI.Host() - gossipPort := 0 - gossipSeed := "" - - topology := &pilosa.Topology{NodeSet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} - - m0.Server.Cluster.Coordinator = m0.Server.URI - m0.Server.Cluster.Topology = topology - m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - - gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m0.Server, nil) - if err != nil { - t.Fatal(err) - } - m0.Server.Cluster.MemberSet = gossipMemberSet0 - m0.Server.Broadcaster = m0.Server - m0.Server.Gossiper = gossipMemberSet0 - m0.Server.Handler.Broadcaster = m0.Server.Broadcaster - m0.Server.Holder.Broadcaster = m0.Server.Broadcaster - m0.Server.BroadcastReceiver = gossipMemberSet0 - - if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { - t.Fatal(err) - } - // Open Cluster management. - if err := m0.Server.Cluster.Open(); err != nil { - t.Fatal(err) - } - - // Configure node1 - - // get the host portion of addr to use for binding - gossipHost = m1.Server.URI.Host() - gossipPort = 0 - gossipSeed = gossipMemberSet0.Seed() - - m1.Server.Cluster.Coordinator = m0.Server.URI - m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - - gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), gossipHost, gossipPort, gossipSeed, m1.Server, nil) - if err != nil { - t.Fatal(err) - } - m1.Server.Cluster.MemberSet = gossipMemberSet1 - m1.Server.Broadcaster = m1.Server - m1.Server.Gossiper = gossipMemberSet1 - m1.Server.Handler.Broadcaster = m1.Server.Broadcaster - m1.Server.Holder.Broadcaster = m1.Server.Broadcaster - m1.Server.BroadcastReceiver = gossipMemberSet1 - - if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { - t.Fatal(err) - } - // Open Cluster management. - if err := m1.Server.Cluster.Open(); err != nil { - t.Fatal(err) - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - // Expected indexes and Frames - expected := map[string][]string{ - "i": []string{"f"}, - } - - // Create a client for each node. - client0 := m0.Client() - client1 := m1.Client() - - // Create indexes and frames on one node. - if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - // Make sure node0 knows about the index and frame created. - schema0, err := client0.Schema(context.Background()) - if err != nil { - t.Fatal(err) - } - received0 := map[string][]string{} - for _, idx := range schema0 { - received0[idx.Name] = []string{} - for _, frame := range idx.Frames { - received0[idx.Name] = append(received0[idx.Name], frame.Name) - } - } - if !reflect.DeepEqual(received0, expected) { - t.Fatalf("unexpected schema on node0: %s", received0) - } - - // Make sure node1 knows about the index and frame created. - schema1, err := client1.Schema(context.Background()) - if err != nil { - t.Fatal(err) - } - received1 := map[string][]string{} - for _, idx := range schema1 { - received1[idx.Name] = []string{} - for _, frame := range idx.Frames { - received1[idx.Name] = append(received1[idx.Name], frame.Name) - } - } - if !reflect.DeepEqual(received1, expected) { - t.Fatalf("unexpected schema on node1: %s", received1) - } - - // Write data on first node. - if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=2400000) - `); err != nil { - t.Fatal(err) - } - - // We have to wait for the broadcast message to be sent before checking state. - time.Sleep(1 * time.Second) - - // Make sure node0 knows about the latest MaxSlice. - maxSlices0, err := client0.MaxSliceByIndex(context.Background()) - if err != nil { - t.Fatal(err) - } - if maxSlices0["i"] != 2 { - t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) - } - - // Make sure node1 knows about the latest MaxSlice. - maxSlices1, err := client1.MaxSliceByIndex(context.Background()) - if err != nil { - t.Fatal(err) - } - if maxSlices1["i"] != 2 { - t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) - } - - // Write input definition to the first node. - if _, err := m0.CreateDefinition("i", "test", `{ - "frames": [{"name": "event-time", - "options": { - "cacheType": "ranked", - "timeQuantum": "YMD" - }}], - "fields": [{"name": "columnID", - "primaryKey": true - }]} - `); err != nil { - t.Fatal(err) - } - - // We have to wait for the broadcast message to be sent before checking state. - time.Sleep(1 * time.Second) - - frame0 := m0.Server.Holder.Frame("i", "event-time") - if frame0 == nil { - t.Fatal("frame not found") - } - frame1 := m1.Server.Holder.Frame("i", "event-time") - if frame1 == nil { - t.Fatal("frame not found") - } -} - // Main represents a test wrapper for main.Main. type Main struct { *server.Command From d2d0756f4a22db4387796e22c671da2c4b04747d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 15 Dec 2017 14:39:35 -0600 Subject: [PATCH 051/311] Define default config values for pilosa server in NewConfig rather than flags. Based on work done in #885. --- cmd/server_test.go | 3 +-- config.go | 66 ++++++++++++++++++++++++++++++---------------- ctl/config_test.go | 7 ++--- ctl/server.go | 60 +++++++++++++++++++++++------------------ server.go | 1 - server/server.go | 3 --- 6 files changed, 82 insertions(+), 58 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index b650e316a..c9b900bde 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -45,7 +45,7 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, + env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" @@ -65,7 +65,6 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) - v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182)) v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() diff --git a/config.go b/config.go index 9dfbb53ac..a43963f93 100644 --- a/config.go +++ b/config.go @@ -26,6 +26,9 @@ const ( ) const ( + // DefaultDataDir is the default data directory. + DefaultDataDir = "~/.pilosa" + // DefaultHost is the default hostname to use. DefaultHost = "localhost" @@ -106,6 +109,8 @@ const ( DefaultGossipGossipInterval = 200 * time.Millisecond DefaultGossipGossipNodes = 3 DefaultGossipGossipToTheDeadTime = 30 * time.Second + + DefaultMetricPollInterval = 0 * time.Minute ) // ClusterTypes set of cluster types. @@ -130,6 +135,23 @@ type Config struct { // GossipSeed DEPRECATED GossipSeed string `toml:"gossip-seed"` + // Limits the number of mutating commands that can be in a single request to + // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. + MaxWritesPerRequest int `toml:"max-writes-per-request"` + + LogPath string `toml:"log-path"` + + // TLS + TLS TLSConfig + + Cluster struct { + Coordinator string `toml:"coordinator"` + ReplicaN int `toml:"replicas"` + Type string `toml:"type"` + Hosts []string `toml:"hosts"` + LongQueryTime Duration `toml:"long-query-time"` + } `toml:"cluster"` + Gossip struct { Port string `toml:"port"` Seed string `toml:"seed"` @@ -144,50 +166,39 @@ type Config struct { GossipToTheDeadTime Duration `toml:"gossip-to-the-dead-time"` } `toml:"gossip"` - Cluster struct { - Coordinator string `toml:"coordinator"` - ReplicaN int `toml:"replicas"` - Type string `toml:"type"` - Hosts []string `toml:"hosts"` - PollInterval Duration `toml:"poll-interval"` - LongQueryTime Duration `toml:"long-query-time"` - } `toml:"cluster"` - AntiEntropy struct { Interval Duration `toml:"interval"` } `toml:"anti-entropy"` - // Limits the number of mutating commands that can be in a single request to - // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. - MaxWritesPerRequest int `toml:"max-writes-per-request"` - - LogPath string `toml:"log-path"` - Metric struct { Service string `toml:"service"` Host string `toml:"host"` PollInterval Duration `toml:"poll-interval"` Diagnostics bool `toml:"diagnostics"` } `toml:"metric"` - - TLS TLSConfig } // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ - Bind: DefaultHost + ":" + DefaultPort, + DataDir: DefaultDataDir, + Bind: ":" + DefaultPort, MaxWritesPerRequest: DefaultMaxWritesPerRequest, + // LogPath: "", + TLS: TLSConfig{}, } + + // Cluster config. + // c.Cluster.Coordinator = "" c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Type = DefaultClusterType c.Cluster.Hosts = []string{} - c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) - c.Metric.Service = DefaultMetrics - c.Metric.Diagnostics = true - c.TLS = TLSConfig{} + c.Cluster.LongQueryTime = Duration(time.Minute) - // Gossip related config. + // Gossip config. + // c.Gossip.Port = "" + // c.Gossip.Seed = "" + // c.Gossip.Key = "" c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) c.Gossip.SuspicionMult = DefaultGossipSuspicionMult c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) @@ -197,6 +208,15 @@ func NewConfig() *Config { c.Gossip.GossipInterval = Duration(DefaultGossipGossipInterval) c.Gossip.GossipToTheDeadTime = Duration(DefaultGossipGossipToTheDeadTime) + // AntiEntropy config. + c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) + + // Metric config. + c.Metric.Service = DefaultMetrics + // c.Metric.Host = "" + c.Metric.PollInterval = Duration(DefaultMetricPollInterval) + c.Metric.Diagnostics = true + return c } diff --git a/ctl/config_test.go b/ctl/config_test.go index a3bd035ac..b54446e34 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -17,11 +17,12 @@ package ctl import ( "bytes" "context" - "github.com/pilosa/pilosa" "io" "os" "strings" "testing" + + "github.com/pilosa/pilosa" ) func TestConfigCommand_Run(t *testing.T) { @@ -38,7 +39,7 @@ func TestConfigCommand_Run(t *testing.T) { if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), pilosa.DefaultHost) { - t.Fatalf("Unexpected config: %s", buf.String()) + } else if !strings.Contains(buf.String(), ":10101") { + t.Fatalf("Unexpected config: \n%s", buf.String()) } } diff --git a/ctl/server.go b/ctl/server.go index ba49aded1..1dddeb0d7 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -24,38 +24,46 @@ import ( // BuildServerFlags attaches a set of flags to the command for a server instance. func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() - flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") - flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") + flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.") - // gossip - flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.") - flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") - - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", 10*time.Second, "Timeout for establishing a stream connection with a remote node for a full state sync.") - flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", 4, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", 30*time.Second, "Interval between complete state syncs.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", 500*time.Millisecond, "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", 1*time.Second, "Interval between random node probes.") - flags.IntVarP(&srv.Config.Gossip.GossipNodes, "gossip.gossip-nodes", "", 3, "Number of random nodes to send gossip messages to per GossipInterval.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipInterval), "gossip.gossip-interval", "", 200*time.Millisecond, "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipToTheDeadTime), "gossip.gossip-to-the-dead-time", "", 30*time.Second, "Interval after which a node has died that we will still try to gossip to it.") - - flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") + flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") + + // TLS + SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) + + // Cluster + flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") + flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") - flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") - flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") + + // Gossip + flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") + flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", srv.Config.Gossip.Seed, "Host with which to seed the gossip membership.") + flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.") + flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.") + flags.IntVarP(&srv.Config.Gossip.GossipNodes, "gossip.gossip-nodes", "", srv.Config.Gossip.GossipNodes, "Number of random nodes to send gossip messages to per GossipInterval.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipInterval), "gossip.gossip-interval", "", (time.Duration)(srv.Config.Gossip.GossipInterval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipToTheDeadTime), "gossip.gossip-to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.GossipToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.") + + // AntiEntropy + flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.") + + // Metric + flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.") + flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.") + flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.") + + // CPU Profiling flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") - flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") - flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") - flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.") - flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") - SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) } diff --git a/server.go b/server.go index 1d03bd78b..cbeeb53a1 100644 --- a/server.go +++ b/server.go @@ -42,7 +42,6 @@ import ( // Default server settings. const ( DefaultAntiEntropyInterval = 10 * time.Minute - DefaultPollingInterval = 60 * time.Second DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) diff --git a/server/server.go b/server/server.go index 27c3c2e4c..d238d1c2f 100644 --- a/server/server.go +++ b/server/server.go @@ -41,9 +41,6 @@ func init() { } const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - // DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. DefaultDiagnosticsInterval = 1 * time.Hour ) From 52897421ada7c0e4cb8f98405aa7774bd42c5434 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 19 Dec 2017 11:07:34 -0600 Subject: [PATCH 052/311] fix comment --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index 74ecbb6c8..f251ac444 100644 --- a/server.go +++ b/server.go @@ -211,7 +211,7 @@ func (s *Server) Open() error { // buffered channel. s.Cluster.ListenForJoins() - // Load local ID. + // Load NodeID. if err := s.Holder.loadNodeID(); err != nil { s.Logger().Println(err) } From 17484fd73ea7246101ccd99338918369faa7fd01 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 20 Dec 2017 14:52:59 -0600 Subject: [PATCH 053/311] make sure the holder has opened before merging NodeStatus --- server.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index f251ac444..dc25103ff 100644 --- a/server.go +++ b/server.go @@ -512,7 +512,18 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { if s.Cluster.State != ClusterStateNormal { return nil } - return s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + + go func() { + // Make sure the holder has opened. + <-s.Holder.opened + + err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + if err != nil { + s.Logger().Printf("merge remote status: %s", err) + } + }() + + return nil } func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { From 780078e427a2d89b6c3ca79858dce7e68b0f42c0 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 27 Dec 2017 17:51:31 -0600 Subject: [PATCH 054/311] add context to errors in gossip member set --- gossip/gossip.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 1ee93263a..84daad8b3 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -82,7 +82,7 @@ func (g *GossipMemberSet) Seed() string { // Open implements the MemberSet interface to start network activity. func (g *GossipMemberSet) Open() error { if g.handler == nil { - return fmt.Errorf("opening GossipMemberSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") + return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()") } err := error(nil) @@ -90,7 +90,7 @@ func (g *GossipMemberSet) Open() error { g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() if err != nil { - return err + return fmt.Errorf("creating memberlist: %s", err) } g.broadcasts = &memberlist.TransmitLimitedQueue{ @@ -104,7 +104,7 @@ func (g *GossipMemberSet) Open() error { uri, err := pilosa.NewURIFromAddress(g.config.gossipSeed) if err != nil { - return err + return fmt.Errorf("new uri from address: %s", err) } // attach to gossip seed node @@ -114,7 +114,7 @@ func (g *GossipMemberSet) Open() error { err = g.joinWithRetry(pilosa.NodeSet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) g.mu.RUnlock() if err != nil { - return err + return fmt.Errorf("joining member set: %s", err) } return nil } @@ -166,14 +166,14 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport port := transport.Net.GetAutoBindPort() host, _, err := net.SplitHostPort(cfg.Bind) if err != nil { - return nil, err + return nil, fmt.Errorf("split host port: %s", err) } var gossipKey []byte if cfg.Gossip.Key != "" { gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key) if err != nil { - return nil, err + return nil, fmt.Errorf("reading gossip key: %s", err) } } @@ -219,17 +219,17 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) port, err := strconv.Atoi(cfg.Gossip.Port) if err != nil { - return nil, err + return nil, fmt.Errorf("convert port: %s", err) } host, _, err := net.SplitHostPort(cfg.Bind) if err != nil { - return nil, err + return nil, fmt.Errorf("split host port: %s", err) } // Set up the transport. transport, err := NewTransport(host, port) if err != nil { - return nil, err + return nil, fmt.Errorf("new tranport: %s", err) } return NewGossipMemberSetWithTransport(name, cfg, transport, server) @@ -239,7 +239,7 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) func (g *GossipMemberSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { - return err + return fmt.Errorf("marshal message: %s", err) } mlist := g.memberlist @@ -267,7 +267,7 @@ func (g *GossipMemberSet) SendSync(pb proto.Message) error { func (g *GossipMemberSet) SendAsync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { - return err + return fmt.Errorf("marshal message: %s", err) } b := &broadcast{ @@ -438,12 +438,12 @@ func NewTransport(host string, port int) (*Transport, error) { net, err := newTransport(conf) if err != nil { - return nil, err + return nil, fmt.Errorf("new transport: %s", err) } uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort())) if err != nil { - return nil, err + return nil, fmt.Errorf("new uri from host port: %s", err) } return &Transport{ From e57c125c082ff2af9826c6f82094e42e687315d1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 4 Jan 2018 10:45:04 -0600 Subject: [PATCH 055/311] getting some minor fixes out of my stash --- cluster.go | 4 +--- holder.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 63a558b2f..4f29196bc 100644 --- a/cluster.go +++ b/cluster.go @@ -995,8 +995,6 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, toCluster.addNodeBasicSorted(nodeAction.uri) } - pbSchema := c.Holder.EncodeSchema() - // multiIndex is a map of sources initialized with all the nodes in toCluster. multiIndex := make(map[URI][]*internal.ResizeSource) @@ -1029,7 +1027,7 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, URI: u.Encode(), Coordinator: encodeURI(c.Coordinator), Sources: sources, - Schema: pbSchema, // Include the schema to ensure it's in sync on the receiving node. + Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), } j.Instructions = append(j.Instructions, instr) diff --git a/holder.go b/holder.go index 8aeb6e1e8..cdbb59549 100644 --- a/holder.go +++ b/holder.go @@ -278,7 +278,7 @@ func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { } } -// EncodeSchema creates and internal representation of schema. +// EncodeSchema creates an internal representation of schema. func (h *Holder) EncodeSchema() *internal.Schema { return &internal.Schema{ Indexes: EncodeIndexes(h.Indexes()), From d64a107dddaa91c11d579f410f22f9201d7a8d8f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 17 Jan 2018 13:26:20 -0600 Subject: [PATCH 056/311] remove deprecated MustNewRunningServer --- test/pilosa.go | 79 -------------------------------------------------- 1 file changed, 79 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 2d264e6ae..d733e99c0 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -4,9 +4,7 @@ import ( "bytes" "io" "io/ioutil" - "net" "os" - "strconv" "testing" "github.com/pilosa/pilosa" @@ -49,18 +47,6 @@ func NewMain() *Main { return m } -/* -// MustRunMain returns a new, running Main. Panic on error. -func MustRunMain() *Main { - m := NewMain() - m.Config.Metric.Diagnostics = false // Disable diagnostics. - if err := m.Run(); err != nil { - panic(err) - } - return m -} -*/ - // Close closes the program and removes the underlying data directory. func (m *Main) Close() error { defer os.RemoveAll(m.Config.DataDir) @@ -144,71 +130,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coor //////////////////////////////////////////////////////////////////////////////////// -func MustNewRunningServer(t *testing.T) *server.Command { - s, err := newServer() - if err != nil { - t.Fatalf("getting new server: %v", err) - } - - err = s.Run() - if err != nil { - t.Fatalf("running new pilosa server: %v", err) - } - return s -} - -func newServer() (*server.Command, error) { - s := server.NewCommand(&bytes.Buffer{}, ioutil.Discard, ioutil.Discard) - - port, err := findPort() - if err != nil { - return nil, errors.Wrap(err, "getting port") - } - s.Config.Bind = "localhost:" + strconv.Itoa(port) - - gport, err := findPort() - if err != nil { - return nil, errors.Wrap(err, "getting gossip port") - } - s.Config.GossipPort = strconv.Itoa(gport) - - s.Config.GossipSeed = "localhost:" + s.Config.GossipPort - s.Config.Cluster.Type = "gossip" - s.Config.Metric.Diagnostics = false - td, err := ioutil.TempDir("", "") - if err != nil { - return nil, errors.Wrap(err, "temp dir") - } - s.Config.DataDir = td - return s, nil -} - -func findPort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", ":0") - if err != nil { - return 0, errors.Wrap(err, "resolving new port addr") - } - l, err := net.ListenTCP("tcp", addr) - if err != nil { - return 0, errors.Wrap(err, "listening to get new port") - } - port := l.Addr().(*net.TCPAddr).Port - err = l.Close() - if err != nil { - return port, errors.Wrap(err, "closing listener") - } - return port, nil - -} - -func MustFindPort(t *testing.T) int { - port, err := findPort() - if err != nil { - t.Fatalf("allocating new port: %v", err) - } - return port -} - type Cluster struct { Servers []*Main } From da5f8f11bb96a8d80f109dca34a15ba26f5c9a0b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Jan 2018 11:58:20 -0600 Subject: [PATCH 057/311] benchmark for snapshotting --- fragment_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/fragment_test.go b/fragment_test.go index 2ccf538f7..14d36a56c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1075,3 +1075,25 @@ func TestFragment_Snapshot_Run(t *testing.T) { t.Fatalf("unexpected count (reopen): %d", n) } } + +func BenchmarkFragment_Snapshot(b *testing.B) { + if *FragmentPath == "" { + b.Skip("no fragment specified") + } + + // Open the fragment specified by the path. + f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + if err := f.Open(); err != nil { + b.Fatal(err) + } + defer f.Close() + + // Reset timer and execute benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := f.Snapshot() + if err != nil { + b.Fatalf("unexpected count (reopen): %s", err) + } + } +} From e1735a60ec31486a3f43ab8aeb14eb539bfb522a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Jan 2018 12:15:15 -0600 Subject: [PATCH 058/311] added alloc logging --- fragment_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/fragment_test.go b/fragment_test.go index 14d36a56c..8a54d3b07 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1090,6 +1090,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { // Reset timer and execute benchmark. b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { err := f.Snapshot() if err != nil { From 1b0fcb0e5abaf1e3eb256f1135cf9b4650ded7eb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 23 Jan 2018 12:38:14 -0600 Subject: [PATCH 059/311] move cluster Main test helpers to the test package --- server/cluster_test.go | 27 ++--- server/server_test.go | 230 ++--------------------------------------- test/pilosa.go | 100 ++++++++++++++++++ 3 files changed, 120 insertions(+), 237 deletions(-) diff --git a/server/cluster_test.go b/server/cluster_test.go index fdf6fdc88..4d1d75348 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -24,15 +24,16 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/test" ) // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { - m0 := MustRunMain() + m0 := test.MustRunMain() defer m0.Close() - m1 := MustRunMain() + m1 := test.MustRunMain() defer m1.Close() // Update cluster config @@ -205,7 +206,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Ensure that an empty node comes up in a NORMAL state. func TestClusterResize_EmptyNode(t *testing.T) { - m0 := MustRunMain() + m0 := test.MustRunMain() defer m0.Close() if m0.Server.Cluster.State != pilosa.ClusterStateNormal { @@ -216,7 +217,7 @@ func TestClusterResize_EmptyNode(t *testing.T) { // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { // Configure node0 - m0 := NewMain() + m0 := test.NewMain() defer m0.Close() gossipHost := "localhost" @@ -227,7 +228,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { } // Configure node1 - m1 := NewMain() + m1 := test.NewMain() defer m1.Close() seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, seed, &coord) @@ -246,7 +247,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { // Configure node0 - m0 := NewMain() + m0 := test.NewMain() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -255,7 +256,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := NewMain() + m1 := test.NewMain() defer m1.Close() var eg errgroup.Group @@ -280,7 +281,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := NewMain() + m0 := test.NewMain() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -299,7 +300,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := NewMain() + m1 := test.NewMain() defer m1.Close() var eg errgroup.Group @@ -326,7 +327,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Run("ContinuousSlices", func(t *testing.T) { // Configure node0 - m0 := NewMain() + m0 := test.NewMain() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -354,7 +355,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := NewMain() + m1 := test.NewMain() defer m1.Close() var eg errgroup.Group @@ -381,7 +382,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Run("SkippedSlice", func(t *testing.T) { // Configure node0 - m0 := NewMain() + m0 := test.NewMain() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -409,7 +410,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := NewMain() + m1 := test.NewMain() defer m1.Close() var eg errgroup.Group diff --git a/server/server_test.go b/server/server_test.go index c1a7e61fa..379175c75 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -15,26 +15,19 @@ package server_test import ( - "bytes" "context" "encoding/json" "fmt" - "io" "io/ioutil" "math/rand" - "net/http" - "os" "reflect" "runtime" "sort" - "strings" "testing" "testing/quick" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/gossip" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -45,7 +38,7 @@ func TestMain_Set_Quick(t *testing.T) { } if err := quick.Check(func(cmds []SetCommand) bool { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create client. @@ -121,7 +114,7 @@ func TestMain_Set_Quick(t *testing.T) { // Ensure program can set row attributes and retrieve them. func TestMain_SetRowAttrs(t *testing.T) { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create frames. @@ -198,7 +191,7 @@ func TestMain_SetRowAttrs(t *testing.T) { // Ensure program can set column attributes and retrieve them. func TestMain_SetColumnAttrs(t *testing.T) { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create frames. @@ -242,7 +235,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { // Ensure program can set column attributes with columnLabel option. func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create frames. @@ -276,7 +269,7 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { - mains1 := NewMainArrayWithCluster(2) + mains1 := test.NewMainArrayWithCluster(2) m0 := mains1[0] // Create frames. @@ -309,7 +302,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Start second cluster. - mains2 := NewMainArrayWithCluster(2) + mains2 := test.NewMainArrayWithCluster(2) m2 := mains2[0] defer m2.Close() @@ -378,191 +371,6 @@ func TestCountOpenFiles(t *testing.T) { } } -// Main represents a test wrapper for main.Main. -type Main struct { - *server.Command - - Stdin bytes.Buffer - Stdout bytes.Buffer - Stderr bytes.Buffer -} - -// NewMain returns a new instance of Main with a temporary data directory and random port. -func NewMain() *Main { - path, err := ioutil.TempDir("", "pilosa-") - if err != nil { - panic(err) - } - - m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} - m.Server.Network = *test.Network - m.Config.DataDir = path - m.Config.Bind = "localhost:0" - m.Config.Cluster.Type = "static" - m.Command.Stdin = &m.Stdin - m.Command.Stdout = &m.Stdout - m.Command.Stderr = &m.Stderr - - if testing.Verbose() { - m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) - m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) - } - - return m -} - -func NewMainArrayWithCluster(size int) []*Main { - cluster, err := test.NewServerCluster(size) - if err != nil { - panic(err) - } - mainArray := make([]*Main, size) - for i := 0; i < size; i++ { - mainArray[i] = &Main{Command: cluster.Servers[i]} - } - return mainArray -} - -// MustRunMain returns a new, running Main. Panic on error. -func MustRunMain() *Main { - m := NewMain() - m.Config.Metric.Diagnostics = false // Disable diagnostics. - if err := m.Run(); err != nil { - panic(err) - } - return m -} - -// Close closes the program and removes the underlying data directory. -func (m *Main) Close() error { - defer os.RemoveAll(m.Config.DataDir) - return m.Command.Close() -} - -// Reopen closes the program and reopens it. -func (m *Main) Reopen() error { - if err := m.Command.Close(); err != nil { - return err - } - - // Create new main with the same config. - config := m.Config - m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) - m.Server.Network = *test.Network - m.Config = config - - // Run new program. - if err := m.Run(); err != nil { - return err - } - return nil -} - -// RunWithTransport runs Main and returns the dynamically allocated gossip port. -func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) { - defer close(m.Started) - - m.Config.Cluster.Type = "gossip" - - /* - TEST: - - SetupServer (just static settings from config) - - OpenListener (sets Server.Name to use in gossip) - - NewTransport (gossip) - - SetupNetworking (does the gossip or static stuff) - uses Server.Name - - Open server - - PRODUCTION: - - SetupServer (just static settings from config) - - SetupNetworking (does the gossip or static stuff) - calls NewTransport - - Open server - calls OpenListener - */ - - // SetupServer - err = m.SetupServer() - if err != nil { - return seed, coord, err - } - - // Open server listener. - // This is used to set Server.Name, which is used as the node - // name for identifying a memberlist node. - err = m.Server.OpenListener() - if err != nil { - return seed, coord, err - } - - // Open gossip transport to use in SetupServer. - transport, err := gossip.NewTransport(host, bindPort) - if err != nil { - return seed, coord, err - } - m.GossipTransport = transport - - if joinSeed != "" { - m.Config.Gossip.Seed = joinSeed - } else { - m.Config.Gossip.Seed = transport.URI.String() - } - seed = m.Config.Gossip.Seed - - // SetupNetworking - err = m.SetupNetworking() - if err != nil { - return seed, coord, err - } - - if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { - return seed, coord, err - } - - if coordinator != nil { - coord = *coordinator - } else { - coord = m.Server.URI - } - m.Server.Cluster.Coordinator = coord - m.Server.Cluster.Static = false - - // Initialize server. - err = m.Server.Open() - if err != nil { - return seed, coord, err - } - - return seed, coord, nil -} - -// URL returns the base URL string for accessing the running program. -func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } - -// Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) - if err != nil { - panic(err) - } - return client -} - -// Query executes a query against the program through the HTTP API. -func (m *Main) Query(index, rawQuery, query string) (string, error) { - resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) - } - return resp.Body, nil -} - -// CreateDefinition. -func (m *Main) CreateDefinition(index, def, query string) (string, error) { - resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) - } - return resp.Body, nil -} - // SetCommand represents a command to set a bit. type SetCommand struct { ID uint64 @@ -619,32 +427,6 @@ func ParseConfig(s string) (pilosa.Config, error) { return c, err } -// MustDo executes http.Do() with an http.NewRequest(). Panic on error. -func MustDo(method, urlStr string, body string) *httpResponse { - req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) - if err != nil { - panic(err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - panic(err) - } - defer resp.Body.Close() - - buf, err := ioutil.ReadAll(resp.Body) - if err != nil { - panic(err) - } - - return &httpResponse{Response: resp, Body: string(buf)} -} - -// httpResponse is a wrapper for http.Response that holds the Body as a string. -type httpResponse struct { - *http.Response - Body string -} - // MustMarshalJSON marshals v into a string. Panic on error. func MustMarshalJSON(v interface{}) string { buf, err := json.Marshal(v) diff --git a/test/pilosa.go b/test/pilosa.go index d733e99c0..1278dc17a 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -2,9 +2,12 @@ package test import ( "bytes" + "fmt" "io" "io/ioutil" + "net/http" "os" + "strings" "testing" "github.com/pilosa/pilosa" @@ -47,12 +50,53 @@ func NewMain() *Main { return m } +func NewMainArrayWithCluster(size int) []*Main { + cluster, err := NewServerCluster(size) + if err != nil { + panic(err) + } + mainArray := make([]*Main, size) + for i := 0; i < size; i++ { + mainArray[i] = cluster.Servers[i] + } + return mainArray +} + +// MustRunMain returns a new, running Main. Panic on error. +func MustRunMain() *Main { + m := NewMain() + m.Config.Metric.Diagnostics = false // Disable diagnostics. + if err := m.Run(); err != nil { + panic(err) + } + return m +} + // Close closes the program and removes the underlying data directory. func (m *Main) Close() error { defer os.RemoveAll(m.Config.DataDir) return m.Command.Close() } +// Reopen closes the program and reopens it. +func (m *Main) Reopen() error { + if err := m.Command.Close(); err != nil { + return err + } + + // Create new main with the same config. + config := m.Config + m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) + m.Server.Network = *Network + m.Config = config + + // Run new program. + if err := m.Run(); err != nil { + return err + } + return nil +} + // RunWithTransport runs Main and returns the dynamically allocated gossip port. func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) { defer close(m.Started) @@ -128,6 +172,36 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coor return seed, coord, nil } +// URL returns the base URL string for accessing the running program. +func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } + +// Client returns a client to connect to the program. +func (m *Main) Client() *pilosa.InternalHTTPClient { + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + if err != nil { + panic(err) + } + return client +} + +// Query executes a query against the program through the HTTP API. +func (m *Main) Query(index, rawQuery, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + +// CreateDefinition. +func (m *Main) CreateDefinition(index, def, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + //////////////////////////////////////////////////////////////////////////////////// type Cluster struct { @@ -169,3 +243,29 @@ func NewServerCluster(size int) (cluster *Cluster, err error) { return cluster, nil } + +// MustDo executes http.Do() with an http.NewRequest(). Panic on error. +func MustDo(method, urlStr string, body string) *httpResponse { + req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) + if err != nil { + panic(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + + return &httpResponse{Response: resp, Body: string(buf)} +} + +// httpResponse is a wrapper for http.Response that holds the Body as a string. +type httpResponse struct { + *http.Response + Body string +} From f12527d53503afbfae1cd3b2965d874f3b37a98f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 23 Jan 2018 15:59:45 -0600 Subject: [PATCH 060/311] put a mutex around Cluster.State --- cluster.go | 34 +++++++++++++++++++++----------- cluster_test.go | 36 +++++++++++++++++----------------- server.go | 2 +- server/cluster_test.go | 44 +++++++++++++++++++++--------------------- test/cluster.go | 4 ++-- 5 files changed, 66 insertions(+), 54 deletions(-) diff --git a/cluster.go b/cluster.go index 4f29196bc..b2ed0e372 100644 --- a/cluster.go +++ b/cluster.go @@ -175,7 +175,7 @@ type Cluster struct { // Required for cluster Resize. Static bool // Static is primarily used for testing in a non-gossip environment. - State string + state string Coordinator URI Holder *Holder Broadcaster Broadcaster @@ -302,9 +302,21 @@ func (c *Cluster) setID(id string) { c.Topology.ClusterID = c.ID } +func (c *Cluster) State() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.state +} + +func (c *Cluster) SetState(state string) { + c.mu.Lock() + defer c.mu.Unlock() + c.setState(state) +} + func (c *Cluster) setState(state string) { // Ignore cases where the state hasn't changed. - if state == c.State { + if state == c.state { return } @@ -321,12 +333,12 @@ func (c *Cluster) setState(state string) { // - ClusterStateStarting // If state is RESIZING -> NORMAL then run cleanup. - if c.State == ClusterStateResizing { + if c.state == ClusterStateResizing { doCleanup = true } } - c.State = state + c.state = state // TODO: consider NOT running cleanup on an active node that has // been removed. @@ -373,7 +385,7 @@ func (c *Cluster) ReceiveNodeState(uri URI, state string) error { } // This method is really only useful during initial startup. - if c.State != ClusterStateStarting { + if c.State() != ClusterStateStarting { return nil } @@ -397,7 +409,7 @@ func (c *Cluster) ReceiveNodeState(uri URI, state string) error { func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ ClusterID: c.ID, - State: c.State, + State: c.state, NodeSet: encodeURIs(c.NodeSet()), } } @@ -763,7 +775,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { func (c *Cluster) Open() error { // Cluster always comes up in state STARTING until cluster membership is determined. - c.State = ClusterStateStarting + c.state = ClusterStateStarting // Load topology file if it exists. if err := c.loadTopology(); err != nil { @@ -820,7 +832,7 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State == ClusterStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) + return c.State() == ClusterStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) } func (c *Cluster) haveTopologyAgreement() bool { @@ -886,7 +898,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { } func (c *Cluster) setStateAndBroadcast(state string) error { - c.setState(state) + c.SetState(state) // Broadcast cluster status changes to the cluster. c.logger().Printf("broadcasting ClusterStatus: %s", state) return c.Broadcaster.SendSync(c.Status()) @@ -1618,7 +1630,7 @@ func (c *Cluster) NodeLeave(uri URI) error { return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.Coordinator) } - if c.State != ClusterStateNormal { + if c.State() != ClusterStateNormal { return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.State) } @@ -1683,7 +1695,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } } - c.setState(cs.State) + c.SetState(cs.State) c.markAsJoined() diff --git a/cluster_test.go b/cluster_test.go index a7fb77c83..b1b62c815 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -255,8 +255,8 @@ func TestCluster_ResizeStates(t *testing.T) { node := tc.Clusters[0] // Ensure that node comes up in state NORMAL. - if node.State != pilosa.ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State) + if node.State() != pilosa.ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State()) } expectedTop := &pilosa.Topology{ @@ -292,8 +292,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensure that node comes up in state NORMAL. - if node.State != pilosa.ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State) + if node.State() != pilosa.ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State()) } // Close TestCluster. @@ -344,10 +344,10 @@ func TestCluster_ResizeStates(t *testing.T) { node1 := tc.Clusters[1] // Ensure that nodes comes up in state NORMAL. - if node0.State != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) - } else if node1.State != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State) + if node0.State() != pilosa.ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) + } else if node1.State() != pilosa.ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State()) } expectedTop := &pilosa.Topology{ @@ -388,8 +388,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensure that node is in state STARTING before the other node joins. - if node0.State != pilosa.ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State) + if node0.State() != pilosa.ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State()) } // Expect an error by adding a node not in the topology. @@ -403,10 +403,10 @@ func TestCluster_ResizeStates(t *testing.T) { node2 := tc.Clusters[2] // Ensure that node comes up in state NORMAL. - if node0.State != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) - } else if node2.State != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State) + if node0.State() != pilosa.ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) + } else if node2.State() != pilosa.ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State()) } // Close TestCluster. @@ -470,10 +470,10 @@ func TestCluster_ResizeStates(t *testing.T) { node1 := tc.Clusters[1] // Ensure that nodes come up in state NORMAL. - if node0.State != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State) - } else if node1.State != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State) + if node0.State() != pilosa.ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) + } else if node1.State() != pilosa.ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State()) } expectedTop := &pilosa.Topology{ diff --git a/server.go b/server.go index 94f823345..aeb1e9519 100644 --- a/server.go +++ b/server.go @@ -510,7 +510,7 @@ func (s *Server) ClusterStatus() (proto.Message, error) { // HandleRemoteStatus receives incoming NodeStatus from remote nodes. func (s *Server) HandleRemoteStatus(pb proto.Message) error { // Ignore NodeStatus messages until the cluster is in a Normal state. - if s.Cluster.State != ClusterStateNormal { + if s.Cluster.State() != ClusterStateNormal { return nil } diff --git a/server/cluster_test.go b/server/cluster_test.go index 4d1d75348..98b723335 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -209,8 +209,8 @@ func TestClusterResize_EmptyNode(t *testing.T) { m0 := test.MustRunMain() defer m0.Close() - if m0.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State) + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State()) } } @@ -236,10 +236,10 @@ func TestClusterResize_EmptyNodes(t *testing.T) { t.Fatal(err) } - if m0.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) - } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) } } @@ -273,10 +273,10 @@ func TestClusterResize_AddNode(t *testing.T) { time.Sleep(1 * time.Second) - if m0.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) - } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) } }) t.Run("WithIndex", func(t *testing.T) { @@ -318,10 +318,10 @@ func TestClusterResize_AddNode(t *testing.T) { // Give the cluster time to settle. time.Sleep(1 * time.Second) - if m0.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) - } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) } }) t.Run("ContinuousSlices", func(t *testing.T) { @@ -373,10 +373,10 @@ func TestClusterResize_AddNode(t *testing.T) { // Give the cluster time to settle. time.Sleep(1 * time.Second) - if m0.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) - } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) } }) t.Run("SkippedSlice", func(t *testing.T) { @@ -428,10 +428,10 @@ func TestClusterResize_AddNode(t *testing.T) { // Give the cluster time to settle. time.Sleep(1 * time.Second) - if m0.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State) - } else if m1.Server.Cluster.State != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State) + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) } }) } diff --git a/test/cluster.go b/test/cluster.go index 5c95c962e..f7f4530ef 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -184,7 +184,7 @@ func (t *TestCluster) AddNode(saveTopology bool) error { } // Wait for the AddNode job to finish. - if c.State != pilosa.ClusterStateNormal { + if c.State() != pilosa.ClusterStateNormal { t.resizeDone = make(chan struct{}) <-t.resizeDone } @@ -266,7 +266,7 @@ func NewTestCluster(n int) *TestCluster { // SetState sets the state of the cluster on each node. func (t *TestCluster) SetState(state string) { for _, c := range t.Clusters { - c.State = state + c.SetState(state) } } From 0cb8b776403d81912972cda11275b32497b100af Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 23 Jan 2018 16:32:32 -0600 Subject: [PATCH 061/311] fix potential race condition: reading from a nil channel --- test/cluster.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/cluster.go b/test/cluster.go index f7f4530ef..ebe48fd35 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "path/filepath" "sort" + "sync" "time" "github.com/gogo/protobuf/proto" @@ -77,6 +78,8 @@ type TestCluster struct { common *commonClusterSettings + mu sync.RWMutex + resizing bool resizeDone chan struct{} } @@ -186,6 +189,9 @@ func (t *TestCluster) AddNode(saveTopology bool) error { // Wait for the AddNode job to finish. if c.State() != pilosa.ClusterStateNormal { t.resizeDone = make(chan struct{}) + t.mu.Lock() + t.resizing = true + t.mu.Unlock() <-t.resizeDone } } @@ -314,9 +320,11 @@ func (t *TestCluster) SendSync(pb proto.Message) error { for _, c := range t.Clusters { c.MergeClusterStatus(obj) } - if obj.State == pilosa.ClusterStateNormal && t.resizeDone != nil { + t.mu.RLock() + if obj.State == pilosa.ClusterStateNormal && t.resizing { close(t.resizeDone) } + t.mu.RUnlock() } return nil From 9b8d1ad4f9353f2891aad63702e4c489d963242c Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 29 Jan 2018 09:34:56 -0600 Subject: [PATCH 062/311] add CreateViewMessage for broadcaster --- broadcast.go | 39 ++-- frame.go | 23 +++ internal/private.pb.go | 393 ++++++++++++++++++++++++++++++++--------- internal/private.proto | 6 + server.go | 9 + 5 files changed, 369 insertions(+), 101 deletions(-) diff --git a/broadcast.go b/broadcast.go index 0a9b671ec..7d00f1107 100644 --- a/broadcast.go +++ b/broadcast.go @@ -130,19 +130,20 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error { // Broadcast message types. const ( - MessageTypeCreateSlice = 1 - MessageTypeCreateIndex = 2 - MessageTypeDeleteIndex = 3 - MessageTypeCreateFrame = 4 - MessageTypeDeleteFrame = 5 - MessageTypeCreateInputDefinition = 6 - MessageTypeDeleteInputDefinition = 7 - MessageTypeDeleteView = 8 - MessageTypeClusterStatus = 9 - MessageTypeResizeInstruction = 10 - MessageTypeResizeInstructionComplete = 11 - MessageTypeSetCoordinator = 12 - MessageTypeNodeState = 13 + MessageTypeCreateSlice = iota + MessageTypeCreateIndex + MessageTypeDeleteIndex + MessageTypeCreateFrame + MessageTypeDeleteFrame + MessageTypeCreateView + MessageTypeDeleteView + MessageTypeCreateInputDefinition + MessageTypeDeleteInputDefinition + MessageTypeClusterStatus + MessageTypeResizeInstruction + MessageTypeResizeInstructionComplete + MessageTypeSetCoordinator + MessageTypeNodeState ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -159,12 +160,14 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateFrame case *internal.DeleteFrameMessage: typ = MessageTypeDeleteFrame + case *internal.CreateViewMessage: + typ = MessageTypeCreateView + case *internal.DeleteViewMessage: + typ = MessageTypeDeleteView case *internal.CreateInputDefinitionMessage: typ = MessageTypeCreateInputDefinition case *internal.DeleteInputDefinitionMessage: typ = MessageTypeDeleteInputDefinition - case *internal.DeleteViewMessage: - typ = MessageTypeDeleteView case *internal.ClusterStatus: typ = MessageTypeClusterStatus case *internal.ResizeInstruction: @@ -201,12 +204,14 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateFrameMessage{} case MessageTypeDeleteFrame: m = &internal.DeleteFrameMessage{} + case MessageTypeCreateView: + m = &internal.CreateViewMessage{} + case MessageTypeDeleteView: + m = &internal.DeleteViewMessage{} case MessageTypeCreateInputDefinition: m = &internal.CreateInputDefinitionMessage{} case MessageTypeDeleteInputDefinition: m = &internal.DeleteInputDefinitionMessage{} - case MessageTypeDeleteView: - m = &internal.DeleteViewMessage{} case MessageTypeClusterStatus: m = &internal.ClusterStatus{} case MessageTypeResizeInstruction: diff --git a/frame.go b/frame.go index 84af0c4fd..b5807dc71 100644 --- a/frame.go +++ b/frame.go @@ -571,7 +571,30 @@ func (f *Frame) RecalculateCaches() { } // CreateViewIfNotExists returns the named view, creating it if necessary. +// Additionally, a CreateViewMessage is sent to the cluster. func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { + + view, err := f.CreateViewIfNotExistsBase(name) + if err != nil { + return nil, err + } + + // Broadcast view creation to the cluster. + err = f.broadcaster.SendSync( + &internal.CreateViewMessage{ + Index: f.index, + Frame: f.name, + View: name, + }) + if err != nil { + return nil, err + } + + return view, nil +} + +// CreateViewIfNotExistsBase returns the named view, creating it if necessary. +func (f *Frame) CreateViewIfNotExistsBase(name string) (*View, error) { // Don't create inverse views if they are not enabled. if !f.InverseEnabled() && IsInverseView(name) { return nil, ErrFrameInverseDisabled diff --git a/internal/private.pb.go b/internal/private.pb.go index 7e4fca2ea..905f004d5 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -34,6 +34,7 @@ NodeStatus ClusterStatus Field + CreateViewMessage DeleteViewMessage ResizeInstruction ResizeSource @@ -800,6 +801,38 @@ func (m *Field) GetMax() int64 { return 0 } +type CreateViewMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` +} + +func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } +func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } +func (*CreateViewMessage) ProtoMessage() {} +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } + +func (m *CreateViewMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateViewMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *CreateViewMessage) GetView() string { + if m != nil { + return m.View + } + return "" +} + type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -809,7 +842,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -844,7 +877,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -899,7 +932,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *ResizeSource) GetURI() *URI { if m != nil { @@ -946,7 +979,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{28} + return fileDescriptorPrivate, []int{29} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -978,7 +1011,7 @@ type SetCoordinatorMessage struct { func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func (m *SetCoordinatorMessage) GetOld() *URI { if m != nil { @@ -1002,7 +1035,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } func (m *Topology) GetNodeSet() []*URI { if m != nil { @@ -1044,6 +1077,7 @@ func init() { proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") proto.RegisterType((*Field)(nil), "internal.Field") + proto.RegisterType((*CreateViewMessage)(nil), "internal.CreateViewMessage") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") proto.RegisterType((*ResizeSource)(nil), "internal.ResizeSource") @@ -2063,6 +2097,42 @@ func (m *Field) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CreateViewMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.View) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + return i, nil +} + func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2810,6 +2880,24 @@ func (m *Field) Size() (n int) { return n } +func (m *CreateViewMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.View) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *DeleteViewMessage) Size() (n int) { var l int _ = l @@ -6399,6 +6487,143 @@ func (m *Field) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateViewMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateViewMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7422,82 +7647,82 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1222 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcf, 0x6f, 0x1b, 0xc5, - 0x17, 0xff, 0xae, 0xd7, 0x76, 0xec, 0xe7, 0xba, 0x75, 0xe7, 0x9b, 0x06, 0x27, 0x8a, 0x5c, 0x33, - 0x07, 0x12, 0x2a, 0x11, 0xc0, 0x95, 0x10, 0x04, 0x55, 0x82, 0xc6, 0xae, 0xba, 0x40, 0x92, 0x76, - 0x9c, 0x16, 0xc1, 0x01, 0x69, 0x62, 0x0f, 0xc9, 0x2a, 0xeb, 0x5d, 0xb3, 0x3b, 0x4e, 0xe2, 0x1e, - 0xb8, 0xc1, 0x01, 0xee, 0x88, 0x3b, 0xff, 0x0c, 0x47, 0xfe, 0x04, 0x14, 0xfe, 0x08, 0x24, 0x2e, - 0xa0, 0x79, 0x3b, 0xb3, 0xbb, 0xfe, 0x95, 0x34, 0xb9, 0xed, 0x7b, 0xf3, 0x79, 0x6f, 0x3e, 0xf3, - 0x7e, 0xcd, 0x2c, 0x54, 0x87, 0xa1, 0x7b, 0xca, 0xa5, 0xd8, 0x1a, 0x86, 0x81, 0x0c, 0x48, 0xc9, - 0xf5, 0xa5, 0x08, 0x7d, 0xee, 0xd1, 0x7d, 0x28, 0x3b, 0x7e, 0x5f, 0x9c, 0xef, 0x0a, 0xc9, 0x49, - 0x13, 0x2a, 0x3b, 0x81, 0x37, 0x1a, 0xf8, 0x5f, 0xf0, 0x43, 0xe1, 0xd5, 0xad, 0xa6, 0xb5, 0x59, - 0x66, 0x59, 0x95, 0x42, 0x1c, 0xb8, 0x03, 0xf1, 0x7c, 0xc4, 0x7d, 0x39, 0x1a, 0xd4, 0x73, 0x31, - 0x22, 0xa3, 0xa2, 0xff, 0x58, 0x50, 0x7e, 0x12, 0xf2, 0x81, 0x40, 0x8f, 0x6b, 0x50, 0x62, 0xc1, - 0x59, 0xd6, 0x5d, 0x22, 0x93, 0xb7, 0xe0, 0xb6, 0xe3, 0x9f, 0x8a, 0x30, 0x12, 0x1d, 0x9f, 0x1f, - 0x7a, 0xa2, 0x8f, 0xee, 0x4a, 0x6c, 0x4a, 0x4b, 0xd6, 0xa1, 0xbc, 0xc3, 0x7b, 0xc7, 0xe2, 0x60, - 0x3c, 0x14, 0x75, 0x1b, 0x9d, 0xa4, 0x8a, 0x64, 0xb5, 0xeb, 0xbe, 0x12, 0xf5, 0x7c, 0xd3, 0xda, - 0xac, 0xb2, 0x54, 0x31, 0xcd, 0xb7, 0x30, 0xc3, 0x97, 0x50, 0xb8, 0xc5, 0xb8, 0x7f, 0x94, 0x70, - 0x28, 0x22, 0x87, 0x09, 0x1d, 0xd9, 0x80, 0xe2, 0x13, 0x57, 0x78, 0xfd, 0xa8, 0xbe, 0xd4, 0xb4, - 0x37, 0x2b, 0xad, 0x3b, 0x5b, 0x26, 0x7e, 0x5b, 0xa8, 0x67, 0x7a, 0x99, 0x52, 0xb8, 0xed, 0x0c, - 0x86, 0x41, 0x28, 0x99, 0x88, 0x86, 0x81, 0x1f, 0x09, 0x52, 0x03, 0xbb, 0x13, 0x86, 0xfa, 0xec, - 0xea, 0x93, 0x7e, 0x0f, 0xb5, 0xc7, 0x5e, 0xd0, 0x3b, 0x69, 0x73, 0xc9, 0x99, 0xf8, 0x6e, 0x24, - 0x22, 0x49, 0x96, 0xa1, 0x80, 0x59, 0xd0, 0xb8, 0x58, 0x50, 0x5a, 0x8c, 0xa4, 0x0e, 0x73, 0x2c, - 0x28, 0x2d, 0xda, 0x63, 0x28, 0xf2, 0x2c, 0x16, 0x94, 0xb6, 0xeb, 0xb9, 0xbd, 0x38, 0x04, 0x79, - 0x16, 0x0b, 0x84, 0x40, 0xfe, 0xa5, 0x2b, 0xce, 0xf4, 0xb9, 0xf1, 0x9b, 0x3a, 0x70, 0x37, 0xb3, - 0xbf, 0xa6, 0xb9, 0x02, 0x45, 0x16, 0x9c, 0x39, 0xed, 0xa8, 0x6e, 0x35, 0xed, 0xcd, 0x3c, 0xd3, - 0x12, 0x46, 0x17, 0xd3, 0xaf, 0x96, 0x72, 0xb8, 0x94, 0x2a, 0xe8, 0x2a, 0x14, 0x30, 0xd4, 0xea, - 0x94, 0xa9, 0xad, 0xfa, 0xa4, 0xff, 0x5a, 0x50, 0xde, 0xe5, 0xe7, 0x48, 0x23, 0x22, 0x8f, 0xa0, - 0xd4, 0x95, 0xdc, 0xef, 0xf3, 0xb0, 0x8f, 0xa0, 0x4a, 0xeb, 0xcd, 0x34, 0x84, 0x09, 0x6c, 0xcb, - 0x60, 0x3a, 0xbe, 0x0c, 0xc7, 0x2c, 0x31, 0x21, 0xdb, 0xb0, 0xa4, 0x6b, 0x02, 0x39, 0x54, 0x5a, - 0xcd, 0x79, 0xd6, 0x49, 0xd9, 0x28, 0x63, 0x63, 0xb0, 0xf6, 0x31, 0x54, 0x27, 0xdc, 0x2a, 0xae, - 0x27, 0x62, 0x6c, 0x32, 0x72, 0x22, 0xc6, 0x2a, 0x76, 0xa7, 0xdc, 0x1b, 0xc5, 0x71, 0xce, 0xb3, - 0x58, 0xd8, 0xce, 0x7d, 0x68, 0xad, 0x6d, 0xc3, 0xad, 0xac, 0xd7, 0xeb, 0xd8, 0xd2, 0x6f, 0x80, - 0xec, 0x84, 0x82, 0x4b, 0x81, 0xf4, 0x76, 0x45, 0x14, 0xf1, 0x23, 0xb1, 0x38, 0xd3, 0x71, 0xf6, - 0x72, 0xd9, 0xec, 0xad, 0x43, 0xd9, 0x89, 0xcc, 0xc1, 0x6d, 0xac, 0xcb, 0x54, 0x41, 0x1f, 0x00, - 0x69, 0x0b, 0x4f, 0x48, 0xa1, 0xfb, 0xf7, 0x12, 0xff, 0xb4, 0x6b, 0xb8, 0x5c, 0x8d, 0x25, 0x1b, - 0x90, 0x57, 0xad, 0x8b, 0x54, 0x2a, 0xad, 0xff, 0xa7, 0x91, 0x4e, 0xe6, 0x04, 0x43, 0x00, 0x75, - 0x8d, 0x53, 0xdd, 0xee, 0x57, 0x1c, 0x70, 0x4e, 0x29, 0x9b, 0xad, 0xec, 0xe9, 0xad, 0x92, 0x01, - 0xa2, 0xb7, 0xfa, 0xc4, 0x9c, 0xf5, 0xa6, 0x5b, 0xd1, 0xaf, 0xb5, 0x56, 0xb5, 0xc4, 0x9e, 0x5a, - 0x8d, 0x6d, 0xf0, 0x7b, 0xf1, 0x91, 0xa7, 0x78, 0x28, 0xdf, 0xaa, 0x87, 0xa2, 0xba, 0xdd, 0xb4, - 0x95, 0x6f, 0x14, 0xe8, 0x43, 0x28, 0x76, 0x7b, 0xc7, 0x62, 0xc0, 0xc9, 0xdb, 0xaa, 0x50, 0xfb, - 0xe2, 0x5c, 0x44, 0xba, 0xcc, 0xef, 0x4c, 0x85, 0x8f, 0x99, 0x75, 0xfa, 0xb3, 0xa5, 0xd9, 0x2f, - 0x60, 0x54, 0xc4, 0xbd, 0xa3, 0x7a, 0x7e, 0x66, 0xe2, 0x28, 0x3d, 0xd3, 0xcb, 0xa4, 0x03, 0x35, - 0xc7, 0x1f, 0x8e, 0x64, 0x5b, 0x7c, 0xeb, 0xfa, 0xae, 0x74, 0x03, 0x3f, 0xaa, 0x17, 0xd1, 0x64, - 0x35, 0xbb, 0xf5, 0x04, 0x82, 0xcd, 0x98, 0xd0, 0x1f, 0x2d, 0xb8, 0x33, 0xa5, 0xbc, 0x82, 0x57, - 0xee, 0x72, 0x5e, 0x1f, 0x24, 0x23, 0xd3, 0x46, 0x60, 0x63, 0x21, 0x9b, 0xc9, 0x09, 0xfa, 0x9b, - 0x05, 0xcb, 0xf3, 0x00, 0x73, 0xd9, 0x34, 0x00, 0x9e, 0x85, 0xee, 0x80, 0x87, 0xe3, 0xcf, 0xc5, - 0x58, 0xdf, 0x1e, 0x19, 0x0d, 0xf9, 0x12, 0x56, 0xa6, 0x7c, 0x7d, 0xda, 0x8b, 0x43, 0x14, 0x93, - 0xba, 0xbf, 0x90, 0x54, 0x8c, 0x63, 0x0b, 0xcc, 0xe9, 0xdf, 0x16, 0xdc, 0x9b, 0xbb, 0x94, 0x56, - 0x9f, 0x95, 0x2d, 0xf4, 0x07, 0x50, 0x7b, 0xa9, 0x06, 0x43, 0x5b, 0x44, 0xd2, 0xf5, 0xb9, 0x42, - 0xea, 0xf2, 0x9c, 0xd1, 0x13, 0x07, 0x4a, 0xa8, 0xdb, 0xe5, 0x43, 0x4d, 0xf3, 0x9d, 0x2b, 0x68, - 0x6e, 0x19, 0xbc, 0x9e, 0x9b, 0x46, 0x54, 0x64, 0x70, 0x8e, 0x9b, 0x4b, 0x01, 0x05, 0x35, 0x11, - 0x27, 0x0c, 0xae, 0x35, 0xd5, 0x02, 0x58, 0x37, 0x93, 0x64, 0x82, 0xc9, 0xe5, 0x3d, 0xf9, 0x11, - 0x40, 0x0a, 0xd5, 0xed, 0x7e, 0x49, 0x7d, 0x66, 0xc0, 0xf4, 0x29, 0xac, 0x9b, 0x31, 0x77, 0x8d, - 0x0d, 0x4d, 0xb5, 0xe4, 0xd2, 0x6a, 0xa1, 0x1d, 0xb0, 0x5f, 0x30, 0x47, 0x5d, 0x75, 0xd8, 0xad, - 0x26, 0x45, 0x5a, 0x52, 0x26, 0x4f, 0x83, 0x48, 0x1a, 0x13, 0xf5, 0xad, 0x74, 0xcf, 0x82, 0x50, - 0x22, 0xe3, 0x2a, 0xc3, 0x6f, 0xea, 0x40, 0x6d, 0x2f, 0xe8, 0x8b, 0xae, 0xe4, 0x32, 0x99, 0x44, - 0xf7, 0xd1, 0x35, 0x3a, 0xac, 0xb4, 0xaa, 0xe9, 0xc1, 0x5e, 0x30, 0x87, 0xe1, 0xa6, 0x6a, 0xc0, - 0x2b, 0x03, 0x33, 0x94, 0x50, 0xa0, 0x3f, 0x59, 0x00, 0xc6, 0xd7, 0x28, 0xba, 0xda, 0xcb, 0xfb, - 0x99, 0x3b, 0x75, 0x76, 0x58, 0x25, 0x4b, 0x2c, 0x73, 0xf3, 0x6e, 0x9a, 0xd9, 0xa4, 0xa3, 0x5e, - 0x4b, 0xf1, 0xb1, 0x5e, 0x9f, 0x9f, 0x53, 0x0f, 0xaa, 0x3b, 0xde, 0x28, 0x92, 0x22, 0xd4, 0x74, - 0x12, 0xce, 0x56, 0x86, 0x33, 0xd9, 0x80, 0x25, 0xa4, 0x2c, 0xa4, 0x1e, 0x01, 0x53, 0x44, 0xcd, - 0x2a, 0x3e, 0x1d, 0x62, 0x7f, 0x4e, 0x3b, 0x79, 0xb6, 0x19, 0x05, 0xed, 0x42, 0x61, 0x71, 0x5f, - 0x13, 0xc8, 0xe3, 0x63, 0x4f, 0xa7, 0x02, 0xdf, 0x79, 0x35, 0xb0, 0x77, 0xdd, 0xb8, 0x76, 0x6c, - 0xa6, 0x3e, 0x51, 0xc3, 0xcf, 0xb1, 0xb6, 0x95, 0x86, 0xab, 0x6b, 0xee, 0x6e, 0x5c, 0x2b, 0x6a, - 0x2e, 0xdf, 0xe4, 0x42, 0x32, 0xef, 0x25, 0x3b, 0xf3, 0x5e, 0xfa, 0x25, 0x07, 0x77, 0x99, 0x88, - 0xdc, 0x57, 0xc2, 0xf1, 0x23, 0x19, 0x8e, 0x92, 0x3e, 0xff, 0x2c, 0x38, 0x74, 0xda, 0xe8, 0xd5, - 0x66, 0xb1, 0x60, 0x32, 0x98, 0x5b, 0x98, 0xc1, 0x77, 0xd5, 0x0b, 0x3b, 0x08, 0xfb, 0xaa, 0xd9, - 0x83, 0x50, 0xe7, 0x64, 0x0a, 0x98, 0x45, 0x90, 0xf7, 0x60, 0xa9, 0x1b, 0x8c, 0xc2, 0x5e, 0x72, - 0x13, 0xac, 0xa4, 0xe0, 0x98, 0x55, 0xbc, 0xcc, 0x0c, 0x2c, 0x93, 0xf1, 0xc2, 0xe5, 0x19, 0x27, - 0x8f, 0xa6, 0x32, 0x8e, 0x6f, 0xdf, 0x4a, 0xeb, 0x8d, 0xd4, 0x60, 0x62, 0x99, 0x4d, 0xa2, 0xe9, - 0x0f, 0x16, 0xdc, 0xca, 0x52, 0x78, 0xad, 0x2e, 0x88, 0x53, 0x91, 0x9b, 0x9b, 0x0a, 0x7b, 0x5e, - 0x2a, 0xf2, 0x69, 0x2a, 0xd2, 0x67, 0x52, 0x21, 0xf3, 0x4c, 0xa2, 0xc7, 0xb0, 0x3a, 0x93, 0x9f, - 0x9d, 0x60, 0x30, 0x54, 0x85, 0x70, 0xd3, 0x3c, 0x2d, 0x43, 0xa1, 0x13, 0x86, 0x3a, 0x43, 0x65, - 0x16, 0x0b, 0xf4, 0x2b, 0xb8, 0xd7, 0x15, 0x32, 0x93, 0x9e, 0x4c, 0xff, 0xef, 0x7b, 0xfd, 0x05, - 0x27, 0xdf, 0xf7, 0xfa, 0x0a, 0xb0, 0x27, 0xce, 0x16, 0x6c, 0xb8, 0x27, 0xce, 0xe8, 0x73, 0x28, - 0x1d, 0x04, 0xc3, 0xc0, 0x0b, 0x8e, 0xc6, 0xd9, 0x16, 0xb3, 0x5e, 0xbf, 0xc5, 0x72, 0x53, 0x2d, - 0xf6, 0xb8, 0xf6, 0xfb, 0x45, 0xc3, 0xfa, 0xe3, 0xa2, 0x61, 0xfd, 0x79, 0xd1, 0xb0, 0x7e, 0xfd, - 0xab, 0xf1, 0xbf, 0xc3, 0x22, 0xfe, 0xfd, 0x3d, 0xfc, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xb9, 0x4f, - 0x86, 0xdb, 0x0e, 0x0e, 0x00, 0x00, + // 1229 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0xb6, 0x63, 0x3f, 0xc7, 0x8d, 0x33, 0xa4, 0xc1, 0x89, 0x22, 0xd7, 0xcc, 0x81, + 0x84, 0x4a, 0x04, 0x70, 0x25, 0x04, 0x41, 0x95, 0xa0, 0xb1, 0xab, 0x2e, 0x90, 0xa4, 0x1d, 0xa7, + 0x45, 0x70, 0x40, 0x9a, 0xd8, 0x43, 0xb2, 0xca, 0x7a, 0xd7, 0xec, 0x8e, 0x93, 0xb8, 0x07, 0x6e, + 0x70, 0x80, 0x3b, 0xe2, 0xce, 0x97, 0xe1, 0xc8, 0x47, 0x40, 0xe1, 0x43, 0x20, 0x71, 0x01, 0xcd, + 0xbf, 0xdd, 0xf5, 0xda, 0x4e, 0x9a, 0x88, 0xdb, 0xbe, 0x37, 0xbf, 0xf7, 0xe6, 0x37, 0xef, 0xdf, + 0xcc, 0x42, 0x75, 0x18, 0xba, 0x67, 0x94, 0xb3, 0xed, 0x61, 0x18, 0xf0, 0x00, 0x95, 0x5c, 0x9f, + 0xb3, 0xd0, 0xa7, 0x1e, 0x3e, 0x80, 0xb2, 0xe3, 0xf7, 0xd9, 0xc5, 0x1e, 0xe3, 0x14, 0x35, 0xa1, + 0xb2, 0x1b, 0x78, 0xa3, 0x81, 0xff, 0x05, 0x3d, 0x62, 0x5e, 0xdd, 0x6a, 0x5a, 0x5b, 0x65, 0x92, + 0x56, 0x09, 0xc4, 0xa1, 0x3b, 0x60, 0xcf, 0x46, 0xd4, 0xe7, 0xa3, 0x41, 0x3d, 0xa7, 0x10, 0x29, + 0x15, 0xfe, 0xc7, 0x82, 0xf2, 0xe3, 0x90, 0x0e, 0x98, 0xf4, 0xb8, 0x0e, 0x25, 0x12, 0x9c, 0xa7, + 0xdd, 0xc5, 0x32, 0x7a, 0x0b, 0xee, 0x38, 0xfe, 0x19, 0x0b, 0x23, 0xd6, 0xf1, 0xe9, 0x91, 0xc7, + 0xfa, 0xd2, 0x5d, 0x89, 0x64, 0xb4, 0x68, 0x03, 0xca, 0xbb, 0xb4, 0x77, 0xc2, 0x0e, 0xc7, 0x43, + 0x56, 0xb7, 0xa5, 0x93, 0x44, 0x11, 0xaf, 0x76, 0xdd, 0x97, 0xac, 0x9e, 0x6f, 0x5a, 0x5b, 0x55, + 0x92, 0x28, 0xb2, 0x7c, 0x0b, 0x53, 0x7c, 0x11, 0x86, 0x45, 0x42, 0xfd, 0xe3, 0x98, 0x43, 0x51, + 0x72, 0x98, 0xd0, 0xa1, 0x4d, 0x28, 0x3e, 0x76, 0x99, 0xd7, 0x8f, 0xea, 0x0b, 0x4d, 0x7b, 0xab, + 0xd2, 0x5a, 0xda, 0x36, 0xf1, 0xdb, 0x96, 0x7a, 0xa2, 0x97, 0x31, 0x86, 0x3b, 0xce, 0x60, 0x18, + 0x84, 0x9c, 0xb0, 0x68, 0x18, 0xf8, 0x11, 0x43, 0x35, 0xb0, 0x3b, 0x61, 0xa8, 0xcf, 0x2e, 0x3e, + 0xf1, 0xf7, 0x50, 0x7b, 0xe4, 0x05, 0xbd, 0xd3, 0x36, 0xe5, 0x94, 0xb0, 0xef, 0x46, 0x2c, 0xe2, + 0x68, 0x05, 0x0a, 0x32, 0x0b, 0x1a, 0xa7, 0x04, 0xa1, 0x95, 0x91, 0xd4, 0x61, 0x56, 0x82, 0xd0, + 0x4a, 0x7b, 0x19, 0x8a, 0x3c, 0x51, 0x82, 0xd0, 0x76, 0x3d, 0xb7, 0xa7, 0x42, 0x90, 0x27, 0x4a, + 0x40, 0x08, 0xf2, 0x2f, 0x5c, 0x76, 0xae, 0xcf, 0x2d, 0xbf, 0xb1, 0x03, 0xcb, 0xa9, 0xfd, 0x35, + 0xcd, 0x55, 0x28, 0x92, 0xe0, 0xdc, 0x69, 0x47, 0x75, 0xab, 0x69, 0x6f, 0xe5, 0x89, 0x96, 0x64, + 0x74, 0x65, 0xfa, 0xc5, 0x52, 0x4e, 0x2e, 0x25, 0x0a, 0xbc, 0x06, 0x05, 0x19, 0x6a, 0x71, 0xca, + 0xc4, 0x56, 0x7c, 0xe2, 0x7f, 0x2d, 0x28, 0xef, 0xd1, 0x0b, 0x49, 0x23, 0x42, 0x0f, 0xa1, 0xd4, + 0xe5, 0xd4, 0xef, 0xd3, 0xb0, 0x2f, 0x41, 0x95, 0xd6, 0x9b, 0x49, 0x08, 0x63, 0xd8, 0xb6, 0xc1, + 0x74, 0x7c, 0x1e, 0x8e, 0x49, 0x6c, 0x82, 0x76, 0x60, 0x41, 0xd7, 0x84, 0xe4, 0x50, 0x69, 0x35, + 0x67, 0x59, 0xc7, 0x65, 0x23, 0x8c, 0x8d, 0xc1, 0xfa, 0xc7, 0x50, 0x9d, 0x70, 0x2b, 0xb8, 0x9e, + 0xb2, 0xb1, 0xc9, 0xc8, 0x29, 0x1b, 0x8b, 0xd8, 0x9d, 0x51, 0x6f, 0xa4, 0xe2, 0x9c, 0x27, 0x4a, + 0xd8, 0xc9, 0x7d, 0x68, 0xad, 0xef, 0xc0, 0x62, 0xda, 0xeb, 0x4d, 0x6c, 0xf1, 0x37, 0x80, 0x76, + 0x43, 0x46, 0x39, 0x93, 0xf4, 0xf6, 0x58, 0x14, 0xd1, 0x63, 0x36, 0x3f, 0xd3, 0x2a, 0x7b, 0xb9, + 0x74, 0xf6, 0x36, 0xa0, 0xec, 0x44, 0xe6, 0xe0, 0xb6, 0xac, 0xcb, 0x44, 0x81, 0xef, 0x03, 0x6a, + 0x33, 0x8f, 0x71, 0xa6, 0xfb, 0xf7, 0x0a, 0xff, 0xb8, 0x6b, 0xb8, 0x5c, 0x8f, 0x45, 0x9b, 0x90, + 0x17, 0xad, 0x2b, 0xa9, 0x54, 0x5a, 0xaf, 0x27, 0x91, 0x8e, 0xe7, 0x04, 0x91, 0x00, 0xec, 0x1a, + 0xa7, 0xba, 0xdd, 0xaf, 0x39, 0xe0, 0x8c, 0x52, 0x36, 0x5b, 0xd9, 0xd9, 0xad, 0xe2, 0x01, 0xa2, + 0xb7, 0xfa, 0xc4, 0x9c, 0xf5, 0xb6, 0x5b, 0xe1, 0xaf, 0xb5, 0x56, 0xb4, 0xc4, 0xbe, 0x58, 0x55, + 0x36, 0xf2, 0x7b, 0xfe, 0x91, 0x33, 0x3c, 0x84, 0x6f, 0xd1, 0x43, 0x51, 0xdd, 0x6e, 0xda, 0xc2, + 0xb7, 0x14, 0xf0, 0x03, 0x28, 0x76, 0x7b, 0x27, 0x6c, 0x40, 0xd1, 0xdb, 0xa2, 0x50, 0xfb, 0xec, + 0x82, 0x45, 0xba, 0xcc, 0x97, 0x32, 0xe1, 0x23, 0x66, 0x1d, 0xff, 0x6c, 0x69, 0xf6, 0x73, 0x18, + 0x15, 0xe5, 0xde, 0x51, 0x3d, 0x3f, 0x35, 0x71, 0x84, 0x9e, 0xe8, 0x65, 0xd4, 0x81, 0x9a, 0xe3, + 0x0f, 0x47, 0xbc, 0xcd, 0xbe, 0x75, 0x7d, 0x97, 0xbb, 0x81, 0x1f, 0xd5, 0x8b, 0xd2, 0x64, 0x2d, + 0xbd, 0xf5, 0x04, 0x82, 0x4c, 0x99, 0xe0, 0x1f, 0x2d, 0x58, 0xca, 0x28, 0xaf, 0xe1, 0x95, 0xbb, + 0x9a, 0xd7, 0x07, 0xf1, 0xc8, 0xb4, 0x25, 0xb0, 0x31, 0x97, 0xcd, 0xe4, 0x04, 0xfd, 0xcd, 0x82, + 0x95, 0x59, 0x80, 0x99, 0x6c, 0x1a, 0x00, 0x4f, 0x43, 0x77, 0x40, 0xc3, 0xf1, 0xe7, 0x6c, 0xac, + 0x6f, 0x8f, 0x94, 0x06, 0x7d, 0x09, 0xab, 0x19, 0x5f, 0x9f, 0xf6, 0x54, 0x88, 0x14, 0xa9, 0x7b, + 0x73, 0x49, 0x29, 0x1c, 0x99, 0x63, 0x8e, 0xff, 0xb6, 0xe0, 0xee, 0xcc, 0xa5, 0xa4, 0xfa, 0xac, + 0x74, 0xa1, 0xdf, 0x87, 0xda, 0x0b, 0x31, 0x18, 0xda, 0x2c, 0xe2, 0xae, 0x4f, 0x05, 0x52, 0x97, + 0xe7, 0x94, 0x1e, 0x39, 0x50, 0x92, 0xba, 0x3d, 0x3a, 0xd4, 0x34, 0xdf, 0xb9, 0x86, 0xe6, 0xb6, + 0xc1, 0xeb, 0xb9, 0x69, 0x44, 0x41, 0x46, 0xce, 0x71, 0x73, 0x29, 0x48, 0x41, 0x4c, 0xc4, 0x09, + 0x83, 0x1b, 0x4d, 0xb5, 0x00, 0x36, 0xcc, 0x24, 0x99, 0x60, 0x72, 0x75, 0x4f, 0x7e, 0x04, 0x90, + 0x40, 0x75, 0xbb, 0x5f, 0x51, 0x9f, 0x29, 0x30, 0x7e, 0x02, 0x1b, 0x66, 0xcc, 0xdd, 0x60, 0x43, + 0x53, 0x2d, 0xb9, 0xa4, 0x5a, 0x70, 0x07, 0xec, 0xe7, 0xc4, 0x11, 0x57, 0x9d, 0xec, 0x56, 0x93, + 0x22, 0x2d, 0x09, 0x93, 0x27, 0x41, 0xc4, 0x8d, 0x89, 0xf8, 0x16, 0xba, 0xa7, 0x41, 0xc8, 0x25, + 0xe3, 0x2a, 0x91, 0xdf, 0xd8, 0x81, 0xda, 0x7e, 0xd0, 0x67, 0x5d, 0x4e, 0x79, 0x3c, 0x89, 0xee, + 0x49, 0xd7, 0xd2, 0x61, 0xa5, 0x55, 0x4d, 0x0e, 0xf6, 0x9c, 0x38, 0x44, 0x6e, 0x2a, 0x06, 0xbc, + 0x30, 0x30, 0x43, 0x49, 0x0a, 0xf8, 0x27, 0x0b, 0xc0, 0xf8, 0x1a, 0x45, 0xd7, 0x7b, 0x79, 0x3f, + 0x75, 0xa7, 0x4e, 0x0f, 0xab, 0x78, 0x89, 0xa4, 0x6e, 0xde, 0x2d, 0x33, 0x9b, 0x74, 0xd4, 0x6b, + 0x09, 0x5e, 0xe9, 0xf5, 0xf9, 0x29, 0xf6, 0xa0, 0xba, 0xeb, 0x8d, 0x22, 0xce, 0x42, 0x4d, 0x27, + 0xe6, 0x6c, 0xa5, 0x38, 0xa3, 0x4d, 0x58, 0x90, 0x94, 0x19, 0xd7, 0x23, 0x20, 0x43, 0xd4, 0xac, + 0xca, 0xa7, 0x83, 0xf2, 0xe7, 0xb4, 0xe3, 0x67, 0x9b, 0x51, 0xe0, 0x2e, 0x14, 0xe6, 0xf7, 0x35, + 0x82, 0xbc, 0x7c, 0xec, 0xe9, 0x54, 0xc8, 0x77, 0x5e, 0x0d, 0xec, 0x3d, 0x57, 0xd5, 0x8e, 0x4d, + 0xc4, 0xa7, 0xd4, 0xd0, 0x0b, 0x59, 0xdb, 0x42, 0x43, 0xc5, 0x35, 0xb7, 0xac, 0x8a, 0x53, 0xcc, + 0xe5, 0xdb, 0x5c, 0x48, 0xe6, 0xbd, 0x64, 0xa7, 0xde, 0x4b, 0x5d, 0x58, 0x56, 0x05, 0xf8, 0x7f, + 0x3a, 0xfd, 0x25, 0x07, 0xcb, 0x84, 0x45, 0xee, 0x4b, 0xe6, 0xf8, 0x11, 0x0f, 0x47, 0xf1, 0xf0, + 0xf8, 0x2c, 0x38, 0x72, 0xda, 0xd2, 0xab, 0x4d, 0x94, 0x60, 0xca, 0x22, 0x37, 0xb7, 0x2c, 0xde, + 0x15, 0xcf, 0xf6, 0x20, 0xec, 0x8b, 0x09, 0x12, 0x84, 0x3a, 0xd1, 0x19, 0x60, 0x1a, 0x81, 0xde, + 0x83, 0x85, 0x6e, 0x30, 0x0a, 0x7b, 0xf1, 0xf5, 0xb2, 0x9a, 0x80, 0x15, 0x2b, 0xb5, 0x4c, 0x0c, + 0x2c, 0x55, 0x46, 0x85, 0xab, 0xcb, 0x08, 0x3d, 0xcc, 0x94, 0x91, 0x7c, 0x50, 0x57, 0x5a, 0x6f, + 0x24, 0x06, 0x13, 0xcb, 0x64, 0x12, 0x8d, 0x7f, 0xb0, 0x60, 0x31, 0x4d, 0xe1, 0x95, 0x5a, 0x4b, + 0xa5, 0x22, 0x37, 0x33, 0x15, 0xf6, 0xac, 0x54, 0xe4, 0x93, 0x54, 0x24, 0x6f, 0xaf, 0x42, 0xea, + 0xed, 0x85, 0x4f, 0x60, 0x6d, 0x2a, 0x3f, 0xbb, 0xc1, 0x60, 0x28, 0x0a, 0xe1, 0xb6, 0x79, 0x5a, + 0x81, 0x42, 0x27, 0x0c, 0x75, 0x86, 0xca, 0x44, 0x09, 0xf8, 0x2b, 0xb8, 0xdb, 0x65, 0x3c, 0x95, + 0x9e, 0xd4, 0x50, 0x39, 0xf0, 0xfa, 0x73, 0x4e, 0x7e, 0xe0, 0xf5, 0x05, 0x60, 0x9f, 0x9d, 0xcf, + 0xd9, 0x70, 0x9f, 0x9d, 0xe3, 0x67, 0x50, 0x3a, 0x0c, 0x86, 0x81, 0x17, 0x1c, 0x8f, 0xd3, 0x7d, + 0x6b, 0xbd, 0x7a, 0xdf, 0xe6, 0x32, 0x7d, 0xfb, 0xa8, 0xf6, 0xfb, 0x65, 0xc3, 0xfa, 0xe3, 0xb2, + 0x61, 0xfd, 0x79, 0xd9, 0xb0, 0x7e, 0xfd, 0xab, 0xf1, 0xda, 0x51, 0x51, 0xfe, 0x52, 0x3e, 0xf8, + 0x2f, 0x00, 0x00, 0xff, 0xff, 0x59, 0x94, 0x30, 0x6d, 0x63, 0x0e, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index cf1b56867..4877707a2 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -145,6 +145,12 @@ message Field { int64 Max = 4; } +message CreateViewMessage { + string Index = 1; + string Frame = 2; + string View = 3; +} + message DeleteViewMessage { string Index = 1; string Frame = 2; diff --git a/server.go b/server.go index aeb1e9519..6b7d49cc5 100644 --- a/server.go +++ b/server.go @@ -408,6 +408,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.CreateViewMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + if f == nil { + return fmt.Errorf("Local Frame not found: %s", obj.Frame) + } + _, err := f.CreateViewIfNotExistsBase(obj.View) + if err != nil { + return err + } case *internal.DeleteViewMessage: f := s.Holder.Frame(obj.Index, obj.Frame) if f == nil { From d3590098f640095d94b99cb55f448673b08e7c26 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 29 Jan 2018 10:31:49 -0600 Subject: [PATCH 063/311] perform FrameRestore on both nodes in test cluster --- server/server_test.go | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 379175c75..352fc6121 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -270,10 +270,11 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.NewMainArrayWithCluster(2) - m0 := mains1[0] + m10 := mains1[0] + m11 := mains1[1] // Create frames. - client := m0.Client() + client := m10.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } @@ -282,7 +283,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Write data on first cluster. - if _, err := m0.Query("i", "", ` + if _, err := m10.Query("i", "", ` SetBit(rowID=1, frame="f", columnID=100) SetBit(rowID=1, frame="f", columnID=1000) SetBit(rowID=1, frame="f", columnID=100000) @@ -295,7 +296,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Query row on first cluster. - if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m10.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { t.Fatal("bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -303,26 +304,37 @@ func TestMain_FrameRestore(t *testing.T) { // Start second cluster. mains2 := test.NewMainArrayWithCluster(2) - m2 := mains2[0] - defer m2.Close() + m20 := mains2[0] + defer m20.Close() + m21 := mains2[1] + defer m21.Close() // Import from first cluster. - client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { t.Fatal("new client:", err) } - if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + if err != nil { + t.Fatal("new client:", err) + } + + if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create new index:", err) } - if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { t.Fatal("create new frame:", err) } - if err := client.RestoreFrame(context.Background(), m0.Server.URI.HostPort(), "i", "f"); err != nil { + + if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil { + t.Fatal("restore frame:", err) + } + if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil { t.Fatal("restore frame:", err) } // Query row on second cluster. - if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m20.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { t.Fatal("another bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("2unexpected result: %s", res) From 216ba7a41ec42ef3407ab5a59555fb4ce2c0757d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 1 Feb 2018 12:59:42 -0600 Subject: [PATCH 064/311] Use NodeID instead of URI for node identification --- broadcast.go | 13 +- client_test.go | 22 +- cluster.go | 517 +++++++++++++++------------ cluster_internal_test.go | 122 ++++--- cluster_test.go | 151 ++++---- ctl/backup_test.go | 4 +- ctl/export_test.go | 6 +- ctl/import_test.go | 12 +- ctl/restore_test.go | 4 +- event.go | 2 +- executor.go | 18 +- fragment.go | 6 +- gossip/gossip.go | 41 ++- handler.go | 84 +++-- handler_test.go | 6 +- holder.go | 54 +-- holder_test.go | 8 +- internal/private.pb.go | 747 ++++++++++++++++++++++++--------------- internal/private.proto | 31 +- pilosa.go | 4 +- server.go | 64 ++-- server/cluster_test.go | 6 +- server/server.go | 13 +- test/cluster.go | 54 +-- test/executor.go | 2 +- test/handler.go | 13 +- uri.go | 10 + 27 files changed, 1145 insertions(+), 869 deletions(-) diff --git a/broadcast.go b/broadcast.go index 7d00f1107..883b62436 100644 --- a/broadcast.go +++ b/broadcast.go @@ -24,11 +24,9 @@ import ( // MemberSet represents an interface for Node membership and inter-node communication. type MemberSet interface { - // Returns a list of all Nodes in the cluster - Nodes() []*Node - // Open starts any network activity implemented by the MemberSet - Open() error + // Node is the local node, used for membership broadcasts. + Open(n *Node) error } // StaticMemberSet represents a basic MemberSet for testing. @@ -41,13 +39,8 @@ func NewStaticMemberSet() *StaticMemberSet { return &StaticMemberSet{} } -// Nodes implements the MemberSet interface and returns a list of nodes in the cluster. -func (s *StaticMemberSet) Nodes() []*Node { - return s.nodes -} - // Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing. -func (s *StaticMemberSet) Open() error { +func (s *StaticMemberSet) Open(n *Node) error { return nil } diff --git a/client_test.go b/client_test.go index 17aafd7f4..f2bd18e50 100644 --- a/client_test.go +++ b/client_test.go @@ -36,10 +36,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { for i := 0; i < numNodes; i++ { hldr[i] = test.MustOpenHolder() server[i] = test.NewServer() - server[i].Handler.URI = server[i].HostURI() server[i].Handler.Cluster = c server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI() server[i].Handler.Holder = hldr[i].Holder + server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i] } return server, hldr } @@ -64,21 +64,21 @@ func TestClient_MultiNode(t *testing.T) { s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[0].Holder - e.URI = cluster.Nodes[0].URI + e.Node = cluster.Nodes[0] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[1].Holder - e.URI = cluster.Nodes[1].URI + e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[2].Holder - e.URI = cluster.Nodes[2].URI + e.Node = cluster.Nodes[2] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } @@ -217,10 +217,10 @@ func TestClient_Import(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -268,10 +268,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -317,10 +317,10 @@ func TestClient_ImportValue(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -355,10 +355,10 @@ func TestClient_BackupRestore(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] c := test.MustNewClient(s.Host(), defaultClient) @@ -420,10 +420,10 @@ func TestClient_BackupInverseView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] c := test.MustNewClient(s.Host(), defaultClient) @@ -457,10 +457,10 @@ func TestClient_BackupInvalidView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] c := test.MustNewClient(s.Host(), defaultClient) @@ -486,10 +486,10 @@ func TestClient_FragmentBlocks(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = s.HostURI() s.Handler.Holder = hldr.Holder + s.Handler.Node = s.Handler.Cluster.Nodes[0] // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) diff --git a/cluster.go b/cluster.go index b2ed0e372..1ed810ea0 100644 --- a/cluster.go +++ b/cluster.go @@ -66,7 +66,47 @@ const ( // Node represents a node in the cluster. type Node struct { - URI URI `json:"uri"` + ID string `json:"id"` + URI URI `json:"uri"` +} + +func (n Node) String() string { + return fmt.Sprintf("Node: %s", n.ID) +} + +// EncodeNodes converts a into its internal representation. +func EncodeNodes(a []*Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = EncodeNode(a[i]) + } + return other +} + +// EncodeNode converts n into its internal representation. +func EncodeNode(n *Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: n.URI.Encode(), + } +} + +func DecodeNodes(a []*internal.Node) []*Node { + if len(a) == 0 { + return nil + } + other := make([]*Node, len(a)) + for i := range a { + other[i] = DecodeNode(a[i]) + } + return other +} + +func DecodeNode(node *internal.Node) *Node { + return &Node{ + ID: node.ID, + URI: decodeURI(node.URI), + } } // Nodes represents a list of nodes. @@ -82,10 +122,10 @@ func (a Nodes) Contains(n *Node) bool { return false } -// ContainsURI returns true if host matches one of the node's uri. -func (a Nodes) ContainsURI(uri URI) bool { +// ContainsID returns true if host matches one of the node's id. +func (a Nodes) ContainsID(id string) bool { for _, n := range a { - if n.URI == uri { + if n.ID == id { return true } } @@ -103,6 +143,17 @@ func (a Nodes) Filter(n *Node) []*Node { return other } +// FilterID returns a new list of nodes with ID removed. +func (a Nodes) FilterID(id string) []*Node { + other := make([]*Node, 0, len(a)) + for _, node := range a { + if node.ID != id { + other = append(other, node) + } + } + return other +} + // FilterURI returns a new list of nodes with URI removed. func (a Nodes) FilterURI(uri URI) []*Node { other := make([]*Node, 0, len(a)) @@ -114,6 +165,15 @@ func (a Nodes) FilterURI(uri URI) []*Node { return other } +// IDs returns a list of all node IDs. +func (a Nodes) IDs() []string { + ids := make([]string, len(a)) + for i, n := range a { + ids[i] = n.ID + } + return ids +} + // URIs returns a list of all uris. func (a Nodes) URIs() []URI { uris := make([]URI, len(a)) @@ -130,24 +190,24 @@ func (a Nodes) Clone() []*Node { return other } -// ByHost implements sort.Interface for []Node based on -// the Host field. -type ByHost []*Node +// byID implements sort.Interface for []Node based on +// the ID field. +type byID []*Node -func (h ByHost) Len() int { return len(h) } -func (h ByHost) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h ByHost) Less(i, j int) bool { return h[i].URI.String() < h[j].URI.String() } +func (h byID) Len() int { return len(h) } +func (h byID) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h byID) Less(i, j int) bool { return h[i].ID < h[j].ID } // nodeAction represents a node that is joining or leaving the cluster. type nodeAction struct { - uri URI + node *Node action string } // Cluster represents a collection of nodes. type Cluster struct { ID string - URI URI + Node *Node Nodes []*Node // TODO phase this out? MemberSet MemberSet @@ -227,16 +287,33 @@ func (c *Cluster) logger() *log.Logger { return log.New(c.LogOutput, "", log.LstdFlags) } +// Coordinator returns the coordinator node. +func (c *Cluster) CoordinatorNode() *Node { + return c.nodeByURI(c.Coordinator) +} + // IsCoordinator is true if this node is the coordinator. func (c *Cluster) IsCoordinator() bool { - return c.Static || c.Coordinator == c.URI + return c.Static || c.Coordinator == c.Node.URI } // SetCoordinator updates the Coordinator to new if it is // currently old. Returns true if the Coordinator changed. -func (c *Cluster) SetCoordinator(oldURI, newURI URI) bool { - if c.Coordinator == oldURI && oldURI != newURI { - c.Coordinator = newURI +func (c *Cluster) SetCoordinator(o, n *Node) bool { + // Get old node. + oldNode := c.nodeByID(o.ID) + if oldNode == nil { + return false + } + + // Get new node. + newNode := c.nodeByID(n.ID) + if newNode == nil { + return false + } + + if c.Coordinator == oldNode.URI && oldNode != newNode { + c.Coordinator = newNode.URI return true } return false @@ -244,12 +321,11 @@ func (c *Cluster) SetCoordinator(oldURI, newURI URI) bool { // AddNode adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddNode(uri URI) error { - c.logger().Printf("add node %s to cluster on %s", uri, c.URI) +func (c *Cluster) AddNode(node *Node) error { + c.logger().Printf("add node %s to cluster on %s", node, c.Node) // add to cluster - _, added := c.addNodeBasicSorted(uri) - if !added { + if !c.addNodeBasicSorted(node) { return nil } @@ -257,7 +333,7 @@ func (c *Cluster) AddNode(uri URI) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.AddURI(uri) { + if !c.Topology.AddID(node.ID) { return nil } @@ -267,10 +343,9 @@ func (c *Cluster) AddNode(uri URI) error { // RemoveNode removes a node from the Cluster and updates and saves the // new topology. -func (c *Cluster) RemoveNode(uri URI) error { +func (c *Cluster) RemoveNode(node *Node) error { // remove from cluster - removed := c.removeNodeBasicSorted(uri) - if !removed { + if !c.removeNodeBasicSorted(node) { return nil } @@ -278,7 +353,7 @@ func (c *Cluster) RemoveNode(uri URI) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.RemoveURI(uri) { + if !c.Topology.RemoveID(node.ID) { return nil } @@ -286,9 +361,9 @@ func (c *Cluster) RemoveNode(uri URI) error { return c.saveTopology() } -// NodeSet returns the list of uris in the cluster. -func (c *Cluster) NodeSet() []URI { - return Nodes(c.Nodes).URIs() +// NodeIDs returns the list of IDs in the cluster. +func (c *Cluster) NodeIDs() []string { + return Nodes(c.Nodes).IDs() } func (c *Cluster) setID(id string) { @@ -310,8 +385,8 @@ func (c *Cluster) State() string { func (c *Cluster) SetState(state string) { c.mu.Lock() - defer c.mu.Unlock() c.setState(state) + c.mu.Unlock() } func (c *Cluster) setState(state string) { @@ -320,7 +395,7 @@ func (c *Cluster) setState(state string) { return } - c.logger().Printf("change cluster state from %s to %s on %s", c.State, state, c.URI) + c.logger().Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) var doCleanup bool @@ -345,7 +420,7 @@ func (c *Cluster) setState(state string) { // It's safe to do a cleanup after state changes back to normal. if doCleanup { var cleaner HolderCleaner - cleaner.URI = c.URI + cleaner.Node = c.Node cleaner.Holder = c.Holder cleaner.Cluster = c cleaner.Closing = c.closing @@ -359,17 +434,17 @@ func (c *Cluster) setState(state string) { func (c *Cluster) SetNodeState(state string) error { if c.IsCoordinator() { - return c.ReceiveNodeState(c.URI, state) + return c.ReceiveNodeState(c.Node.ID, state) } // Send node state to coordinator. ns := &internal.NodeStateMessage{ - URI: c.URI.Encode(), - State: state, + NodeID: c.Node.ID, + State: state, } c.logger().Printf("Sending State %s (%s)", state, c.Coordinator) - if err := c.sendTo(c.Coordinator, ns); err != nil { + if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -379,7 +454,7 @@ func (c *Cluster) SetNodeState(state string) error { // ReceiveNodeState sets node state in Topology in order for the // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. -func (c *Cluster) ReceiveNodeState(uri URI, state string) error { +func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { if !c.IsCoordinator() { return nil } @@ -389,8 +464,8 @@ func (c *Cluster) ReceiveNodeState(uri URI, state string) error { return nil } - c.Topology.nodeStates[uri] = state - c.logger().Printf("received state %s (%s)", state, uri) + c.Topology.nodeStates[nodeID] = state + c.logger().Printf("received state %s (%s)", state, nodeID) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { @@ -410,12 +485,22 @@ func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ ClusterID: c.ID, State: c.state, - NodeSet: encodeURIs(c.NodeSet()), + Nodes: EncodeNodes(c.Nodes), } } -// NodeByURI returns a node reference by uri. -func (c *Cluster) NodeByURI(uri URI) *Node { +// nodeByID returns a node reference by ID. +func (c *Cluster) nodeByID(id string) *Node { + for _, n := range c.Nodes { + if n.ID == id { + return n + } + } + return nil +} + +// nodeByURI returns a node reference by node URI. +func (c *Cluster) nodeByURI(uri URI) *Node { for _, n := range c.Nodes { if n.URI == uri { return n @@ -424,37 +509,36 @@ func (c *Cluster) NodeByURI(uri URI) *Node { return nil } -// nodePositionByURI returns the position of the node in slice c.Nodes. -func (c *Cluster) nodePositionByURI(uri URI) int { +// nodePositionByID returns the position of the node in slice c.Nodes. +func (c *Cluster) nodePositionByID(nodeID string) int { for i, n := range c.Nodes { - if n.URI == uri { + if n.ID == nodeID { return i } } return -1 } -// addNodeBasicSorted adds a node to the cluster, sorted by uri. +// addNodeBasicSorted adds a node to the cluster, sorted by id. // Returns a pointer to the node and true if the node was added. -func (c *Cluster) addNodeBasicSorted(uri URI) (*Node, bool) { - n := c.NodeByURI(uri) +func (c *Cluster) addNodeBasicSorted(node *Node) bool { + n := c.nodeByID(node.ID) if n != nil { - return n, false + return false } - n = &Node{URI: uri} - c.Nodes = append(c.Nodes, n) + c.Nodes = append(c.Nodes, node) // All hosts must be merged in the same order on all nodes in the cluster. - sort.Sort(ByHost(c.Nodes)) + sort.Sort(byID(c.Nodes)) - return n, true + return true } // removeNodeBasicSorted removes a node from the cluster, maintaining // the sort order. Returns true if the node was removed. -func (c *Cluster) removeNodeBasicSorted(uri URI) bool { - i := c.nodePositionByURI(uri) +func (c *Cluster) removeNodeBasicSorted(node *Node) bool { + i := c.nodePositionByID(node.ID) if i < 0 { return false } @@ -492,7 +576,7 @@ func fragsDiff(a, b []frag) []frag { return ret } -type fragsByHost map[URI][]frag +type fragsByHost map[string][]frag func (a fragsByHost) add(b fragsByHost) fragsByHost { for k, v := range b { @@ -539,7 +623,7 @@ func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFram // for each frame/view combination: for frame, views := range frameViews { for _, view := range views { - t[n.URI] = append(t[n.URI], frag{frame, view, i}) + t[n.ID] = append(t[n.ID], frag{frame, view, i}) } } } @@ -550,57 +634,57 @@ func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFram // diff compares c with another cluster and determines if a node is being // added or removed. An error is returned for any case other than where // exactly one node is added or removed. -func (c *Cluster) diff(other *Cluster) (action string, uri URI, err error) { +func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) { lenFrom := len(c.Nodes) lenTo := len(other.Nodes) // Determine if a node is being added or removed. if lenFrom == lenTo { - return action, uri, errors.New("clusters are the same size") + return action, nodeID, errors.New("clusters are the same size") } if lenFrom < lenTo { // Adding a node. if lenTo-lenFrom > 1 { - return action, uri, errors.New("adding more than one node at a time is not supported") + return action, nodeID, errors.New("adding more than one node at a time is not supported") } action = ResizeJobActionAdd - // Determine the URI that is being added. + // Determine the node ID that is being added. for _, n := range other.Nodes { - if c.NodeByURI(n.URI) == nil { - uri = n.URI + if c.nodeByID(n.ID) == nil { + nodeID = n.ID break } } } else if lenFrom > lenTo { // Removing a node. if lenFrom-lenTo > 1 { - return action, uri, errors.New("removing more than one node at a time is not supported") + return action, nodeID, errors.New("removing more than one node at a time is not supported") } action = ResizeJobActionRemove - // Determine the URI that is being removed. + // Determine the node ID that is being removed. for _, n := range c.Nodes { - if other.NodeByURI(n.URI) == nil { - uri = n.URI + if other.nodeByID(n.ID) == nil { + nodeID = n.ID break } } } - return action, uri, nil + return action, nodeID, nil } // fragSources returns a list of ResizeSources - for each node in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[URI][]*internal.ResizeSource, error) { - m := make(map[URI][]*internal.ResizeSource) +func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.ResizeSource, error) { + m := make(map[string][]*internal.ResizeSource) // Determine if a node is being added or removed. - action, diffURI, err := c.diff(to) + action, diffNodeID, err := c.diff(to) if err != nil { return nil, err } // Initialize the map with all the nodes in `to`. for _, n := range to.Nodes { - m[n.URI] = nil + m[n.ID] = nil } // If a node is being added, the source can be confined to the @@ -625,50 +709,50 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[URI][]*internal.Resi // srcFrags is the frag map based on a source cluster of replica = 1. srcFrags := srcCluster.fragsByHost(idx) - // srcHostsByFrag is the inverse representation of srcFrags. - srcHostsByFrag := make(map[frag]URI) - for uri, frags := range srcFrags { + // srcNodesByFrag is the inverse representation of srcFrags. + srcNodesByFrag := make(map[frag]string) + for nodeID, frags := range srcFrags { // If a node is being removed, don't consider it as a source. - if action == ResizeJobActionRemove && uri == diffURI { + if action == ResizeJobActionRemove && nodeID == diffNodeID { continue } for _, frag := range frags { - srcHostsByFrag[frag] = uri + srcNodesByFrag[frag] = nodeID } } - // Get the frag diff for each host. + // Get the frag diff for each nodeID. diffs := make(fragsByHost) - for host, frags := range tFrags { - if _, ok := fFrags[host]; ok { - diffs[host] = fragsDiff(frags, fFrags[host]) + for nodeID, frags := range tFrags { + if _, ok := fFrags[nodeID]; ok { + diffs[nodeID] = fragsDiff(frags, fFrags[nodeID]) } else { - diffs[host] = frags + diffs[nodeID] = frags } } // Get the ResizeSource for each diff. - for host, diff := range diffs { - m[host] = []*internal.ResizeSource{} + for nodeID, diff := range diffs { + m[nodeID] = []*internal.ResizeSource{} for _, frag := range diff { - // If there is no valid source URI for a fragment, + // If there is no valid source node ID for a fragment, // it likely means that the replica factor was not // high enough for the remaining nodes to contain // the fragment. - srcHost, ok := srcHostsByFrag[frag] + srcNodeID, ok := srcNodesByFrag[frag] if !ok { return nil, errors.New("not enough data to perform resize") } src := &internal.ResizeSource{ - URI: (srcHost).Encode(), + Node: EncodeNode(c.nodeByID(srcNodeID)), Index: idx.Name(), Frame: frag.frame, View: frag.view, Slice: frag.slice, } - m[host] = append(m[host], src) + m[nodeID] = append(m[nodeID], src) } } @@ -693,8 +777,8 @@ func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node { } // OwnsFragment returns true if a host owns a fragment. -func (c *Cluster) OwnsFragment(uri URI, index string, slice uint64) bool { - return Nodes(c.FragmentNodes(index, slice)).ContainsURI(uri) +func (c *Cluster) OwnsFragment(nodeID string, index string, slice uint64) bool { + return Nodes(c.FragmentNodes(index, slice)).ContainsID(nodeID) } // PartitionNodes returns a list of nodes that own a partition. @@ -735,14 +819,14 @@ func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { } // ContainsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) ContainsSlices(index string, maxSlice uint64, uri URI) []uint64 { +func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { p := c.Partition(index, i) // Determine the nodes for partition. nodes := c.PartitionNodes(p) - for _, node := range nodes { - if node.URI == uri { + for _, n := range nodes { + if n.ID == node.ID { slices = append(slices, i) } } @@ -793,7 +877,7 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. - c.AddNode(c.URI) + c.AddNode(c.Node) // Start the EventReceiver. if err := c.EventReceiver.Start(c); err != nil { @@ -801,7 +885,7 @@ func (c *Cluster) Open() error { } // Open MemberSet communication. - if err := c.MemberSet.Open(); err != nil { + if err := c.MemberSet.Open(c.Node); err != nil { return fmt.Errorf("opening MemberSet: %v", err) } @@ -832,21 +916,21 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) + return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) } func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - return URISlicesAreEqual(c.Topology.NodeSet, c.NodeSet()) + return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) } func (c *Cluster) allNodesReady() bool { if c.Static { return true } - for _, uri := range c.Topology.NodeSet { + for _, uri := range c.Topology.NodeIDs { if c.Topology.nodeStates[uri] != NodeStateReady { return false } @@ -885,9 +969,9 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { } // Add/remove uri to/from the cluster. if j.action == ResizeJobActionRemove { - return c.RemoveNode(nodeAction.uri) + return c.RemoveNode(nodeAction.node) } else if j.action == ResizeJobActionAdd { - return c.AddNode(nodeAction.uri) + return c.AddNode(nodeAction.node) } case ResizeJobStateAborted: if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { @@ -904,8 +988,7 @@ func (c *Cluster) setStateAndBroadcast(state string) error { return c.Broadcaster.SendSync(c.Status()) } -func (c *Cluster) sendTo(to URI, msg proto.Message) error { - node := &Node{URI: to} +func (c *Cluster) sendTo(node *Node, msg proto.Message) error { if err := c.Broadcaster.SendTo(node, msg); err != nil { return err } @@ -991,8 +1074,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { // Broadcaster is associated to the ResizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { - - j := NewResizeJob(Nodes(c.Nodes).URIs(), nodeAction.uri, nodeAction.action) + j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action) j.Broadcaster = c.Broadcaster // toCluster is a clone of Cluster with the new node added/removed for comparison. @@ -1002,16 +1084,16 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN if nodeAction.action == ResizeJobActionRemove { - toCluster.removeNodeBasicSorted(nodeAction.uri) + toCluster.removeNodeBasicSorted(nodeAction.node) } else if nodeAction.action == ResizeJobActionAdd { - toCluster.addNodeBasicSorted(nodeAction.uri) + toCluster.addNodeBasicSorted(nodeAction.node) } // multiIndex is a map of sources initialized with all the nodes in toCluster. - multiIndex := make(map[URI][]*internal.ResizeSource) + multiIndex := make(map[string][]*internal.ResizeSource) for _, n := range toCluster.Nodes { - multiIndex[n.URI] = nil + multiIndex[n.ID] = nil } // Add to multiIndex the instructions for each index. @@ -1021,23 +1103,23 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, return nil, err } - for u, sources := range fragSources { + for id, sources := range fragSources { for _, src := range sources { - multiIndex[u] = append(multiIndex[u], src) + multiIndex[id] = append(multiIndex[id], src) } } } - for u, sources := range multiIndex { + for id, sources := range multiIndex { // If a host doesn't need to request data, mark it as complete. if len(sources) == 0 { - j.URIs[u] = true + j.IDs[id] = true continue } instr := &internal.ResizeInstruction{ JobID: j.ID, - URI: u.Encode(), - Coordinator: encodeURI(c.Coordinator), + Node: EncodeNode(toCluster.nodeByID(id)), + Coordinator: EncodeNode(c.CoordinatorNode()), Sources: sources, Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), @@ -1063,7 +1145,7 @@ func (c *Cluster) CompleteCurrentJob(state string) error { // FollowResizeInstruction is run by any node that receives a ResizeInstruction. func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - c.logger().Printf("follow resize instruction on %s", c.URI) + c.logger().Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { @@ -1082,7 +1164,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Prepare the return message. complete := &internal.ResizeInstructionComplete{ JobID: instr.JobID, - URI: instr.URI, + Node: instr.Node, Error: "", } @@ -1096,13 +1178,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.URI, c.RemoteClient) // TODO: ClientOptions + client := NewInternalHTTPClientFromURI(&c.Node.URI, c.RemoteClient) // TODO: ClientOptions // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.logger().Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.URI) + c.logger().Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - srcURI := decodeURI(src.URI) + srcURI := decodeURI(src.Node.URI) // Retrieve frame. f := c.Holder.Frame(src.Index, src.Frame) @@ -1123,7 +1205,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Stream slice from remote node. - c.logger().Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.URI) + c.logger().Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found @@ -1137,7 +1219,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } return err } else if rd == nil { - return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.URI) + return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) } // Write to local frame and always close reader. @@ -1156,7 +1238,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err complete.Error = err.Error() } - if err := c.sendTo(decodeURI(instr.Coordinator), complete); err != nil { + if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) } }() @@ -1180,12 +1262,10 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc return fmt.Errorf("ResizeJob %d is no longer running", j.ID) } - uri := decodeURI(complete.URI) - // Mark host complete. - j.URIs[uri] = true + j.IDs[complete.Node.ID] = true - if !j.urisArePending() { + if !j.nodesArePending() { j.result <- ResizeJobStateDone } @@ -1203,7 +1283,7 @@ func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } type ResizeJob struct { ID int64 - URIs map[URI]bool + IDs map[string]bool Instructions []*internal.ResizeInstruction Broadcaster Broadcaster @@ -1223,32 +1303,32 @@ func (j *ResizeJob) logger() *log.Logger { } // NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(existingURIs []URI, uri URI, action string) *ResizeJob { +func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { // Build a map of uris to track their resize status. // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. - uris := make(map[URI]bool) + ids := make(map[string]bool) if action == ResizeJobActionRemove { - for _, u := range existingURIs { + for _, n := range existingNodes { // Exclude the removed node from the map. - if u == uri { + if n.ID == node.ID { continue } - uris[u] = false + ids[n.ID] = false } } else if action == ResizeJobActionAdd { - for _, u := range existingURIs { - uris[u] = false + for _, n := range existingNodes { + ids[n.ID] = false } // Include the added node in the map for tracking. - uris[uri] = false + ids[node.ID] = false } return &ResizeJob{ ID: rand.Int63(), - URIs: uris, + IDs: ids, action: action, result: make(chan string), LogOutput: os.Stderr, @@ -1280,7 +1360,7 @@ func (j *ResizeJob) Run() error { j.SetState(ResizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. - if !j.urisArePending() { + if !j.nodesArePending() { j.logger().Printf("ResizeJob contains no pending tasks; mark as done") j.result <- ResizeJobStateDone return nil @@ -1305,9 +1385,9 @@ func (j *ResizeJob) isComplete() bool { } } -// urisArePending returns true if any uri is still working on the resize. -func (j *ResizeJob) urisArePending() bool { - for _, complete := range j.URIs { +// nodesArePending returns true if any node is still working on the resize. +func (j *ResizeJob) nodesArePending() bool { + for _, complete := range j.IDs { if !complete { return true } @@ -1322,7 +1402,8 @@ func (j *ResizeJob) distributeResizeInstructions() error { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. node := &Node{ - URI: decodeURI(instr.URI), + ID: instr.Node.ID, + URI: decodeURI(instr.Node.URI), } j.logger().Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -1332,32 +1413,16 @@ func (j *ResizeJob) distributeResizeInstructions() error { return nil } -type NodeSet []URI +type NodeIDs []string -func (n NodeSet) Len() int { return len(n) } -func (n NodeSet) Swap(i, j int) { n[i], n[j] = n[j], n[i] } -func (n NodeSet) Less(i, j int) bool { return n[i].String() < n[j].String() } +func (n NodeIDs) Len() int { return len(n) } +func (n NodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] } +func (n NodeIDs) Less(i, j int) bool { return n[i] < n[j] } -func (u NodeSet) ToHostPortStrings() []string { - other := make([]string, 0, len(u)) - for _, uri := range u { - other = append(other, uri.HostPort()) - } - return other -} - -func (u NodeSet) ToStrings() []string { - other := make([]string, 0, len(u)) - for _, uri := range u { - other = append(other, uri.String()) - } - return other -} - -// ContainsURI returns true if uri matches one of the nodesets's uris. -func (n NodeSet) ContainsURI(uri URI) bool { - for _, nuri := range n { - if nuri == uri { +// ContainsID returns true if idi matches one of the nodesets's IDs. +func (n NodeIDs) ContainsID(id string) bool { + for _, nid := range n { + if nid == id { return true } } @@ -1367,71 +1432,71 @@ func (n NodeSet) ContainsURI(uri URI) bool { // Topology represents the list of hosts in the cluster. type Topology struct { mu sync.RWMutex - NodeSet []URI + NodeIDs []string ClusterID string // nodeStates holds the state of each node according to // the coordinator. Used during startup and data load. - nodeStates map[URI]string + nodeStates map[string]string } func NewTopology() *Topology { return &Topology{ - nodeStates: make(map[URI]string), + nodeStates: make(map[string]string), } } -// ContainsURI returns true if uri matches one of the topology's uris. -func (t *Topology) ContainsURI(uri URI) bool { +// ContainsID returns true if id matches one of the topology's IDs. +func (t *Topology) ContainsID(id string) bool { t.mu.RLock() defer t.mu.RUnlock() - return t.containsURI(uri) + return t.containsID(id) } -func (t *Topology) containsURI(uri URI) bool { - return NodeSet(t.NodeSet).ContainsURI(uri) +func (t *Topology) containsID(id string) bool { + return NodeIDs(t.NodeIDs).ContainsID(id) } -func (t *Topology) positionByURI(uri URI) int { - for i, turi := range t.NodeSet { - if turi == uri { +func (t *Topology) positionByID(nodeID string) int { + for i, tid := range t.NodeIDs { + if tid == nodeID { return i } } return -1 } -// AddURI adds the uri to the topology and returns true if added. -func (t *Topology) AddURI(uri URI) bool { +// AddID adds the node ID to the topology and returns true if added. +func (t *Topology) AddID(nodeID string) bool { t.mu.Lock() defer t.mu.Unlock() - if t.containsURI(uri) { + if t.containsID(nodeID) { return false } - t.NodeSet = append(t.NodeSet, uri) + t.NodeIDs = append(t.NodeIDs, nodeID) - sort.Slice(t.NodeSet, + sort.Slice(t.NodeIDs, func(i, j int) bool { - return t.NodeSet[i].String() < t.NodeSet[j].String() + return t.NodeIDs[i] < t.NodeIDs[j] }) return true } -// RemoveURI removes the uri from the topology and returns true if removed. -func (t *Topology) RemoveURI(uri URI) bool { +// RemoveID removes the node ID from the topology and returns true if removed. +func (t *Topology) RemoveID(nodeID string) bool { t.mu.Lock() defer t.mu.Unlock() - i := t.positionByURI(uri) + i := t.positionByID(nodeID) if i < 0 { return false } - copy(t.NodeSet[i:], t.NodeSet[i+1:]) - t.NodeSet[len(t.NodeSet)-1] = URI{} - t.NodeSet = t.NodeSet[:len(t.NodeSet)-1] + copy(t.NodeIDs[i:], t.NodeIDs[i+1:]) + t.NodeIDs[len(t.NodeIDs)-1] = "" + t.NodeIDs = t.NodeIDs[:len(t.NodeIDs)-1] return true } @@ -1485,7 +1550,7 @@ func encodeTopology(topology *Topology) *internal.Topology { } return &internal.Topology{ ClusterID: topology.ClusterID, - NodeSet: encodeURIs(topology.NodeSet), + NodeIDs: topology.NodeIDs, } } @@ -1496,19 +1561,16 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { t := NewTopology() t.ClusterID = topology.ClusterID - t.NodeSet = decodeURIs(topology.NodeSet) - sort.Slice(t.NodeSet, + t.NodeIDs = topology.NodeIDs + sort.Slice(t.NodeIDs, func(i, j int) bool { - return t.NodeSet[i].String() < t.NodeSet[j].String() + return t.NodeIDs[i] < t.NodeIDs[j] }) return t, nil } func (c *Cluster) considerTopology() error { - - c.ID = c.Topology.ClusterID - // Create ClusterID if one does not already exist. if c.ID == "" { u := uuid.NewV4() @@ -1521,17 +1583,17 @@ func (c *Cluster) considerTopology() error { } // If there is no .topology file, it's safe to proceed. - if len(c.Topology.NodeSet) == 0 { + if len(c.Topology.NodeIDs) == 0 { return nil } // The local node (coordinator) must be in the .topology. - if !c.Topology.ContainsURI(c.Coordinator) { - return fmt.Errorf("coordinator %s is not in topology: %v", c.Coordinator, c.Topology.NodeSet) + if !c.Topology.ContainsID(c.Node.ID) { + return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.NodeIDs) } // If local node is the only thing in .topology, continue. - //if len(c.Topology.NodeSet) == 1 { + //if len(c.Topology.NodeIDs) == 1 { // return nil //} @@ -1543,7 +1605,7 @@ func (c *Cluster) considerTopology() error { // ReceiveEvent represents an implementation of EventHandler. func (c *Cluster) ReceiveEvent(e *NodeEvent) error { // Ignore events sent from this node. - if e.URI == c.URI { + if e.Node.ID == c.Node.ID { return nil } @@ -1554,7 +1616,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { if !c.IsCoordinator() { return nil } - return c.nodeJoin(e.URI) + return c.nodeJoin(e.Node) case NodeLeave: // Automatic nodeLeave is intentionally not implemented. case NodeUpdate: @@ -1564,16 +1626,16 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { return nil } -func (c *Cluster) nodeJoin(uri URI) error { +func (c *Cluster) nodeJoin(node *Node) error { if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. - if !c.Topology.ContainsURI(uri) { - err := fmt.Sprintf("host is not in topology: %v", uri) + if !c.Topology.ContainsID(node.ID) { + err := fmt.Sprintf("host is not in topology: %s", node.ID) c.logger().Print(err) return errors.New(err) } - if err := c.AddNode(uri); err != nil { + if err := c.AddNode(node); err != nil { return err } @@ -1594,20 +1656,20 @@ func (c *Cluster) nodeJoin(uri URI) error { } else { // Send the status to the remote node. This lets the remote node // know that it can proceed with opening its Holder. - return c.sendTo(uri, c.Status()) + return c.sendTo(node, c.Status()) } return nil } // Don't do anything else if the cluster already contains the node. - if c.NodeByURI(uri) != nil { + if c.nodeByID(node.ID) != nil { return nil } // If the holder does not yet contain data, go ahead and add the node. if !c.Holder.HasData() { - if err := c.AddNode(uri); err != nil { + if err := c.AddNode(node); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1618,34 +1680,37 @@ func (c *Cluster) nodeJoin(uri URI) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return err } - c.joiningLeavingNodes <- nodeAction{uri, ResizeJobActionAdd} + c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} return nil } // NodeLeave initiates the removal of a node from the cluster. -func (c *Cluster) NodeLeave(uri URI) error { +func (c *Cluster) NodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. if !c.IsCoordinator() { - return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.Coordinator) + return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.CoordinatorNode().ID) } if c.State() != ClusterStateNormal { return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.State) } - return c.nodeLeave(uri) + return c.nodeLeave(node) } -func (c *Cluster) nodeLeave(uri URI) error { +func (c *Cluster) nodeLeave(node *Node) error { + // Get the actual node in the local cluster. + n := c.nodeByID(node.ID) + // Don't do anything else if the cluster doesn't contain the node. - if c.NodeByURI(uri) == nil { + if n == nil { return nil } // If the holder does not yet contain data, go ahead and remove the node. if !c.Holder.HasData() { - if err := c.RemoveNode(uri); err != nil { + if err := c.RemoveNode(n); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1656,7 +1721,7 @@ func (c *Cluster) nodeLeave(uri URI) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return err } - c.joiningLeavingNodes <- nodeAction{uri, ResizeJobActionRemove} + c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} return nil } @@ -1671,26 +1736,32 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { // Set ClusterID. c.setID(cs.ClusterID) - officialURIs := decodeURIs(cs.NodeSet) + officialNodes := DecodeNodes(cs.Nodes) // Add all nodes from the coordinator. - for _, uri := range officialURIs { - if err := c.AddNode(uri); err != nil { + for _, node := range officialNodes { + if err := c.AddNode(node); err != nil { return err } } // Remove any nodes not specified by the coordinator - // except for self. - for _, uri := range c.NodeSet() { + // except for self. Generate a list to remove first + // so that nodes aren't removed mid-loop. + nodeIDsToRemove := []string{} + for _, node := range c.Nodes { // Don't remove this node. - if uri == c.URI { + if node.ID == c.Node.ID { continue } - if NodeSet(officialURIs).ContainsURI(uri) { + if Nodes(officialNodes).ContainsID(node.ID) { continue } - if err := c.RemoveNode(uri); err != nil { + nodeIDsToRemove = append(nodeIDsToRemove, node.ID) + } + + for _, nodeID := range nodeIDsToRemove { + if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { return err } } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 75eb86f85..6c2c98e40 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -34,9 +34,12 @@ func TestFragCombos(t *testing.T) { t.Fatal(err) } + node0 := &Node{ID: "node0", URI: *uri0} + node1 := &Node{ID: "node1", URI: *uri1} + c := NewCluster() - c.addNodeBasicSorted(*uri0) - c.addNodeBasicSorted(*uri1) + c.addNodeBasicSorted(node0) + c.addNodeBasicSorted(node1) tests := []struct { idx string @@ -49,8 +52,8 @@ func TestFragCombos(t *testing.T) { maxSlice: uint64(2), frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, expected: fragsByHost{ - URI{"http", "host0", 10101}: []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, - URI{"http", "host1", 10101}: []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, + "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, + "node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, }, }, { @@ -58,8 +61,8 @@ func TestFragCombos(t *testing.T) { maxSlice: uint64(3), frameViews: viewsByFrame{"f": []string{"v0"}}, expected: fragsByHost{ - URI{"http", "host0", 10101}: []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, - URI{"http", "host1", 10101}: []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, + "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, + "node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, }, }, } @@ -106,34 +109,39 @@ func TestFragSources(t *testing.T) { t.Fatal(err) } + node0 := &Node{ID: "node0", URI: *uri0} + node1 := &Node{ID: "node1", URI: *uri1} + node2 := &Node{ID: "node2", URI: *uri2} + node3 := &Node{ID: "node3", URI: *uri3} + c1 := NewCluster() c1.ReplicaN = 1 - c1.addNodeBasicSorted(*uri0) - c1.addNodeBasicSorted(*uri1) + c1.addNodeBasicSorted(node0) + c1.addNodeBasicSorted(node1) c2 := NewCluster() c2.ReplicaN = 1 - c2.addNodeBasicSorted(*uri0) - c2.addNodeBasicSorted(*uri1) - c2.addNodeBasicSorted(*uri2) + c2.addNodeBasicSorted(node0) + c2.addNodeBasicSorted(node1) + c2.addNodeBasicSorted(node2) c3 := NewCluster() c3.ReplicaN = 2 - c3.addNodeBasicSorted(*uri0) - c3.addNodeBasicSorted(*uri1) + c3.addNodeBasicSorted(node0) + c3.addNodeBasicSorted(node1) c4 := NewCluster() c4.ReplicaN = 2 - c4.addNodeBasicSorted(*uri0) - c4.addNodeBasicSorted(*uri1) - c4.addNodeBasicSorted(*uri2) + c4.addNodeBasicSorted(node0) + c4.addNodeBasicSorted(node1) + c4.addNodeBasicSorted(node2) c5 := NewCluster() c5.ReplicaN = 2 - c5.addNodeBasicSorted(*uri0) - c5.addNodeBasicSorted(*uri1) - c5.addNodeBasicSorted(*uri2) - c5.addNodeBasicSorted(*uri3) + c5.addNodeBasicSorted(node0) + c5.addNodeBasicSorted(node1) + c5.addNodeBasicSorted(node2) + c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{}) @@ -161,19 +169,19 @@ func TestFragSources(t *testing.T) { from *Cluster to *Cluster idx *Index - expected map[URI][]*internal.ResizeSource + expected map[string][]*internal.ResizeSource err string }{ { from: c1, to: c2, idx: idx, - expected: map[URI][]*internal.ResizeSource{ - URI{"http", "host0", 10101}: []*internal.ResizeSource{}, - URI{"http", "host1", 10101}: []*internal.ResizeSource{}, - URI{"http", "host2", 10101}: []*internal.ResizeSource{ - {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(0)}, - {&internal.URI{"http", "host1", 10101}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*internal.ResizeSource{ + "node0": []*internal.ResizeSource{}, + "node1": []*internal.ResizeSource{}, + "node2": []*internal.ResizeSource{ + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -182,13 +190,13 @@ func TestFragSources(t *testing.T) { from: c4, to: c3, idx: idx, - expected: map[URI][]*internal.ResizeSource{ - URI{"http", "host0", 10101}: []*internal.ResizeSource{ - {&internal.URI{"http", "host1", 10101}, "i", "f", "standard", uint64(1)}, + expected: map[string][]*internal.ResizeSource{ + "node0": []*internal.ResizeSource{ + {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}}, "i", "f", "standard", uint64(1)}, }, - URI{"http", "host1", 10101}: []*internal.ResizeSource{ - {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(0)}, - {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(2)}, + "node1": []*internal.ResizeSource{ + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -197,15 +205,15 @@ func TestFragSources(t *testing.T) { from: c5, to: c4, idx: idx, - expected: map[URI][]*internal.ResizeSource{ - URI{"http", "host0", 10101}: []*internal.ResizeSource{ - {&internal.URI{"http", "host2", 10101}, "i", "f", "standard", uint64(0)}, - {&internal.URI{"http", "host2", 10101}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*internal.ResizeSource{ + "node0": []*internal.ResizeSource{ + {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}}, "i", "f", "standard", uint64(2)}, }, - URI{"http", "host1", 10101}: []*internal.ResizeSource{ - {&internal.URI{"http", "host0", 10101}, "i", "f", "standard", uint64(3)}, + "node1": []*internal.ResizeSource{ + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(3)}, }, - URI{"http", "host2", 10101}: []*internal.ResizeSource{}, + "node2": []*internal.ResizeSource{}, }, err: "", }, @@ -265,33 +273,37 @@ func TestResizeJob(t *testing.T) { t.Fatal(err) } + node0 := &Node{ID: "node0", URI: *uri0} + node1 := &Node{ID: "node1", URI: *uri1} + node2 := &Node{ID: "node2", URI: *uri2} + tests := []struct { - existingURIs []URI - uri URI - action string - expectedURIs map[URI]bool + existingNodes []*Node + node *Node + action string + expectedIDs map[string]bool }{ { - existingURIs: []URI{*uri0, *uri1}, - uri: *uri2, - action: ResizeJobActionAdd, - expectedURIs: map[URI]bool{*uri0: false, *uri1: false, *uri2: false}, + existingNodes: []*Node{node0, node1}, + node: node2, + action: ResizeJobActionAdd, + expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, }, { - existingURIs: []URI{*uri0, *uri1, *uri2}, - uri: *uri2, - action: ResizeJobActionRemove, - expectedURIs: map[URI]bool{*uri0: false, *uri1: false}, + existingNodes: []*Node{node0, node1, node2}, + node: node2, + action: ResizeJobActionRemove, + expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, }, } for _, test := range tests { - actual := NewResizeJob(test.existingURIs, test.uri, test.action) + actual := NewResizeJob(test.existingNodes, test.node, test.action) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(actual.URIs, test.expectedURIs) { - t.Errorf("expected: %v, but got: %v", test.expectedURIs, actual.URIs) + if !reflect.DeepEqual(actual.IDs, test.expectedIDs) { + t.Errorf("expected: %v, but got: %v", test.expectedIDs, actual.IDs) } } } diff --git a/cluster_test.go b/cluster_test.go index b1b62c815..9bcacac9a 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -106,7 +106,7 @@ func TestCluster_OwnsSlices(t *testing.T) { func TestCluster_ContainsSlices(t *testing.T) { c := test.NewCluster(5) c.ReplicaN = 3 - slices := c.ContainsSlices("test", 10, test.NewURIFromHostPort("host2", 0)) + slices := c.ContainsSlices("test", 10, c.Nodes[2]) if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { t.Fatalf("unexpected slices for node's index: %v", slices) @@ -119,15 +119,16 @@ func TestCluster_Nodes(t *testing.T) { uri2 := test.NewURIFromHostPort("node2", 0) uri3 := test.NewURIFromHostPort("node3", 0) - nodes := []*pilosa.Node{ - {URI: uri0}, - {URI: uri1}, - {URI: uri2}, - } + node0 := &pilosa.Node{ID: "node0", URI: uri0} + node1 := &pilosa.Node{ID: "node1", URI: uri1} + node2 := &pilosa.Node{ID: "node2", URI: uri2} + node3 := &pilosa.Node{ID: "node3", URI: uri3} - t.Run("NodeSet", func(t *testing.T) { - actual := pilosa.Nodes(nodes).URIs() - expected := []pilosa.URI{uri0, uri1, uri2} + nodes := []*pilosa.Node{node0, node1, node2} + + t.Run("NodeIDs", func(t *testing.T) { + actual := pilosa.Nodes(nodes).IDs() + expected := []string{node0.ID, node1.ID, node2.ID} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -150,19 +151,8 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Contains", func(t *testing.T) { - actualTrue := pilosa.Nodes(nodes).Contains(nodes[1]) - actualFalse := pilosa.Nodes(nodes).Contains(&pilosa.Node{}) - if !reflect.DeepEqual(actualTrue, true) { - t.Errorf("expected: %v, but got: %v", true, actualTrue) - } - if !reflect.DeepEqual(actualFalse, false) { - t.Errorf("expected: %v, but got: %v", false, actualTrue) - } - }) - - t.Run("ContainsURI", func(t *testing.T) { - actualTrue := pilosa.Nodes(nodes).ContainsURI(uri1) - actualFalse := pilosa.Nodes(nodes).ContainsURI(uri3) + actualTrue := pilosa.Nodes(nodes).Contains(node1) + actualFalse := pilosa.Nodes(nodes).Contains(node3) if !reflect.DeepEqual(actualTrue, true) { t.Errorf("expected: %v, but got: %v", true, actualTrue) } @@ -185,58 +175,66 @@ func TestCluster_Coordinator(t *testing.T) { uri1 := test.NewURIFromHostPort("node1", 0) uri2 := test.NewURIFromHostPort("node2", 0) + node1 := &pilosa.Node{ID: "node1", URI: uri1} + node2 := &pilosa.Node{ID: "node2", URI: uri2} + c1 := *pilosa.NewCluster() - c1.URI = uri1 - c1.Coordinator = uri1 + c1.Node = node1 + c1.Coordinator = node1.URI c2 := *pilosa.NewCluster() - c2.URI = uri2 - c2.Coordinator = uri1 + c2.Node = node2 + c2.Coordinator = node1.URI t.Run("IsCoordinator", func(t *testing.T) { if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.URI) + t.Errorf("!IsCoordinator error: %v", c1.Node) } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.URI) + t.Errorf("IsCoordinator error: %v", c2.Node) } }) } func TestCluster_Topology(t *testing.T) { - c1 := test.NewCluster(1) + c1 := test.NewCluster(1) // automatically creates Node{ID: "node0"} - uri1 := test.NewURIFromHostPort("node1", 0) - uri2 := test.NewURIFromHostPort("node2", 0) - base := test.NewURIFromHostPort("host0", 0) + uri0 := test.NewURIFromHostPort("host0", 0) + uri1 := test.NewURIFromHostPort("host1", 0) + uri2 := test.NewURIFromHostPort("host2", 0) invalid := test.NewURIFromHostPort("invalid", 0) + node0 := &pilosa.Node{ID: "node0", URI: uri0} + node1 := &pilosa.Node{ID: "node1", URI: uri1} + node2 := &pilosa.Node{ID: "node2", URI: uri2} + nodeinvalid := &pilosa.Node{ID: "nodeinvalid", URI: invalid} + t.Run("AddNode", func(t *testing.T) { - err := c1.AddNode(uri1) + err := c1.AddNode(node1) if err != nil { t.Fatal(err) } // add the same host. - err = c1.AddNode(uri1) + err = c1.AddNode(node1) if err != nil { t.Fatal(err) } - err = c1.AddNode(uri2) + err = c1.AddNode(node2) if err != nil { t.Fatal(err) } - actual := c1.NodeSet() - expected := []pilosa.URI{base, uri1, uri2} + actual := c1.NodeIDs() + expected := []string{node0.ID, node1.ID, node2.ID} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) - t.Run("ContainsURI", func(t *testing.T) { - if !c1.Topology.ContainsURI(uri1) { - t.Errorf("!ContainsHost error: %v", uri1) - } else if c1.Topology.ContainsURI(invalid) { - t.Errorf("ContainsHost error: %v", invalid) + t.Run("ContainsID", func(t *testing.T) { + if !c1.Topology.ContainsID(node1.ID) { + t.Errorf("!ContainsHost error: %v", node1.ID) + } else if c1.Topology.ContainsID(nodeinvalid.ID) { + t.Errorf("ContainsHost error: %v", nodeinvalid.ID) } }) } @@ -260,12 +258,12 @@ func TestCluster_ResizeStates(t *testing.T) { } expectedTop := &pilosa.Topology{ - NodeSet: []pilosa.URI{node.URI}, + NodeIDs: []string{node.Node.ID}, } // Verify topology file. - if !reflect.DeepEqual(node.Topology.NodeSet, expectedTop.NodeSet) { - t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeSet, node.Topology.NodeSet) + if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) } // Close TestCluster. @@ -282,7 +280,7 @@ func TestCluster_ResizeStates(t *testing.T) { // write topology to data file top := &pilosa.Topology{ - NodeSet: []pilosa.URI{node.URI}, + NodeIDs: []string{node.Node.ID}, } tc.WriteTopology(node.Path, top) @@ -310,14 +308,12 @@ func TestCluster_ResizeStates(t *testing.T) { // write topology to data file top := &pilosa.Topology{ - NodeSet: []pilosa.URI{ - test.NewURIFromHostPort("some-other-host", 0), - }, + NodeIDs: []string{"some-other-host"}, } tc.WriteTopology(node.Path, top) // Open TestCluster. - expected := "considerTopology: coordinator http://host0:0 is not in topology: [http://some-other-host:0]" + expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" err := tc.Open() if err == nil || err.Error() != expected { t.Errorf("did not receive expected error: %s", expected) @@ -351,14 +347,14 @@ func TestCluster_ResizeStates(t *testing.T) { } expectedTop := &pilosa.Topology{ - NodeSet: []pilosa.URI{node0.URI, node1.URI}, + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, } // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeSet, expectedTop.NodeSet) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeSet, node0.Topology.NodeSet) - } else if !reflect.DeepEqual(node1.Topology.NodeSet, expectedTop.NodeSet) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeSet, node1.Topology.NodeSet) + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) } // Close TestCluster. @@ -372,13 +368,9 @@ func TestCluster_ResizeStates(t *testing.T) { tc.AddNode(false) node0 := tc.Clusters[0] - u0 := test.NewURIFromHostPort("host0", 0) - //u1 := test.NewURIFromHostPort("host1", 0) - u2 := test.NewURIFromHostPort("host2", 0) - // write topology to data file top := &pilosa.Topology{ - NodeSet: []pilosa.URI{u0, u2}, + NodeIDs: []string{"node0", "node2"}, } tc.WriteTopology(node0.Path, top) @@ -393,7 +385,7 @@ func TestCluster_ResizeStates(t *testing.T) { } // Expect an error by adding a node not in the topology. - expectedError := "host is not in topology: http://host1:0" + expectedError := "host is not in topology: node1" err := tc.AddNode(false) if err == nil || err.Error() != expectedError { t.Errorf("did not receive expected error: %s", expectedError) @@ -477,14 +469,14 @@ func TestCluster_ResizeStates(t *testing.T) { } expectedTop := &pilosa.Topology{ - NodeSet: []pilosa.URI{node0.URI, node1.URI}, + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, } // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeSet, expectedTop.NodeSet) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeSet, node0.Topology.NodeSet) - } else if !reflect.DeepEqual(node1.Topology.NodeSet, expectedTop.NodeSet) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeSet, node1.Topology.NodeSet) + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) } // Bits @@ -519,28 +511,23 @@ func TestCluster_ResizeStates(t *testing.T) { // Ensures that coordinator can be changed. func TestCluster_SetCoordinator(t *testing.T) { t.Run("SetCoordinator", func(t *testing.T) { - c := test.NewCluster(1) - oldURI, err := pilosa.NewURIFromAddress("localhost:8888") - if err != nil { - t.Fatal(err) - } - c.Coordinator = *oldURI + c := test.NewCluster(2) - newURI, err := pilosa.NewURIFromAddress("localhost:9999") - if err != nil { - t.Fatal(err) - } + oldNode := c.Nodes[0] + newNode := c.Nodes[1] // Set coordinator to the same value. - c.SetCoordinator(c.Coordinator, *oldURI) - if c.Coordinator != *oldURI { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, *oldURI) + if set := c.SetCoordinator(oldNode, oldNode); set { + t.Errorf("did not expect coordinator to change") + } else if c.Coordinator != oldNode.URI { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) } // Set coordinator to a new value. - c.SetCoordinator(c.Coordinator, *newURI) - if c.Coordinator != *newURI { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, *newURI) + if set := c.SetCoordinator(oldNode, newNode); !set { + t.Errorf("expected coordinator to change") + } else if c.Coordinator != newNode.URI { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) } }) } diff --git a/ctl/backup_test.go b/ctl/backup_test.go index bb541b344..d23b1a917 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,7 +50,9 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = *uri + node := &pilosa.Node{ID: "node", URI: *uri} + + s.Handler.Node = node s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder diff --git a/ctl/export_test.go b/ctl/export_test.go index 9e4381d39..5d1d5a4d7 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,9 +63,11 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = *uri + node := &pilosa.Node{ID: "node", URI: *uri} + + s.Handler.Node = node s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri + s.Handler.Cluster.Nodes[0] = node s.Handler.Holder = hldr.Holder cm.Host = s.Host() diff --git a/ctl/import_test.go b/ctl/import_test.go index af38292cd..dcf341187 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,9 +69,11 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = *uri + node := &pilosa.Node{ID: "node", URI: *uri} + + s.Handler.Node = node s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri + s.Handler.Cluster.Nodes[0] = node s.Handler.Holder = hldr.Holder cm.Host = s.Host() @@ -109,9 +111,11 @@ func TestImportCommand_RunValue(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = *uri + node := &pilosa.Node{ID: "node", URI: *uri} + + s.Handler.Node = node s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri + s.Handler.Cluster.Nodes[0] = node s.Handler.Holder = hldr.Holder cm.Host = s.Host() diff --git a/ctl/restore_test.go b/ctl/restore_test.go index 60a0be9f6..bb8eb3b5f 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,7 +52,9 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.URI = *uri + node := &pilosa.Node{ID: "node", URI: *uri} + + s.Handler.Node = node s.Handler.Cluster = test.NewCluster(1) s.Handler.Cluster.Nodes[0].URI = *uri s.Handler.Holder = hldr.Holder diff --git a/event.go b/event.go index 0b7485f76..5df69361b 100644 --- a/event.go +++ b/event.go @@ -27,7 +27,7 @@ const ( // NodeEvent is a single event related to node activity in the cluster. type NodeEvent struct { Event NodeEventType - URI URI + Node *Node } // EventHandler is the interface for the pilosa object which knows how to diff --git a/executor.go b/executor.go index 3460d04df..bd8d63ae2 100644 --- a/executor.go +++ b/executor.go @@ -40,7 +40,7 @@ type Executor struct { Holder *Holder // Local hostname & cluster configuration. - URI URI + Node *Node Cluster *Cluster // Client used for remote requests. @@ -956,7 +956,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql ret := false for _, node := range e.Cluster.FragmentNodes(index, slice) { // Update locally if host matches. - if node.URI == e.URI { + if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) if err != nil { return false, err @@ -1061,7 +1061,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C for _, node := range e.Cluster.FragmentNodes(index, slice) { // Update locally if host matches. - if node.URI == e.URI { + if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) if err != nil { return false, err @@ -1140,7 +1140,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1198,7 +1198,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1285,7 +1285,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1344,7 +1344,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterURI(e.URI) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1451,7 +1451,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.NodeByURI(e.URI)} + nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} } // Start mapping across all primary owners. @@ -1507,7 +1507,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod resp := mapResponse{node: n, slices: nodeSlices} // Send local slices to mapper, otherwise remote exec. - if n.URI == e.URI { + if n.ID == e.Node.ID { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) diff --git a/fragment.go b/fragment.go index 3df788629..3b09b925e 100644 --- a/fragment.go +++ b/fragment.go @@ -1685,7 +1685,7 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - URI URI + Node *Node Cluster *Cluster RemoteClient *http.Client @@ -1715,7 +1715,7 @@ func (s *FragmentSyncer) SyncFragment() error { blockSets := make([][]FragmentBlock, 0, len(nodes)) for _, node := range nodes { // Read local blocks. - if node.URI == s.URI { + if node.ID == s.Node.ID { b := s.Fragment.Blocks() blockSets = append(blockSets, b) continue @@ -1789,7 +1789,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { var pairSets []PairSet var clients []InternalClient for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { - if s.URI == node.URI { + if s.Node.ID == node.ID { continue } diff --git a/gossip/gossip.go b/gossip/gossip.go index 243d19738..d0868be92 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -43,6 +43,7 @@ var _ memberlist.Delegate = &GossipMemberSet{} // GossipMemberSet represents a gossip implementation of MemberSet using memberlist. type GossipMemberSet struct { mu sync.RWMutex + node *pilosa.Node memberlist *memberlist.Memberlist handler pilosa.BroadcastHandler @@ -55,20 +56,6 @@ type GossipMemberSet struct { LogOutput io.Writer } -// Nodes implements the MemberSet interface and returns a list of nodes in the cluster. -func (g *GossipMemberSet) Nodes() []*pilosa.Node { - g.mu.RLock() - defer g.mu.RUnlock() - - a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) - for _, n := range g.memberlist.Members() { - uri, _ := pilosa.NewURIFromAddress(n.Name) - // TODO don't swallow the error above - a = append(a, &pilosa.Node{URI: *uri}) - } - return a -} - // Start implements the BroadcastReceiver interface and sets the BroadcastHandler. func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { g.handler = h @@ -81,11 +68,13 @@ func (g *GossipMemberSet) Seed() string { } // Open implements the MemberSet interface to start network activity. -func (g *GossipMemberSet) Open() error { +func (g *GossipMemberSet) Open(n *pilosa.Node) error { if g.handler == nil { return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()") } + g.node = n + err := error(nil) g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) @@ -112,7 +101,7 @@ func (g *GossipMemberSet) Open() error { nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds g.mu.RLock() - err = g.joinWithRetry(pilosa.NodeSet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) + err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings()) g.mu.RUnlock() if err != nil { return errors.Wrap(err, "joinWithRetry") @@ -280,7 +269,12 @@ func (g *GossipMemberSet) SendAsync(pb proto.Message) error { // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - return []byte{} + buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) + if err != nil { + g.logger().Printf("marshal message error: %s", err) + return []byte{} + } + return buf } // NotifyMsg implementation of the memberlist.Delegate interface @@ -387,14 +381,19 @@ func (g *GossipEventReceiver) listen() { continue } - uri, _ := pilosa.NewURIFromAddress(e.Node.Name) - // TODO: don't swallow this error + // Get the node from the event.Node meta data. + var n internal.Node + if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { + // TODO: consider logging error + continue + } + node := pilosa.DecodeNode(&n) ne := &pilosa.NodeEvent{ Event: nodeEventType, - URI: *uri, + Node: node, } - _ = g.eventHandler.ReceiveEvent(ne) // TODO: don't swallow this error + _ = g.eventHandler.ReceiveEvent(ne) } } diff --git a/handler.go b/handler.go index c89abc3bf..a12e7b391 100644 --- a/handler.go +++ b/handler.go @@ -57,7 +57,7 @@ type Handler struct { StatusHandler StatusHandler // Local hostname & cluster configuration. - URI URI + Node *Node Cluster *Cluster RemoteClient *http.Client @@ -268,8 +268,8 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { cs := pb.(*internal.ClusterStatus) if err := json.NewEncoder(w).Encode(getStatusResponse{ - State: cs.State, - NodeSet: decodeURIs(cs.NodeSet), + State: cs.State, + Nodes: DecodeNodes(cs.Nodes), }); err != nil { h.logger().Printf("write status response error: %s", err) } @@ -280,8 +280,8 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - NodeSet []URI `json:"nodes"` + State string `json:"state"` + Nodes []*Node `json:"nodes"` } // handlePostQuery handles /query requests. @@ -1206,8 +1206,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI, req.Index, req.Slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) + if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) { + msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice) http.Error(w, msg, http.StatusPreconditionFailed) return } @@ -1276,8 +1276,8 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI, req.Index, req.Slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) + if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) { + msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice) http.Error(w, msg, http.StatusPreconditionFailed) return } @@ -1342,8 +1342,8 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI, index, slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice) + if !h.Cluster.OwnsFragment(h.Node.ID, index, slice) { + msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, index, slice) http.Error(w, msg, http.StatusPreconditionFailed) return } @@ -1570,7 +1570,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // Loop over each slice and import it if this node owns it. for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !h.Cluster.OwnsFragment(h.URI, indexName, slice) { + if !h.Cluster.OwnsFragment(h.Node.ID, indexName, slice) { continue } @@ -1964,31 +1964,30 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r return } - oldURI := h.Cluster.Coordinator + oldNode := h.Cluster.nodeByURI(h.Cluster.Coordinator) + if oldNode == nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + newNode := h.Cluster.nodeByID(req.ID) + if newNode == nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } - var newURI *URI if err := func() error { - newURI, err = NewURIFromAddress(req.Address) - if err != nil { - return fmt.Errorf("problem with set-coordinator address: %s", err) - } - - //if !Nodes(h.Cluster.Nodes).ContainsURI(*newURI) { - // return fmt.Errorf("set-coordinator node does not exist: %s", newURI) - //} - // Send the set-coordinator message to all nodes. err := h.Broadcaster.SendSync( &internal.SetCoordinatorMessage{ - Old: (&h.Cluster.Coordinator).Encode(), - New: newURI.Encode(), + Old: EncodeNode(oldNode), + New: EncodeNode(newNode), }) if err != nil { return fmt.Errorf("problem sending SetCoordinator message: %s", err) } // Set Coordinator on local node. - h.Cluster.SetCoordinator(oldURI, *newURI) + _ = h.Cluster.SetCoordinator(oldNode, newNode) return nil }(); err != nil { @@ -1998,20 +1997,20 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r // Encode response. if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ - Old: &oldURI, - New: newURI, + Old: oldNode, + New: newNode, }); err != nil { h.logger().Printf("response encoding error: %s", err) } } type setCoordinatorRequest struct { - Address string `json:"address"` + ID string `json:"id"` } type setCoordinatorResponse struct { - Old *URI `json:"old"` - New *URI `json:"new"` + Old *Node `json:"old"` + New *Node `json:"new"` } // handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. @@ -2024,19 +2023,16 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht return } - var removeURI *URI + removeNode := h.Cluster.nodeByID(req.ID) + if removeNode == nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } if err := func() error { - removeURI, err = NewURIFromAddress(req.Address) - if err != nil { - return fmt.Errorf("problem with remove node address: %s", err) - } - - // TODO: make sure the address is in the cluster - // TODO: prevent removing the coordinator node // Start the resize process (similar to NodeJoin) - err := h.Cluster.NodeLeave(*removeURI) + err := h.Cluster.NodeLeave(removeNode) if err != nil { return err } @@ -2049,18 +2045,18 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht // Encode response. if err := json.NewEncoder(w).Encode(removeNodeResponse{ - Remove: removeURI, + Remove: removeNode, }); err != nil { h.logger().Printf("response encoding error: %s", err) } } type removeNodeRequest struct { - Address string `json:"address"` + ID string `json:"id"` } type removeNodeResponse struct { - Remove *URI `json:"remove"` + Remove *Node `json:"remove"` } // handlePostClusterResizeAbort handles POST /cluster/resize/abort request. @@ -2221,7 +2217,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(h.Holder.NodeID)) + _, err := w.Write([]byte(h.Cluster.Node.ID)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/handler_test.go b/handler_test.go index b89b7f59c..0f2b2dc00 100644 --- a/handler_test.go +++ b/handler_test.go @@ -147,7 +147,7 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"scheme":"http","host":"localhost","port":10101}]}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -1215,8 +1215,8 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"uri":{"scheme":"http","host":"host2"}},{"uri":{"scheme":"http","host":"host0"}}]`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"}},{"id":"node0","uri":{"scheme":"http","host":"host0"}}]`+"\n" { + t.Fatalf("unexpected body: %q", body) } } diff --git a/holder.go b/holder.go index c282e2686..4c6c10423 100644 --- a/holder.go +++ b/holder.go @@ -69,8 +69,6 @@ type Holder struct { CacheFlushInterval time.Duration LogOutput io.Writer - - NodeID string } // NewHolder returns a new instance of Holder. @@ -528,25 +526,29 @@ func (h *Holder) setFileLimit() { func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } -func (h *Holder) loadNodeID() error { +func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, "ID") nodeID := "" - nodeIDBytes, err := ioutil.ReadFile(idPath) - if err == nil { - h.NodeID = strings.TrimSpace(string(nodeIDBytes)) - } else if os.IsNotExist(err) { - u := uuid.NewV4() - nodeID = u.String() - err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) - if err != nil { - return err - } - h.NodeID = nodeID - } else if err != nil { - return err + + h.logger().Printf("load NodeID: %s", idPath) + if err := os.MkdirAll(h.Path, 0777); err != nil { + return "", err } - return nil + nodeIDBytes, err := ioutil.ReadFile(idPath) + if err == nil { + nodeID = strings.TrimSpace(string(nodeIDBytes)) + } else if os.IsNotExist(err) { + nodeID = uuid.NewV4().String() + err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) + if err != nil { + return "", err + } + } else if err != nil { + return "", err + } + + return nodeID, nil } // HolderSyncer is an active anti-entropy tool that compares the local holder @@ -554,7 +556,7 @@ func (h *Holder) loadNodeID() error { type HolderSyncer struct { Holder *Holder - URI URI + Node *Node Cluster *Cluster RemoteClient *http.Client @@ -610,7 +612,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.URI, di.Name, slice) { + if !s.Cluster.OwnsFragment(s.Node.ID, di.Name, slice) { continue } @@ -652,7 +654,7 @@ func (s *HolderSyncer) syncIndex(index string) error { s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag}) // Sync with every other host. - for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { + for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) // Retrieve attributes from differing blocks. @@ -663,7 +665,7 @@ func (s *HolderSyncer) syncIndex(index string) error { } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.URI.HostPort()}) + s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.ID}) // Update local copy. if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil { @@ -698,7 +700,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) // Sync with every other host. - for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { + for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) // Retrieve attributes from differing blocks. @@ -711,7 +713,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.URI.HostPort()}) + s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID}) // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { @@ -751,7 +753,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ Fragment: frag, - URI: s.URI, + Node: s.Node, Cluster: s.Cluster, Closing: s.Closing, RemoteClient: s.RemoteClient, @@ -765,7 +767,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // HolderCleaner removes fragments and data files that are no longer used. type HolderCleaner struct { - URI URI + Node *Node Holder *Holder Cluster *Cluster @@ -794,7 +796,7 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.URI) + containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) // Get the fragments registered in memory. for _, frame := range index.Frames() { diff --git a/holder_test.go b/holder_test.go index 16f4c8abf..3422fe081 100644 --- a/holder_test.go +++ b/holder_test.go @@ -419,7 +419,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder - e.URI = cluster.Nodes[1].URI + e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } @@ -487,7 +487,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Set up syncer. syncer := pilosa.HolderSyncer{ Holder: hldr0.Holder, - URI: cluster.Nodes[0].URI, + Node: cluster.Nodes[0], Cluster: cluster, RemoteClient: pilosa.GetHTTPClient(nil), Stats: pilosa.NopStatsClient, @@ -586,7 +586,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Set up cleaner for replication 2. cleaner2 := pilosa.HolderCleaner{ - URI: cluster.Nodes[0].URI, + Node: cluster.Nodes[0], Holder: hldr0.Holder, Cluster: cluster, } @@ -629,7 +629,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Set up cleaner for replication 1. cleaner1 := pilosa.HolderCleaner{ - URI: cluster.Nodes[0].URI, + Node: cluster.Nodes[0], Holder: hldr0.Holder, Cluster: cluster, } diff --git a/internal/private.pb.go b/internal/private.pb.go index 905f004d5..ae6247bb6 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -30,6 +30,7 @@ CreateInputDefinitionMessage DeleteInputDefinitionMessage URI + Node NodeStateMessage NodeStatus ClusterStatus @@ -673,21 +674,45 @@ func (m *URI) GetPort() uint32 { return 0 } +type Node struct { + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` +} + +func (m *Node) Reset() { *m = Node{} } +func (m *Node) String() string { return proto.CompactTextString(m) } +func (*Node) ProtoMessage() {} +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } + +func (m *Node) GetID() string { + if m != nil { + return m.ID + } + return "" +} + +func (m *Node) GetURI() *URI { + if m != nil { + return m.URI + } + return nil +} + type NodeStateMessage struct { - URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} -func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } -func (m *NodeStateMessage) GetURI() *URI { +func (m *NodeStateMessage) GetNodeID() string { if m != nil { - return m.URI + return m.NodeID } - return nil + return "" } func (m *NodeStateMessage) GetState() string { @@ -698,7 +723,7 @@ func (m *NodeStateMessage) GetState() string { } type NodeStatus struct { - URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` } @@ -706,11 +731,11 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } -func (m *NodeStatus) GetURI() *URI { +func (m *NodeStatus) GetNode() *Node { if m != nil { - return m.URI + return m.Node } return nil } @@ -730,15 +755,22 @@ func (m *NodeStatus) GetSchema() *Schema { } type ClusterStatus struct { - State string `protobuf:"bytes,1,opt,name=State,proto3" json:"State,omitempty"` - NodeSet []*URI `protobuf:"bytes,2,rep,name=NodeSet" json:"NodeSet,omitempty"` - ClusterID string `protobuf:"bytes,3,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` } func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } + +func (m *ClusterStatus) GetClusterID() string { + if m != nil { + return m.ClusterID + } + return "" +} func (m *ClusterStatus) GetState() string { if m != nil { @@ -747,20 +779,13 @@ func (m *ClusterStatus) GetState() string { return "" } -func (m *ClusterStatus) GetNodeSet() []*URI { +func (m *ClusterStatus) GetNodes() []*Node { if m != nil { - return m.NodeSet + return m.Nodes } return nil } -func (m *ClusterStatus) GetClusterID() string { - if m != nil { - return m.ClusterID - } - return "" -} - type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -771,7 +796,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *Field) GetName() string { if m != nil { @@ -810,7 +835,7 @@ type CreateViewMessage struct { func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -842,7 +867,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -867,8 +892,8 @@ func (m *DeleteViewMessage) GetView() string { type ResizeInstruction struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` - Coordinator *URI `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` @@ -877,7 +902,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -886,14 +911,14 @@ func (m *ResizeInstruction) GetJobID() int64 { return 0 } -func (m *ResizeInstruction) GetURI() *URI { +func (m *ResizeInstruction) GetNode() *Node { if m != nil { - return m.URI + return m.Node } return nil } -func (m *ResizeInstruction) GetCoordinator() *URI { +func (m *ResizeInstruction) GetCoordinator() *Node { if m != nil { return m.Coordinator } @@ -922,7 +947,7 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { } type ResizeSource struct { - URI *URI `protobuf:"bytes,1,opt,name=URI" json:"URI,omitempty"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` @@ -932,11 +957,11 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } -func (m *ResizeSource) GetURI() *URI { +func (m *ResizeSource) GetNode() *Node { if m != nil { - return m.URI + return m.Node } return nil } @@ -971,7 +996,7 @@ func (m *ResizeSource) GetSlice() uint64 { type ResizeInstructionComplete struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` } @@ -979,7 +1004,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{29} + return fileDescriptorPrivate, []int{30} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -989,9 +1014,9 @@ func (m *ResizeInstructionComplete) GetJobID() int64 { return 0 } -func (m *ResizeInstructionComplete) GetURI() *URI { +func (m *ResizeInstructionComplete) GetNode() *Node { if m != nil { - return m.URI + return m.Node } return nil } @@ -1004,23 +1029,23 @@ func (m *ResizeInstructionComplete) GetError() string { } type SetCoordinatorMessage struct { - Old *URI `protobuf:"bytes,1,opt,name=Old" json:"Old,omitempty"` - New *URI `protobuf:"bytes,2,opt,name=New" json:"New,omitempty"` + Old *Node `protobuf:"bytes,1,opt,name=Old" json:"Old,omitempty"` + New *Node `protobuf:"bytes,2,opt,name=New" json:"New,omitempty"` } func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } -func (m *SetCoordinatorMessage) GetOld() *URI { +func (m *SetCoordinatorMessage) GetOld() *Node { if m != nil { return m.Old } return nil } -func (m *SetCoordinatorMessage) GetNew() *URI { +func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { return m.New } @@ -1028,21 +1053,14 @@ func (m *SetCoordinatorMessage) GetNew() *URI { } type Topology struct { - NodeSet []*URI `protobuf:"bytes,1,rep,name=NodeSet" json:"NodeSet,omitempty"` - ClusterID string `protobuf:"bytes,2,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` } func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } - -func (m *Topology) GetNodeSet() []*URI { - if m != nil { - return m.NodeSet - } - return nil -} +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } func (m *Topology) GetClusterID() string { if m != nil { @@ -1051,6 +1069,13 @@ func (m *Topology) GetClusterID() string { return "" } +func (m *Topology) GetNodeIDs() []string { + if m != nil { + return m.NodeIDs + } + return nil +} + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -1073,6 +1098,7 @@ func init() { proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") proto.RegisterType((*URI)(nil), "internal.URI") + proto.RegisterType((*Node)(nil), "internal.Node") proto.RegisterType((*NodeStateMessage)(nil), "internal.NodeStateMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") @@ -1933,6 +1959,40 @@ func (m *URI) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *Node) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Node) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.ID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ID))) + i += copy(dAtA[i:], m.ID) + } + if m.URI != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n11, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n11 + } + return i, nil +} + func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1948,15 +2008,11 @@ func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.URI != nil { + if len(m.NodeID) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n11, err := m.URI.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n11 + i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeID))) + i += copy(dAtA[i:], m.NodeID) } if len(m.State) > 0 { dAtA[i] = 0x12 @@ -1982,11 +2038,11 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.URI != nil { + if m.Node != nil { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n12, err := m.URI.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n12, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -2030,15 +2086,21 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.State) > 0 { + if len(m.ClusterID) > 0 { dAtA[i] = 0xa i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i += copy(dAtA[i:], m.ClusterID) + } + if len(m.State) > 0 { + dAtA[i] = 0x12 + i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if len(m.NodeSet) > 0 { - for _, msg := range m.NodeSet { - dAtA[i] = 0x12 + if len(m.Nodes) > 0 { + for _, msg := range m.Nodes { + dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) @@ -2048,12 +2110,6 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.ClusterID) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) - i += copy(dAtA[i:], m.ClusterID) - } return i, nil } @@ -2189,11 +2245,11 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) } - if m.URI != nil { + if m.Node != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n15, err := m.URI.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n15, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -2259,11 +2315,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.URI != nil { + if m.Node != nil { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n19, err := m.URI.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n19, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -2315,11 +2371,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) } - if m.URI != nil { + if m.Node != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n20, err := m.URI.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n20, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -2387,24 +2443,27 @@ func (m *Topology) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.NodeSet) > 0 { - for _, msg := range m.NodeSet { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } if len(m.ClusterID) > 0 { - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) i += copy(dAtA[i:], m.ClusterID) } + if len(m.NodeIDs) > 0 { + for _, s := range m.NodeIDs { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -2808,11 +2867,25 @@ func (m *URI) Size() (n int) { return n } +func (m *Node) Size() (n int) { + var l int + _ = l + l = len(m.ID) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.URI != nil { + l = m.URI.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *NodeStateMessage) Size() (n int) { var l int _ = l - if m.URI != nil { - l = m.URI.Size() + l = len(m.NodeID) + if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } l = len(m.State) @@ -2825,8 +2898,8 @@ func (m *NodeStateMessage) Size() (n int) { func (m *NodeStatus) Size() (n int) { var l int _ = l - if m.URI != nil { - l = m.URI.Size() + if m.Node != nil { + l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } if m.MaxSlices != nil { @@ -2843,20 +2916,20 @@ func (m *NodeStatus) Size() (n int) { func (m *ClusterStatus) Size() (n int) { var l int _ = l + l = len(m.ClusterID) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } l = len(m.State) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.NodeSet) > 0 { - for _, e := range m.NodeSet { + if len(m.Nodes) > 0 { + for _, e := range m.Nodes { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } } - l = len(m.ClusterID) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } return n } @@ -2922,8 +2995,8 @@ func (m *ResizeInstruction) Size() (n int) { if m.JobID != 0 { n += 1 + sovPrivate(uint64(m.JobID)) } - if m.URI != nil { - l = m.URI.Size() + if m.Node != nil { + l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } if m.Coordinator != nil { @@ -2950,8 +3023,8 @@ func (m *ResizeInstruction) Size() (n int) { func (m *ResizeSource) Size() (n int) { var l int _ = l - if m.URI != nil { - l = m.URI.Size() + if m.Node != nil { + l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } l = len(m.Index) @@ -2978,8 +3051,8 @@ func (m *ResizeInstructionComplete) Size() (n int) { if m.JobID != 0 { n += 1 + sovPrivate(uint64(m.JobID)) } - if m.URI != nil { - l = m.URI.Size() + if m.Node != nil { + l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } l = len(m.Error) @@ -3006,16 +3079,16 @@ func (m *SetCoordinatorMessage) Size() (n int) { func (m *Topology) Size() (n int) { var l int _ = l - if len(m.NodeSet) > 0 { - for _, e := range m.NodeSet { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - } l = len(m.ClusterID) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if len(m.NodeIDs) > 0 { + for _, s := range m.NodeIDs { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } return n } @@ -5941,7 +6014,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } return nil } -func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { +func (m *Node) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -5964,13 +6037,42 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: NodeStateMessage: wiretype end group for non-group") + return fmt.Errorf("proto: Node: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: NodeStateMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Node: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) } @@ -6003,6 +6105,85 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeStateMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeStateMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) @@ -6084,7 +6265,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -6108,10 +6289,10 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.URI == nil { - m.URI = &URI{} + if m.Node == nil { + m.Node = &Node{} } - if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6232,6 +6413,35 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClusterID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) } @@ -6260,9 +6470,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } m.State = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NodeSet", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -6286,40 +6496,11 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeSet = append(m.NodeSet, &URI{}) - if err := m.NodeSet[len(m.NodeSet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Nodes = append(m.Nodes, &Node{}) + if err := m.Nodes[len(m.Nodes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ClusterID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -6811,7 +6992,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -6835,10 +7016,10 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.URI == nil { - m.URI = &URI{} + if m.Node == nil { + m.Node = &Node{} } - if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6869,7 +7050,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Coordinator == nil { - m.Coordinator = &URI{} + m.Coordinator = &Node{} } if err := m.Coordinator.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -7024,7 +7205,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7048,10 +7229,10 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.URI == nil { - m.URI = &URI{} + if m.Node == nil { + m.Node = &Node{} } - if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -7232,7 +7413,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -7256,10 +7437,10 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.URI == nil { - m.URI = &URI{} + if m.Node == nil { + m.Node = &Node{} } - if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -7369,7 +7550,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Old == nil { - m.Old = &URI{} + m.Old = &Node{} } if err := m.Old.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -7402,7 +7583,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.New == nil { - m.New = &URI{} + m.New = &Node{} } if err := m.New.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -7459,37 +7640,6 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NodeSet", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.NodeSet = append(m.NodeSet, &URI{}) - if err := m.NodeSet[len(m.NodeSet)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) } @@ -7518,6 +7668,35 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } m.ClusterID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeIDs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeIDs = append(m.NodeIDs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7647,82 +7826,84 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1229 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0xb6, 0x63, 0x3f, 0xc7, 0x8d, 0x33, 0xa4, 0xc1, 0x89, 0x22, 0xd7, 0xcc, 0x81, - 0x84, 0x4a, 0x04, 0x70, 0x25, 0x04, 0x41, 0x95, 0xa0, 0xb1, 0xab, 0x2e, 0x90, 0xa4, 0x1d, 0xa7, - 0x45, 0x70, 0x40, 0x9a, 0xd8, 0x43, 0xb2, 0xca, 0x7a, 0xd7, 0xec, 0x8e, 0x93, 0xb8, 0x07, 0x6e, - 0x70, 0x80, 0x3b, 0xe2, 0xce, 0x97, 0xe1, 0xc8, 0x47, 0x40, 0xe1, 0x43, 0x20, 0x71, 0x01, 0xcd, - 0xbf, 0xdd, 0xf5, 0xda, 0x4e, 0x9a, 0x88, 0xdb, 0xbe, 0x37, 0xbf, 0xf7, 0xe6, 0x37, 0xef, 0xdf, - 0xcc, 0x42, 0x75, 0x18, 0xba, 0x67, 0x94, 0xb3, 0xed, 0x61, 0x18, 0xf0, 0x00, 0x95, 0x5c, 0x9f, - 0xb3, 0xd0, 0xa7, 0x1e, 0x3e, 0x80, 0xb2, 0xe3, 0xf7, 0xd9, 0xc5, 0x1e, 0xe3, 0x14, 0x35, 0xa1, - 0xb2, 0x1b, 0x78, 0xa3, 0x81, 0xff, 0x05, 0x3d, 0x62, 0x5e, 0xdd, 0x6a, 0x5a, 0x5b, 0x65, 0x92, - 0x56, 0x09, 0xc4, 0xa1, 0x3b, 0x60, 0xcf, 0x46, 0xd4, 0xe7, 0xa3, 0x41, 0x3d, 0xa7, 0x10, 0x29, - 0x15, 0xfe, 0xc7, 0x82, 0xf2, 0xe3, 0x90, 0x0e, 0x98, 0xf4, 0xb8, 0x0e, 0x25, 0x12, 0x9c, 0xa7, - 0xdd, 0xc5, 0x32, 0x7a, 0x0b, 0xee, 0x38, 0xfe, 0x19, 0x0b, 0x23, 0xd6, 0xf1, 0xe9, 0x91, 0xc7, - 0xfa, 0xd2, 0x5d, 0x89, 0x64, 0xb4, 0x68, 0x03, 0xca, 0xbb, 0xb4, 0x77, 0xc2, 0x0e, 0xc7, 0x43, - 0x56, 0xb7, 0xa5, 0x93, 0x44, 0x11, 0xaf, 0x76, 0xdd, 0x97, 0xac, 0x9e, 0x6f, 0x5a, 0x5b, 0x55, - 0x92, 0x28, 0xb2, 0x7c, 0x0b, 0x53, 0x7c, 0x11, 0x86, 0x45, 0x42, 0xfd, 0xe3, 0x98, 0x43, 0x51, - 0x72, 0x98, 0xd0, 0xa1, 0x4d, 0x28, 0x3e, 0x76, 0x99, 0xd7, 0x8f, 0xea, 0x0b, 0x4d, 0x7b, 0xab, - 0xd2, 0x5a, 0xda, 0x36, 0xf1, 0xdb, 0x96, 0x7a, 0xa2, 0x97, 0x31, 0x86, 0x3b, 0xce, 0x60, 0x18, - 0x84, 0x9c, 0xb0, 0x68, 0x18, 0xf8, 0x11, 0x43, 0x35, 0xb0, 0x3b, 0x61, 0xa8, 0xcf, 0x2e, 0x3e, - 0xf1, 0xf7, 0x50, 0x7b, 0xe4, 0x05, 0xbd, 0xd3, 0x36, 0xe5, 0x94, 0xb0, 0xef, 0x46, 0x2c, 0xe2, - 0x68, 0x05, 0x0a, 0x32, 0x0b, 0x1a, 0xa7, 0x04, 0xa1, 0x95, 0x91, 0xd4, 0x61, 0x56, 0x82, 0xd0, - 0x4a, 0x7b, 0x19, 0x8a, 0x3c, 0x51, 0x82, 0xd0, 0x76, 0x3d, 0xb7, 0xa7, 0x42, 0x90, 0x27, 0x4a, - 0x40, 0x08, 0xf2, 0x2f, 0x5c, 0x76, 0xae, 0xcf, 0x2d, 0xbf, 0xb1, 0x03, 0xcb, 0xa9, 0xfd, 0x35, - 0xcd, 0x55, 0x28, 0x92, 0xe0, 0xdc, 0x69, 0x47, 0x75, 0xab, 0x69, 0x6f, 0xe5, 0x89, 0x96, 0x64, - 0x74, 0x65, 0xfa, 0xc5, 0x52, 0x4e, 0x2e, 0x25, 0x0a, 0xbc, 0x06, 0x05, 0x19, 0x6a, 0x71, 0xca, - 0xc4, 0x56, 0x7c, 0xe2, 0x7f, 0x2d, 0x28, 0xef, 0xd1, 0x0b, 0x49, 0x23, 0x42, 0x0f, 0xa1, 0xd4, - 0xe5, 0xd4, 0xef, 0xd3, 0xb0, 0x2f, 0x41, 0x95, 0xd6, 0x9b, 0x49, 0x08, 0x63, 0xd8, 0xb6, 0xc1, - 0x74, 0x7c, 0x1e, 0x8e, 0x49, 0x6c, 0x82, 0x76, 0x60, 0x41, 0xd7, 0x84, 0xe4, 0x50, 0x69, 0x35, - 0x67, 0x59, 0xc7, 0x65, 0x23, 0x8c, 0x8d, 0xc1, 0xfa, 0xc7, 0x50, 0x9d, 0x70, 0x2b, 0xb8, 0x9e, - 0xb2, 0xb1, 0xc9, 0xc8, 0x29, 0x1b, 0x8b, 0xd8, 0x9d, 0x51, 0x6f, 0xa4, 0xe2, 0x9c, 0x27, 0x4a, - 0xd8, 0xc9, 0x7d, 0x68, 0xad, 0xef, 0xc0, 0x62, 0xda, 0xeb, 0x4d, 0x6c, 0xf1, 0x37, 0x80, 0x76, - 0x43, 0x46, 0x39, 0x93, 0xf4, 0xf6, 0x58, 0x14, 0xd1, 0x63, 0x36, 0x3f, 0xd3, 0x2a, 0x7b, 0xb9, - 0x74, 0xf6, 0x36, 0xa0, 0xec, 0x44, 0xe6, 0xe0, 0xb6, 0xac, 0xcb, 0x44, 0x81, 0xef, 0x03, 0x6a, - 0x33, 0x8f, 0x71, 0xa6, 0xfb, 0xf7, 0x0a, 0xff, 0xb8, 0x6b, 0xb8, 0x5c, 0x8f, 0x45, 0x9b, 0x90, - 0x17, 0xad, 0x2b, 0xa9, 0x54, 0x5a, 0xaf, 0x27, 0x91, 0x8e, 0xe7, 0x04, 0x91, 0x00, 0xec, 0x1a, - 0xa7, 0xba, 0xdd, 0xaf, 0x39, 0xe0, 0x8c, 0x52, 0x36, 0x5b, 0xd9, 0xd9, 0xad, 0xe2, 0x01, 0xa2, - 0xb7, 0xfa, 0xc4, 0x9c, 0xf5, 0xb6, 0x5b, 0xe1, 0xaf, 0xb5, 0x56, 0xb4, 0xc4, 0xbe, 0x58, 0x55, - 0x36, 0xf2, 0x7b, 0xfe, 0x91, 0x33, 0x3c, 0x84, 0x6f, 0xd1, 0x43, 0x51, 0xdd, 0x6e, 0xda, 0xc2, - 0xb7, 0x14, 0xf0, 0x03, 0x28, 0x76, 0x7b, 0x27, 0x6c, 0x40, 0xd1, 0xdb, 0xa2, 0x50, 0xfb, 0xec, - 0x82, 0x45, 0xba, 0xcc, 0x97, 0x32, 0xe1, 0x23, 0x66, 0x1d, 0xff, 0x6c, 0x69, 0xf6, 0x73, 0x18, - 0x15, 0xe5, 0xde, 0x51, 0x3d, 0x3f, 0x35, 0x71, 0x84, 0x9e, 0xe8, 0x65, 0xd4, 0x81, 0x9a, 0xe3, - 0x0f, 0x47, 0xbc, 0xcd, 0xbe, 0x75, 0x7d, 0x97, 0xbb, 0x81, 0x1f, 0xd5, 0x8b, 0xd2, 0x64, 0x2d, - 0xbd, 0xf5, 0x04, 0x82, 0x4c, 0x99, 0xe0, 0x1f, 0x2d, 0x58, 0xca, 0x28, 0xaf, 0xe1, 0x95, 0xbb, - 0x9a, 0xd7, 0x07, 0xf1, 0xc8, 0xb4, 0x25, 0xb0, 0x31, 0x97, 0xcd, 0xe4, 0x04, 0xfd, 0xcd, 0x82, - 0x95, 0x59, 0x80, 0x99, 0x6c, 0x1a, 0x00, 0x4f, 0x43, 0x77, 0x40, 0xc3, 0xf1, 0xe7, 0x6c, 0xac, - 0x6f, 0x8f, 0x94, 0x06, 0x7d, 0x09, 0xab, 0x19, 0x5f, 0x9f, 0xf6, 0x54, 0x88, 0x14, 0xa9, 0x7b, - 0x73, 0x49, 0x29, 0x1c, 0x99, 0x63, 0x8e, 0xff, 0xb6, 0xe0, 0xee, 0xcc, 0xa5, 0xa4, 0xfa, 0xac, - 0x74, 0xa1, 0xdf, 0x87, 0xda, 0x0b, 0x31, 0x18, 0xda, 0x2c, 0xe2, 0xae, 0x4f, 0x05, 0x52, 0x97, - 0xe7, 0x94, 0x1e, 0x39, 0x50, 0x92, 0xba, 0x3d, 0x3a, 0xd4, 0x34, 0xdf, 0xb9, 0x86, 0xe6, 0xb6, - 0xc1, 0xeb, 0xb9, 0x69, 0x44, 0x41, 0x46, 0xce, 0x71, 0x73, 0x29, 0x48, 0x41, 0x4c, 0xc4, 0x09, - 0x83, 0x1b, 0x4d, 0xb5, 0x00, 0x36, 0xcc, 0x24, 0x99, 0x60, 0x72, 0x75, 0x4f, 0x7e, 0x04, 0x90, - 0x40, 0x75, 0xbb, 0x5f, 0x51, 0x9f, 0x29, 0x30, 0x7e, 0x02, 0x1b, 0x66, 0xcc, 0xdd, 0x60, 0x43, - 0x53, 0x2d, 0xb9, 0xa4, 0x5a, 0x70, 0x07, 0xec, 0xe7, 0xc4, 0x11, 0x57, 0x9d, 0xec, 0x56, 0x93, - 0x22, 0x2d, 0x09, 0x93, 0x27, 0x41, 0xc4, 0x8d, 0x89, 0xf8, 0x16, 0xba, 0xa7, 0x41, 0xc8, 0x25, - 0xe3, 0x2a, 0x91, 0xdf, 0xd8, 0x81, 0xda, 0x7e, 0xd0, 0x67, 0x5d, 0x4e, 0x79, 0x3c, 0x89, 0xee, - 0x49, 0xd7, 0xd2, 0x61, 0xa5, 0x55, 0x4d, 0x0e, 0xf6, 0x9c, 0x38, 0x44, 0x6e, 0x2a, 0x06, 0xbc, - 0x30, 0x30, 0x43, 0x49, 0x0a, 0xf8, 0x27, 0x0b, 0xc0, 0xf8, 0x1a, 0x45, 0xd7, 0x7b, 0x79, 0x3f, - 0x75, 0xa7, 0x4e, 0x0f, 0xab, 0x78, 0x89, 0xa4, 0x6e, 0xde, 0x2d, 0x33, 0x9b, 0x74, 0xd4, 0x6b, - 0x09, 0x5e, 0xe9, 0xf5, 0xf9, 0x29, 0xf6, 0xa0, 0xba, 0xeb, 0x8d, 0x22, 0xce, 0x42, 0x4d, 0x27, - 0xe6, 0x6c, 0xa5, 0x38, 0xa3, 0x4d, 0x58, 0x90, 0x94, 0x19, 0xd7, 0x23, 0x20, 0x43, 0xd4, 0xac, - 0xca, 0xa7, 0x83, 0xf2, 0xe7, 0xb4, 0xe3, 0x67, 0x9b, 0x51, 0xe0, 0x2e, 0x14, 0xe6, 0xf7, 0x35, - 0x82, 0xbc, 0x7c, 0xec, 0xe9, 0x54, 0xc8, 0x77, 0x5e, 0x0d, 0xec, 0x3d, 0x57, 0xd5, 0x8e, 0x4d, - 0xc4, 0xa7, 0xd4, 0xd0, 0x0b, 0x59, 0xdb, 0x42, 0x43, 0xc5, 0x35, 0xb7, 0xac, 0x8a, 0x53, 0xcc, - 0xe5, 0xdb, 0x5c, 0x48, 0xe6, 0xbd, 0x64, 0xa7, 0xde, 0x4b, 0x5d, 0x58, 0x56, 0x05, 0xf8, 0x7f, - 0x3a, 0xfd, 0x25, 0x07, 0xcb, 0x84, 0x45, 0xee, 0x4b, 0xe6, 0xf8, 0x11, 0x0f, 0x47, 0xf1, 0xf0, - 0xf8, 0x2c, 0x38, 0x72, 0xda, 0xd2, 0xab, 0x4d, 0x94, 0x60, 0xca, 0x22, 0x37, 0xb7, 0x2c, 0xde, - 0x15, 0xcf, 0xf6, 0x20, 0xec, 0x8b, 0x09, 0x12, 0x84, 0x3a, 0xd1, 0x19, 0x60, 0x1a, 0x81, 0xde, - 0x83, 0x85, 0x6e, 0x30, 0x0a, 0x7b, 0xf1, 0xf5, 0xb2, 0x9a, 0x80, 0x15, 0x2b, 0xb5, 0x4c, 0x0c, - 0x2c, 0x55, 0x46, 0x85, 0xab, 0xcb, 0x08, 0x3d, 0xcc, 0x94, 0x91, 0x7c, 0x50, 0x57, 0x5a, 0x6f, - 0x24, 0x06, 0x13, 0xcb, 0x64, 0x12, 0x8d, 0x7f, 0xb0, 0x60, 0x31, 0x4d, 0xe1, 0x95, 0x5a, 0x4b, - 0xa5, 0x22, 0x37, 0x33, 0x15, 0xf6, 0xac, 0x54, 0xe4, 0x93, 0x54, 0x24, 0x6f, 0xaf, 0x42, 0xea, - 0xed, 0x85, 0x4f, 0x60, 0x6d, 0x2a, 0x3f, 0xbb, 0xc1, 0x60, 0x28, 0x0a, 0xe1, 0xb6, 0x79, 0x5a, - 0x81, 0x42, 0x27, 0x0c, 0x75, 0x86, 0xca, 0x44, 0x09, 0xf8, 0x2b, 0xb8, 0xdb, 0x65, 0x3c, 0x95, - 0x9e, 0xd4, 0x50, 0x39, 0xf0, 0xfa, 0x73, 0x4e, 0x7e, 0xe0, 0xf5, 0x05, 0x60, 0x9f, 0x9d, 0xcf, - 0xd9, 0x70, 0x9f, 0x9d, 0xe3, 0x67, 0x50, 0x3a, 0x0c, 0x86, 0x81, 0x17, 0x1c, 0x8f, 0xd3, 0x7d, - 0x6b, 0xbd, 0x7a, 0xdf, 0xe6, 0x32, 0x7d, 0xfb, 0xa8, 0xf6, 0xfb, 0x65, 0xc3, 0xfa, 0xe3, 0xb2, - 0x61, 0xfd, 0x79, 0xd9, 0xb0, 0x7e, 0xfd, 0xab, 0xf1, 0xda, 0x51, 0x51, 0xfe, 0x52, 0x3e, 0xf8, - 0x2f, 0x00, 0x00, 0xff, 0xff, 0x59, 0x94, 0x30, 0x6d, 0x63, 0x0e, 0x00, 0x00, + // 1258 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5b, 0x6f, 0x1b, 0x45, + 0x14, 0x66, 0xbd, 0xb6, 0x63, 0x1f, 0xd7, 0xa9, 0x33, 0xb4, 0xc1, 0xad, 0x22, 0xd7, 0x8c, 0x10, + 0x0d, 0x95, 0x88, 0x8a, 0x2b, 0x71, 0x09, 0xaa, 0x54, 0x12, 0xbb, 0xea, 0x02, 0x49, 0xcb, 0x38, + 0x2d, 0x12, 0x48, 0x48, 0x13, 0x7b, 0x48, 0x57, 0x59, 0xef, 0x9a, 0xdd, 0x75, 0x12, 0xf7, 0x81, + 0x47, 0x84, 0x84, 0x78, 0x47, 0xbc, 0xf2, 0x67, 0x78, 0xe4, 0x27, 0xa0, 0xf0, 0x23, 0x90, 0x78, + 0x01, 0x9d, 0xb9, 0xec, 0xae, 0xaf, 0x21, 0x85, 0xb7, 0x3d, 0xdf, 0xb9, 0xcc, 0x37, 0xe7, 0x9c, + 0x39, 0x33, 0x0b, 0xd5, 0x61, 0xe8, 0x9e, 0xf0, 0x58, 0x6c, 0x0d, 0xc3, 0x20, 0x0e, 0x48, 0xc9, + 0xf5, 0x63, 0x11, 0xfa, 0xdc, 0xa3, 0x8f, 0xa1, 0xec, 0xf8, 0x7d, 0x71, 0xb6, 0x27, 0x62, 0x4e, + 0x9a, 0x50, 0xd9, 0x0d, 0xbc, 0xd1, 0xc0, 0xff, 0x94, 0x1f, 0x0a, 0xaf, 0x6e, 0x35, 0xad, 0xcd, + 0x32, 0xcb, 0x42, 0x68, 0x71, 0xe0, 0x0e, 0xc4, 0x67, 0x23, 0xee, 0xc7, 0xa3, 0x41, 0x3d, 0xa7, + 0x2c, 0x32, 0x10, 0xfd, 0xcb, 0x82, 0xf2, 0xc3, 0x90, 0x0f, 0x84, 0x8c, 0x78, 0x13, 0x4a, 0x2c, + 0x38, 0xcd, 0x86, 0x4b, 0x64, 0xf2, 0x26, 0xac, 0x3a, 0xfe, 0x89, 0x08, 0x23, 0xd1, 0xf1, 0xf9, + 0xa1, 0x27, 0xfa, 0x32, 0x5c, 0x89, 0x4d, 0xa1, 0x64, 0x03, 0xca, 0xbb, 0xbc, 0xf7, 0x5c, 0x1c, + 0x8c, 0x87, 0xa2, 0x6e, 0xcb, 0x20, 0x29, 0x90, 0x68, 0xbb, 0xee, 0x0b, 0x51, 0xcf, 0x37, 0xad, + 0xcd, 0x2a, 0x4b, 0x81, 0x69, 0xbe, 0x85, 0x19, 0xbe, 0x84, 0xc2, 0x15, 0xc6, 0xfd, 0xa3, 0x84, + 0x43, 0x51, 0x72, 0x98, 0xc0, 0xc8, 0x6d, 0x28, 0x3e, 0x74, 0x85, 0xd7, 0x8f, 0xea, 0x2b, 0x4d, + 0x7b, 0xb3, 0xd2, 0xba, 0xba, 0x65, 0xf2, 0xb7, 0x25, 0x71, 0xa6, 0xd5, 0x94, 0xc2, 0xaa, 0x33, + 0x18, 0x06, 0x61, 0xcc, 0x44, 0x34, 0x0c, 0xfc, 0x48, 0x90, 0x1a, 0xd8, 0x9d, 0x30, 0xd4, 0x7b, + 0xc7, 0x4f, 0xfa, 0x2d, 0xd4, 0x76, 0xbc, 0xa0, 0x77, 0xdc, 0xe6, 0x31, 0x67, 0xe2, 0x9b, 0x91, + 0x88, 0x62, 0x72, 0x0d, 0x0a, 0xb2, 0x0a, 0xda, 0x4e, 0x09, 0x88, 0xca, 0x4c, 0xea, 0x34, 0x2b, + 0x01, 0x51, 0xe9, 0x2f, 0x53, 0x91, 0x67, 0x4a, 0x40, 0xb4, 0xeb, 0xb9, 0x3d, 0x95, 0x82, 0x3c, + 0x53, 0x02, 0x21, 0x90, 0x7f, 0xe6, 0x8a, 0x53, 0xbd, 0x6f, 0xf9, 0x4d, 0x1d, 0x58, 0xcb, 0xac, + 0xaf, 0x69, 0xae, 0x43, 0x91, 0x05, 0xa7, 0x4e, 0x3b, 0xaa, 0x5b, 0x4d, 0x7b, 0x33, 0xcf, 0xb4, + 0x24, 0xb3, 0x2b, 0xcb, 0x8f, 0xaa, 0x9c, 0x54, 0xa5, 0x00, 0xbd, 0x01, 0x05, 0x99, 0x6a, 0xdc, + 0x65, 0xea, 0x8b, 0x9f, 0xf4, 0x6f, 0x0b, 0xca, 0x7b, 0xfc, 0x4c, 0xd2, 0x88, 0xc8, 0x7d, 0x28, + 0x75, 0x63, 0xee, 0xf7, 0x79, 0xd8, 0x97, 0x46, 0x95, 0xd6, 0xeb, 0x69, 0x0a, 0x13, 0xb3, 0x2d, + 0x63, 0xd3, 0xf1, 0xe3, 0x70, 0xcc, 0x12, 0x17, 0xb2, 0x0d, 0x2b, 0xba, 0x27, 0x24, 0x87, 0x4a, + 0xab, 0x39, 0xcf, 0x3b, 0x69, 0x1b, 0x74, 0x36, 0x0e, 0x37, 0x3f, 0x84, 0xea, 0x44, 0x58, 0xe4, + 0x7a, 0x2c, 0xc6, 0xa6, 0x22, 0xc7, 0x62, 0x8c, 0xb9, 0x3b, 0xe1, 0xde, 0x48, 0xe5, 0x39, 0xcf, + 0x94, 0xb0, 0x9d, 0x7b, 0xdf, 0xba, 0xb9, 0x0d, 0x57, 0xb2, 0x51, 0x2f, 0xe3, 0x4b, 0xbf, 0x02, + 0xb2, 0x1b, 0x0a, 0x1e, 0x0b, 0x49, 0x6f, 0x4f, 0x44, 0x11, 0x3f, 0x12, 0x8b, 0x2b, 0xad, 0xaa, + 0x97, 0xcb, 0x56, 0x6f, 0x03, 0xca, 0x4e, 0x64, 0x36, 0x6e, 0xcb, 0xbe, 0x4c, 0x01, 0x7a, 0x07, + 0x48, 0x5b, 0x78, 0x22, 0x16, 0xfa, 0xfc, 0x2e, 0x89, 0x4f, 0xbb, 0x86, 0xcb, 0xc5, 0xb6, 0xe4, + 0x36, 0xe4, 0xf1, 0xe8, 0x4a, 0x2a, 0x95, 0xd6, 0xab, 0x69, 0xa6, 0x93, 0x39, 0xc1, 0xa4, 0x01, + 0x75, 0x4d, 0x50, 0x7d, 0xdc, 0x2f, 0xd8, 0xe0, 0x9c, 0x56, 0x36, 0x4b, 0xd9, 0xd3, 0x4b, 0x25, + 0x03, 0x44, 0x2f, 0xf5, 0xc0, 0xec, 0xf5, 0x65, 0x97, 0xa2, 0x5f, 0x68, 0x14, 0x8f, 0xc4, 0x3e, + 0x6a, 0x95, 0x8f, 0xfc, 0x5e, 0xbc, 0xe5, 0x29, 0x1e, 0x18, 0x1b, 0xcf, 0x50, 0x54, 0xb7, 0x9b, + 0x36, 0xc6, 0x96, 0x02, 0xbd, 0x07, 0xc5, 0x6e, 0xef, 0xb9, 0x18, 0x70, 0xf2, 0x16, 0x36, 0x6a, + 0x5f, 0x9c, 0x89, 0x48, 0xb7, 0xf9, 0xd5, 0xa9, 0xf4, 0x31, 0xa3, 0xa7, 0x3f, 0x58, 0x9a, 0xfd, + 0x02, 0x46, 0x45, 0xb9, 0x76, 0x54, 0xcf, 0xcf, 0x4c, 0x1c, 0xc4, 0x99, 0x56, 0x93, 0x0e, 0xd4, + 0x1c, 0x7f, 0x38, 0x8a, 0xdb, 0xe2, 0x6b, 0xd7, 0x77, 0x63, 0x37, 0xf0, 0xa3, 0x7a, 0x51, 0xba, + 0xdc, 0xc8, 0x2e, 0x3d, 0x61, 0xc1, 0x66, 0x5c, 0xe8, 0x77, 0x16, 0x5c, 0x9d, 0x02, 0x2f, 0xe0, + 0x95, 0x5b, 0xce, 0xeb, 0xdd, 0x64, 0x64, 0xda, 0xd2, 0xb0, 0xb1, 0x90, 0xcd, 0xe4, 0x04, 0xfd, + 0xc5, 0x82, 0x6b, 0xf3, 0x0c, 0xe6, 0xb2, 0x69, 0x00, 0x3c, 0x09, 0xdd, 0x01, 0x0f, 0xc7, 0x9f, + 0x88, 0xb1, 0xbe, 0x3d, 0x32, 0x08, 0xf9, 0x1c, 0xd6, 0xa7, 0x62, 0x7d, 0xd4, 0x53, 0x29, 0x52, + 0xa4, 0x6e, 0x2d, 0x24, 0xa5, 0xec, 0xd8, 0x02, 0x77, 0xfa, 0xa7, 0x05, 0xd7, 0xe7, 0xaa, 0xd2, + 0xee, 0xb3, 0xb2, 0x8d, 0x7e, 0x07, 0x6a, 0xcf, 0x70, 0x30, 0xb4, 0x45, 0x14, 0xbb, 0x3e, 0x47, + 0x4b, 0xdd, 0x9e, 0x33, 0x38, 0x71, 0xa0, 0x24, 0xb1, 0x3d, 0x3e, 0xd4, 0x34, 0xdf, 0xbe, 0x80, + 0xe6, 0x96, 0xb1, 0xd7, 0x73, 0xd3, 0x88, 0x48, 0x46, 0xce, 0x71, 0x73, 0x29, 0x48, 0x01, 0x27, + 0xe2, 0x84, 0xc3, 0xa5, 0xa6, 0x5a, 0x00, 0x1b, 0x66, 0x92, 0x4c, 0x30, 0x59, 0x7e, 0x26, 0x3f, + 0x00, 0x48, 0x4d, 0xf5, 0x71, 0x5f, 0xd2, 0x9f, 0x19, 0x63, 0xfa, 0x08, 0x36, 0xcc, 0x98, 0xbb, + 0xc4, 0x82, 0xa6, 0x5b, 0x72, 0x69, 0xb7, 0xd0, 0x0e, 0xd8, 0x4f, 0x99, 0x83, 0x57, 0x9d, 0x3c, + 0xad, 0xa6, 0x44, 0x5a, 0x42, 0x97, 0x47, 0x41, 0x14, 0x1b, 0x17, 0xfc, 0x46, 0xec, 0x49, 0x10, + 0xc6, 0x92, 0x71, 0x95, 0xc9, 0x6f, 0xfa, 0x1e, 0xe4, 0xf7, 0x83, 0xbe, 0x20, 0xab, 0x90, 0x73, + 0xda, 0x3a, 0x46, 0xce, 0x69, 0x93, 0x5b, 0x32, 0xbc, 0x9e, 0x21, 0xd5, 0x74, 0x73, 0x4f, 0x99, + 0xc3, 0x50, 0x43, 0x1f, 0x40, 0x0d, 0x1d, 0xbb, 0x31, 0x8f, 0x93, 0x11, 0xb6, 0x0e, 0x45, 0xc4, + 0x92, 0x40, 0x5a, 0x92, 0x17, 0x02, 0xda, 0x99, 0x21, 0x26, 0x05, 0xfa, 0xa3, 0x05, 0x60, 0x42, + 0x8c, 0x22, 0x42, 0x15, 0x13, 0xe9, 0x5a, 0x69, 0xad, 0xa6, 0x4b, 0x22, 0xca, 0x14, 0xcb, 0x77, + 0x32, 0xd7, 0xf0, 0xec, 0x7c, 0x4b, 0x54, 0x2c, 0x73, 0x59, 0x6f, 0x9a, 0x71, 0xa6, 0x0b, 0x55, + 0x4b, 0xed, 0x15, 0xae, 0x53, 0x86, 0x37, 0x40, 0x75, 0xd7, 0x1b, 0x45, 0xb1, 0x08, 0x35, 0x23, + 0x7c, 0x2e, 0x28, 0x20, 0xd9, 0x51, 0x0a, 0xcc, 0xdf, 0x14, 0x79, 0x03, 0x0a, 0xc8, 0xd4, 0x9c, + 0xc9, 0xe9, 0x6d, 0x28, 0x25, 0xed, 0x42, 0x61, 0xf1, 0x1c, 0x20, 0x90, 0x97, 0x8f, 0x43, 0x5d, + 0x3a, 0xf9, 0x2e, 0xac, 0x81, 0xbd, 0xe7, 0xaa, 0x5e, 0xb3, 0x19, 0x7e, 0x4a, 0x84, 0x9f, 0xc9, + 0xb3, 0x80, 0x08, 0xc7, 0x6b, 0x71, 0x4d, 0x35, 0x33, 0xce, 0xf1, 0x97, 0xb9, 0xc0, 0xcc, 0xfb, + 0xca, 0xce, 0xbc, 0xaf, 0xba, 0xb0, 0xa6, 0x1a, 0xf6, 0xff, 0x0c, 0xfa, 0x73, 0x0e, 0xd6, 0x98, + 0x88, 0xdc, 0x17, 0xc2, 0xf1, 0xa3, 0x38, 0x1c, 0x25, 0xc3, 0xe6, 0xe3, 0xe0, 0x50, 0xa7, 0xda, + 0x66, 0x4a, 0x48, 0xda, 0x22, 0xb7, 0xa4, 0x2d, 0xee, 0xe2, 0x4b, 0x3f, 0x08, 0xfb, 0x38, 0x74, + 0x82, 0x50, 0x17, 0x7a, 0xda, 0x34, 0x6b, 0x42, 0xee, 0xc2, 0x4a, 0x37, 0x18, 0x85, 0xbd, 0xe4, + 0x4a, 0x5a, 0x4f, 0xad, 0x15, 0x33, 0xa5, 0x66, 0xc6, 0x2c, 0xd3, 0x47, 0x85, 0xe5, 0x7d, 0x44, + 0xee, 0x4f, 0xf5, 0x91, 0x7c, 0x84, 0x57, 0x5a, 0xaf, 0xa5, 0x0e, 0x13, 0x6a, 0x36, 0x69, 0x4d, + 0xbf, 0xb7, 0xe0, 0x4a, 0x96, 0xc2, 0xbf, 0x3a, 0x18, 0x49, 0x45, 0x72, 0x73, 0x2b, 0x62, 0xcf, + 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0x64, 0x2b, 0x64, 0x9e, 0x6c, 0xf4, 0x18, 0x6e, 0xcc, 0x94, + 0x69, 0x37, 0x18, 0x0c, 0xb1, 0x1f, 0xfe, 0x43, 0xb9, 0xae, 0x41, 0xa1, 0x13, 0x86, 0xba, 0x50, + 0x65, 0xa6, 0x04, 0xfa, 0x25, 0x5c, 0xef, 0x8a, 0x38, 0x53, 0x24, 0xd3, 0x6d, 0x4d, 0xb0, 0x1f, + 0x7b, 0xfd, 0x05, 0xdb, 0x47, 0x15, 0x5a, 0xec, 0x8b, 0xd3, 0x05, 0x6b, 0xa2, 0x8a, 0xee, 0x40, + 0xe9, 0x20, 0x18, 0x06, 0x5e, 0x70, 0x34, 0xbe, 0xe0, 0x58, 0xd7, 0x61, 0x45, 0x4d, 0x2d, 0xf5, + 0x28, 0x28, 0x33, 0x23, 0xee, 0xd4, 0x7e, 0x3d, 0x6f, 0x58, 0xbf, 0x9d, 0x37, 0xac, 0xdf, 0xcf, + 0x1b, 0xd6, 0x4f, 0x7f, 0x34, 0x5e, 0x39, 0x2c, 0xca, 0xff, 0xcf, 0x7b, 0xff, 0x04, 0x00, 0x00, + 0xff, 0xff, 0x03, 0x63, 0x93, 0xf4, 0x90, 0x0e, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 4877707a2..1686b3be2 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -121,21 +121,26 @@ message URI { uint32 Port = 3; } +message Node { + string ID = 1; + URI URI = 2; +} + message NodeStateMessage { - URI URI = 1; + string NodeID = 1; string State = 2; } message NodeStatus { - URI URI = 1; + Node Node = 1; MaxSlices MaxSlices = 2; Schema Schema = 3; } message ClusterStatus { - string State = 1; - repeated URI NodeSet = 2; - string ClusterID = 3; + string ClusterID = 1; + string State = 2; + repeated Node Nodes = 3; } message Field { @@ -159,15 +164,15 @@ message DeleteViewMessage { message ResizeInstruction { int64 JobID = 1; - URI URI = 2; - URI Coordinator = 3; + Node Node = 2; + Node Coordinator = 3; repeated ResizeSource Sources = 4; Schema Schema = 5; ClusterStatus ClusterStatus = 6; } message ResizeSource { - URI URI = 1; + Node Node = 1; string Index = 2; string Frame = 3; string View = 4; @@ -176,17 +181,17 @@ message ResizeSource { message ResizeInstructionComplete { int64 JobID = 1; - URI URI = 2; + Node Node = 2; string Error = 3; } message SetCoordinatorMessage { - URI Old = 1; - URI New = 2; + Node Old = 1; + Node New = 2; } message Topology { - repeated URI NodeSet = 1; - string ClusterID = 2; + string ClusterID = 1; + repeated string NodeIDs = 2; } diff --git a/pilosa.go b/pilosa.go index af4186513..6e7777804 100644 --- a/pilosa.go +++ b/pilosa.go @@ -162,8 +162,8 @@ func StringInSlice(a string, list []string) bool { return false } -// URISlicesAreEqual determines if two string slices are equal. -func URISlicesAreEqual(a, b []URI) bool { +// StringSlicesAreEqual determines if two string slices are equal. +func StringSlicesAreEqual(a, b []string) bool { if a == nil && b == nil { return true diff --git a/server.go b/server.go index 6b7d49cc5..7652a6c5b 100644 --- a/server.go +++ b/server.go @@ -58,9 +58,6 @@ type Server struct { wg sync.WaitGroup closing chan struct{} - // Unique name identifying the server. - Name string - // Data storage and HTTP interface. Holder *Holder Handler *Handler @@ -70,8 +67,8 @@ type Server struct { RemoteClient *http.Client // Cluster configuration. - // Host is replaced with actual host after opening if port is ":0". Network string + NodeID string URI URI Cluster *Cluster diagnostics *diagnostics.Diagnostics @@ -128,16 +125,15 @@ func (s *Server) Open() error { } } - // Set Cluster URI. - s.Cluster.URI = s.URI + // Get or create NodeID. + s.NodeID = s.LoadNodeID() - // Find the Node ID and append that tag to stats. - for i, n := range s.Cluster.Nodes { - if n.URI == s.URI { - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) - break - } - } + // Set Cluster Node. + node := &Node{ID: s.NodeID, URI: s.URI} + s.Cluster.Node = node + + // Append the NodeID tag to stats. + s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) // Peek at the holder to determine if there is data on disk. // Don't actually load the data until after the Cluster @@ -151,7 +147,7 @@ func (s *Server) Open() error { // Create executor for executing queries. e := NewExecutor(s.RemoteClient) e.Holder = s.Holder - e.URI = s.URI + e.Node = node e.Cluster = s.Cluster e.MaxWritesPerRequest = s.MaxWritesPerRequest @@ -163,7 +159,7 @@ func (s *Server) Open() error { s.Handler.Broadcaster = s.Broadcaster s.Handler.BroadcastHandler = s s.Handler.StatusHandler = s - s.Handler.URI = s.URI + s.Handler.Node = node s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput @@ -211,11 +207,6 @@ func (s *Server) Open() error { // buffered channel. s.Cluster.ListenForJoins() - // Load NodeID. - if err := s.Holder.loadNodeID(); err != nil { - s.Logger().Println(err) - } - // Start background monitoring. s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() @@ -259,11 +250,6 @@ func (s *Server) OpenListener() error { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) } - // If name is not provided in the config, default to the URI. - if s.Name == "" { - s.Name = s.URI.String() - } - return nil } @@ -286,6 +272,20 @@ func (s *Server) Close() error { return nil } +// LoadNodeID gets NodeID from disk, or creates a new value. +// If server.NodeID is already set, a new ID is not created. +func (s *Server) LoadNodeID() string { + if s.NodeID != "" { + return s.NodeID + } + nodeID, err := s.Holder.loadNodeID() + if err != nil { + s.Logger().Printf("loading NodeID: %v", err) + return s.NodeID + } + return nodeID +} + // Addr returns the address of the listener. func (s *Server) Addr() net.Addr { if s.ln == nil { @@ -336,7 +336,7 @@ func (s *Server) monitorAntiEntropy() { // Initialize syncer with local holder and remote client. var syncer HolderSyncer syncer.Holder = s.Holder - syncer.URI = s.URI + syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing syncer.RemoteClient = s.RemoteClient @@ -442,9 +442,9 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.SetCoordinatorMessage: - s.Cluster.SetCoordinator(DecodeURI(obj.Old), DecodeURI(obj.New)) + s.Cluster.SetCoordinator(DecodeNode(obj.Old), DecodeNode(obj.New)) case *internal.NodeStateMessage: - err := s.Cluster.ReceiveNodeState(DecodeURI(obj.URI), obj.State) + err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) if err != nil { return err } @@ -503,7 +503,7 @@ func (s *Server) LocalStatus() (proto.Message, error) { } ns := internal.NodeStatus{ - URI: encodeURI(s.URI), + Node: EncodeNode(s.Cluster.Node), MaxSlices: s.Holder.EncodeMaxSlices(), Schema: s.Holder.EncodeSchema(), } @@ -538,7 +538,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // Ignore status updates from self. - if s.URI == decodeURI(ns.URI) { + if s.NodeID == DecodeNode(ns.Node).ID { return nil } @@ -594,10 +594,10 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.SetInterval(s.DiagnosticInterval) s.diagnostics.Open() s.diagnostics.Set("Host", s.URI.host) - s.diagnostics.Set("Cluster", strings.Join(NodeSet(s.Cluster.NodeSet()).ToStrings(), ",")) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) - s.diagnostics.Set("NodeID", s.Holder.NodeID) + s.diagnostics.Set("NodeID", s.NodeID) s.diagnostics.Set("ClusterID", s.Cluster.ID) s.diagnostics.EnrichWithOSInfo() diff --git a/server/cluster_test.go b/server/cluster_test.go index 98b723335..f8789f169 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -38,8 +38,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Update cluster config m0.Server.Cluster.Nodes = []*pilosa.Node{ - {URI: m0.Server.URI}, - {URI: m1.Server.URI}, + {ID: m0.Server.NodeID, URI: m0.Server.URI}, + {ID: m1.Server.NodeID, URI: m1.Server.URI}, } m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes @@ -50,7 +50,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Config.Gossip.Seed = "" m0.Server.Cluster.Coordinator = m0.Server.URI - m0.Server.Cluster.Topology = &pilosa.Topology{NodeSet: []pilosa.URI{m0.Server.URI, m1.Server.URI}} + m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) if err != nil { diff --git a/server/server.go b/server/server.go index d238d1c2f..5b2b2010d 100644 --- a/server/server.go +++ b/server/server.go @@ -128,11 +128,6 @@ func (m *Command) SetupServer() error { } m.Server.URI = *uri - // If using a dynamically allocated port, server.Name will get set later. - if m.Config.Bind != "localhost:0" { - m.Server.Name = m.Server.URI.String() - } - cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN cluster.Holder = m.Server.Holder @@ -218,7 +213,6 @@ func (m *Command) SetupServer() error { func (m *Command) SetupNetworking() error { switch m.Config.Cluster.Type { case pilosa.ClusterGossip: - // Set internal port (string). gossipPortStr := pilosa.DefaultGossipPort // Config.GossipPort is deprecated, so Config.Gossip.Port has priority @@ -245,11 +239,10 @@ func (m *Command) SetupNetworking() error { } } + m.Server.NodeID = m.Server.LoadNodeID() + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - if m.Server.Name == "" { - return fmt.Errorf("must provide a valid name for gossip membership") - } - gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.Name, m.Config, transport, m.Server) + gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) if err != nil { return err } diff --git a/test/cluster.go b/test/cluster.go index ebe48fd35..55f5ad074 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "path/filepath" - "sort" "sync" "time" @@ -30,10 +29,14 @@ func NewCluster(n int) *pilosa.Cluster { for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ + ID: fmt.Sprintf("node%d", i), URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) } + c.Node = c.Nodes[0] + c.Coordinator = c.Nodes[0].URI + return c } @@ -84,7 +87,7 @@ type TestCluster struct { } type commonClusterSettings struct { - NodeSet pilosa.NodeSet + Nodes []*pilosa.Node } func (t *TestCluster) CreateIndex(name string) error { @@ -115,7 +118,7 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * nodes := c0.FragmentNodes(index, slice) for _, node := range nodes { - c := t.clusterByURI(node.URI) + c := t.clusterByID(node.ID) if c == nil { continue } @@ -139,7 +142,7 @@ func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name s nodes := c0.FragmentNodes(index, slice) for _, node := range nodes { - c := t.clusterByURI(node.URI) + c := t.clusterByID(node.ID) if c == nil { continue } @@ -156,9 +159,9 @@ func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name s return nil } -func (t *TestCluster) clusterByURI(uri pilosa.URI) *pilosa.Cluster { +func (t *TestCluster) clusterByID(id string) *pilosa.Cluster { for _, c := range t.Clusters { - if c.URI == uri { + if c.Node.ID == id { return c } } @@ -179,7 +182,7 @@ func (t *TestCluster) AddNode(saveTopology bool) error { coord := t.Clusters[0] ev := &pilosa.NodeEvent{ Event: pilosa.NodeJoin, - URI: c.URI, + Node: c.Node, } if err := coord.ReceiveEvent(ev); err != nil { @@ -211,11 +214,20 @@ func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { + id := fmt.Sprintf("node%d", i) uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) + node := &pilosa.Node{ + ID: id, + URI: uri, + } + // add URI to common - t.common.NodeSet = append(t.common.NodeSet, uri) - sort.Sort(t.common.NodeSet) + //t.common.NodeIDs = append(t.common.NodeIDs, id) + //sort.Sort(t.common.NodeIDs) + + // add node to common + t.common.Nodes = append(t.common.Nodes, node) // create node-specific temp directory path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) @@ -235,14 +247,14 @@ func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, err c.Topology = pilosa.NewTopology() c.Holder = h c.MemberSet = pilosa.NewStaticMemberSet() - c.URI = uri - c.Coordinator = t.common.NodeSet[0] // the first node is the coordinator + c.Node = node + c.Coordinator = t.common.Nodes[0].URI // the first node is the coordinator c.Broadcaster = t // add nodes if saveTopology { - for _, u := range t.common.NodeSet { - c.AddNode(u) + for _, n := range t.common.Nodes { + c.AddNode(n) } } @@ -344,7 +356,7 @@ func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { return err } case *internal.ResizeInstructionComplete: - coord := t.clusterByURI(to.URI) + coord := t.clusterByID(to.ID) go coord.MarkResizeInstructionComplete(obj) } return nil @@ -356,7 +368,7 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) // Prepare the return message. complete := &internal.ResizeInstructionComplete{ JobID: instr.JobID, - URI: instr.URI, + Node: instr.Node, Error: "", } @@ -365,8 +377,8 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) // figure out which node it was meant for, then call the operation on that cluster // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) - instrURI := pilosa.DecodeURI(instr.URI) - destCluster := t.clusterByURI(instrURI) + instrNode := pilosa.DecodeNode(instr.Node) + destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { @@ -374,8 +386,8 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) } for _, src := range instr.Sources { - srcURI := pilosa.DecodeURI(src.URI) - srcCluster := t.clusterByURI(srcURI) + srcNode := pilosa.DecodeNode(src.Node) + srcCluster := t.clusterByID(srcNode.ID) srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) @@ -414,9 +426,7 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) complete.Error = err.Error() } - node := &pilosa.Node{ - URI: pilosa.DecodeURI(instr.Coordinator), - } + node := pilosa.DecodeNode(instr.Coordinator) if err := t.SendTo(node, complete); err != nil { return err } diff --git a/test/executor.go b/test/executor.go index 5d6872626..85445719d 100644 --- a/test/executor.go +++ b/test/executor.go @@ -26,7 +26,7 @@ func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster - e.URI = cluster.Nodes[0].URI + e.Node = cluster.Nodes[0] return e } diff --git a/test/handler.go b/test/handler.go index 6c6703f84..4859443c6 100644 --- a/test/handler.go +++ b/test/handler.go @@ -67,7 +67,6 @@ func NewServer() *Server { if err != nil { panic(err) } - s.Handler.URI = *uri // Handler test messages can no-op. s.Handler.Broadcaster = pilosa.NopBroadcaster @@ -75,6 +74,8 @@ func NewServer() *Server { s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].URI = *uri + s.Handler.Node = s.Handler.Cluster.Nodes[0] + return s } @@ -85,10 +86,16 @@ func (s *Server) LocalStatus() (proto.Message, error) { // ClusterStatus exists so that test.Server implements StatusHandler. func (s *Server) ClusterStatus() (proto.Message, error) { + id := "test-node" uri := pilosa.DefaultURI() + node := &pilosa.Node{ + ID: id, + URI: *uri, + } return &internal.ClusterStatus{ - State: pilosa.ClusterStateNormal, - NodeSet: []*internal.URI{uri.Encode()}, + ClusterID: "", + State: pilosa.ClusterStateNormal, + Nodes: pilosa.EncodeNodes([]*pilosa.Node{node}), }, nil } diff --git a/uri.go b/uri.go index 388650044..beb5060b5 100644 --- a/uri.go +++ b/uri.go @@ -57,6 +57,16 @@ func DefaultURI() *URI { } } +type URIs []URI + +func (u URIs) HostPortStrings() []string { + s := make([]string, len(u)) + for i, a := range u { + s[i] = a.HostPort() + } + return s +} + // NewURIFromHostPort returns a URI with specified host and port. func NewURIFromHostPort(host string, port uint16) (*URI, error) { uri := DefaultURI() From 86dbbbf393fb6a4f3202184d7235c6dbfc70cf63 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 2 Feb 2018 15:51:24 -0600 Subject: [PATCH 065/311] don't require oldNode in SetCoordinator() --- cluster.go | 14 +-- cluster_test.go | 4 +- handler.go | 9 +- internal/private.pb.go | 223 ++++++++++++++++------------------------- internal/private.proto | 3 +- server.go | 2 +- 6 files changed, 93 insertions(+), 162 deletions(-) diff --git a/cluster.go b/cluster.go index 1ed810ea0..d531c90d6 100644 --- a/cluster.go +++ b/cluster.go @@ -297,22 +297,16 @@ func (c *Cluster) IsCoordinator() bool { return c.Static || c.Coordinator == c.Node.URI } -// SetCoordinator updates the Coordinator to new if it is -// currently old. Returns true if the Coordinator changed. -func (c *Cluster) SetCoordinator(o, n *Node) bool { - // Get old node. - oldNode := c.nodeByID(o.ID) - if oldNode == nil { - return false - } - +// SetCoordinator updates the Coordinator to n. +// Returns true if the Coordinator changed. +func (c *Cluster) SetCoordinator(n *Node) bool { // Get new node. newNode := c.nodeByID(n.ID) if newNode == nil { return false } - if c.Coordinator == oldNode.URI && oldNode != newNode { + if c.Coordinator != newNode.URI { c.Coordinator = newNode.URI return true } diff --git a/cluster_test.go b/cluster_test.go index 9bcacac9a..5a5fa4de4 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -517,14 +517,14 @@ func TestCluster_SetCoordinator(t *testing.T) { newNode := c.Nodes[1] // Set coordinator to the same value. - if set := c.SetCoordinator(oldNode, oldNode); set { + if c.SetCoordinator(oldNode) { t.Errorf("did not expect coordinator to change") } else if c.Coordinator != oldNode.URI { t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) } // Set coordinator to a new value. - if set := c.SetCoordinator(oldNode, newNode); !set { + if !c.SetCoordinator(newNode) { t.Errorf("expected coordinator to change") } else if c.Coordinator != newNode.URI { t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) diff --git a/handler.go b/handler.go index a12e7b391..dd438e977 100644 --- a/handler.go +++ b/handler.go @@ -1964,11 +1964,6 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r return } - oldNode := h.Cluster.nodeByURI(h.Cluster.Coordinator) - if oldNode == nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } newNode := h.Cluster.nodeByID(req.ID) if newNode == nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1979,7 +1974,6 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r // Send the set-coordinator message to all nodes. err := h.Broadcaster.SendSync( &internal.SetCoordinatorMessage{ - Old: EncodeNode(oldNode), New: EncodeNode(newNode), }) if err != nil { @@ -1987,7 +1981,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r } // Set Coordinator on local node. - _ = h.Cluster.SetCoordinator(oldNode, newNode) + _ = h.Cluster.SetCoordinator(newNode) return nil }(); err != nil { @@ -1997,7 +1991,6 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r // Encode response. if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ - Old: oldNode, New: newNode, }); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/internal/private.pb.go b/internal/private.pb.go index ae6247bb6..b584a10a9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1029,8 +1029,7 @@ func (m *ResizeInstructionComplete) GetError() string { } type SetCoordinatorMessage struct { - Old *Node `protobuf:"bytes,1,opt,name=Old" json:"Old,omitempty"` - New *Node `protobuf:"bytes,2,opt,name=New" json:"New,omitempty"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` } func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } @@ -1038,13 +1037,6 @@ func (m *SetCoordinatorMessage) String() string { return proto.Compac func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } -func (m *SetCoordinatorMessage) GetOld() *Node { - if m != nil { - return m.Old - } - return nil -} - func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { return m.New @@ -2405,26 +2397,16 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Old != nil { + if m.New != nil { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Old.Size())) - n21, err := m.Old.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n21, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n21 } - if m.New != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n22, err := m.New.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n22 - } return i, nil } @@ -3065,10 +3047,6 @@ func (m *ResizeInstructionComplete) Size() (n int) { func (m *SetCoordinatorMessage) Size() (n int) { var l int _ = l - if m.Old != nil { - l = m.Old.Size() - n += 1 + l + sovPrivate(uint64(l)) - } if m.New != nil { l = m.New.Size() n += 1 + l + sovPrivate(uint64(l)) @@ -7524,39 +7502,6 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Old", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Old == nil { - m.Old = &Node{} - } - if err := m.Old.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field New", wireType) } @@ -7826,84 +7771,84 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1258 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5b, 0x6f, 0x1b, 0x45, - 0x14, 0x66, 0xbd, 0xb6, 0x63, 0x1f, 0xd7, 0xa9, 0x33, 0xb4, 0xc1, 0xad, 0x22, 0xd7, 0x8c, 0x10, - 0x0d, 0x95, 0x88, 0x8a, 0x2b, 0x71, 0x09, 0xaa, 0x54, 0x12, 0xbb, 0xea, 0x02, 0x49, 0xcb, 0x38, - 0x2d, 0x12, 0x48, 0x48, 0x13, 0x7b, 0x48, 0x57, 0x59, 0xef, 0x9a, 0xdd, 0x75, 0x12, 0xf7, 0x81, - 0x47, 0x84, 0x84, 0x78, 0x47, 0xbc, 0xf2, 0x67, 0x78, 0xe4, 0x27, 0xa0, 0xf0, 0x23, 0x90, 0x78, - 0x01, 0x9d, 0xb9, 0xec, 0xae, 0xaf, 0x21, 0x85, 0xb7, 0x3d, 0xdf, 0xb9, 0xcc, 0x37, 0xe7, 0x9c, - 0x39, 0x33, 0x0b, 0xd5, 0x61, 0xe8, 0x9e, 0xf0, 0x58, 0x6c, 0x0d, 0xc3, 0x20, 0x0e, 0x48, 0xc9, - 0xf5, 0x63, 0x11, 0xfa, 0xdc, 0xa3, 0x8f, 0xa1, 0xec, 0xf8, 0x7d, 0x71, 0xb6, 0x27, 0x62, 0x4e, - 0x9a, 0x50, 0xd9, 0x0d, 0xbc, 0xd1, 0xc0, 0xff, 0x94, 0x1f, 0x0a, 0xaf, 0x6e, 0x35, 0xad, 0xcd, - 0x32, 0xcb, 0x42, 0x68, 0x71, 0xe0, 0x0e, 0xc4, 0x67, 0x23, 0xee, 0xc7, 0xa3, 0x41, 0x3d, 0xa7, - 0x2c, 0x32, 0x10, 0xfd, 0xcb, 0x82, 0xf2, 0xc3, 0x90, 0x0f, 0x84, 0x8c, 0x78, 0x13, 0x4a, 0x2c, - 0x38, 0xcd, 0x86, 0x4b, 0x64, 0xf2, 0x26, 0xac, 0x3a, 0xfe, 0x89, 0x08, 0x23, 0xd1, 0xf1, 0xf9, - 0xa1, 0x27, 0xfa, 0x32, 0x5c, 0x89, 0x4d, 0xa1, 0x64, 0x03, 0xca, 0xbb, 0xbc, 0xf7, 0x5c, 0x1c, - 0x8c, 0x87, 0xa2, 0x6e, 0xcb, 0x20, 0x29, 0x90, 0x68, 0xbb, 0xee, 0x0b, 0x51, 0xcf, 0x37, 0xad, - 0xcd, 0x2a, 0x4b, 0x81, 0x69, 0xbe, 0x85, 0x19, 0xbe, 0x84, 0xc2, 0x15, 0xc6, 0xfd, 0xa3, 0x84, - 0x43, 0x51, 0x72, 0x98, 0xc0, 0xc8, 0x6d, 0x28, 0x3e, 0x74, 0x85, 0xd7, 0x8f, 0xea, 0x2b, 0x4d, - 0x7b, 0xb3, 0xd2, 0xba, 0xba, 0x65, 0xf2, 0xb7, 0x25, 0x71, 0xa6, 0xd5, 0x94, 0xc2, 0xaa, 0x33, - 0x18, 0x06, 0x61, 0xcc, 0x44, 0x34, 0x0c, 0xfc, 0x48, 0x90, 0x1a, 0xd8, 0x9d, 0x30, 0xd4, 0x7b, - 0xc7, 0x4f, 0xfa, 0x2d, 0xd4, 0x76, 0xbc, 0xa0, 0x77, 0xdc, 0xe6, 0x31, 0x67, 0xe2, 0x9b, 0x91, - 0x88, 0x62, 0x72, 0x0d, 0x0a, 0xb2, 0x0a, 0xda, 0x4e, 0x09, 0x88, 0xca, 0x4c, 0xea, 0x34, 0x2b, - 0x01, 0x51, 0xe9, 0x2f, 0x53, 0x91, 0x67, 0x4a, 0x40, 0xb4, 0xeb, 0xb9, 0x3d, 0x95, 0x82, 0x3c, - 0x53, 0x02, 0x21, 0x90, 0x7f, 0xe6, 0x8a, 0x53, 0xbd, 0x6f, 0xf9, 0x4d, 0x1d, 0x58, 0xcb, 0xac, - 0xaf, 0x69, 0xae, 0x43, 0x91, 0x05, 0xa7, 0x4e, 0x3b, 0xaa, 0x5b, 0x4d, 0x7b, 0x33, 0xcf, 0xb4, - 0x24, 0xb3, 0x2b, 0xcb, 0x8f, 0xaa, 0x9c, 0x54, 0xa5, 0x00, 0xbd, 0x01, 0x05, 0x99, 0x6a, 0xdc, - 0x65, 0xea, 0x8b, 0x9f, 0xf4, 0x6f, 0x0b, 0xca, 0x7b, 0xfc, 0x4c, 0xd2, 0x88, 0xc8, 0x7d, 0x28, - 0x75, 0x63, 0xee, 0xf7, 0x79, 0xd8, 0x97, 0x46, 0x95, 0xd6, 0xeb, 0x69, 0x0a, 0x13, 0xb3, 0x2d, - 0x63, 0xd3, 0xf1, 0xe3, 0x70, 0xcc, 0x12, 0x17, 0xb2, 0x0d, 0x2b, 0xba, 0x27, 0x24, 0x87, 0x4a, - 0xab, 0x39, 0xcf, 0x3b, 0x69, 0x1b, 0x74, 0x36, 0x0e, 0x37, 0x3f, 0x84, 0xea, 0x44, 0x58, 0xe4, - 0x7a, 0x2c, 0xc6, 0xa6, 0x22, 0xc7, 0x62, 0x8c, 0xb9, 0x3b, 0xe1, 0xde, 0x48, 0xe5, 0x39, 0xcf, - 0x94, 0xb0, 0x9d, 0x7b, 0xdf, 0xba, 0xb9, 0x0d, 0x57, 0xb2, 0x51, 0x2f, 0xe3, 0x4b, 0xbf, 0x02, - 0xb2, 0x1b, 0x0a, 0x1e, 0x0b, 0x49, 0x6f, 0x4f, 0x44, 0x11, 0x3f, 0x12, 0x8b, 0x2b, 0xad, 0xaa, - 0x97, 0xcb, 0x56, 0x6f, 0x03, 0xca, 0x4e, 0x64, 0x36, 0x6e, 0xcb, 0xbe, 0x4c, 0x01, 0x7a, 0x07, - 0x48, 0x5b, 0x78, 0x22, 0x16, 0xfa, 0xfc, 0x2e, 0x89, 0x4f, 0xbb, 0x86, 0xcb, 0xc5, 0xb6, 0xe4, - 0x36, 0xe4, 0xf1, 0xe8, 0x4a, 0x2a, 0x95, 0xd6, 0xab, 0x69, 0xa6, 0x93, 0x39, 0xc1, 0xa4, 0x01, - 0x75, 0x4d, 0x50, 0x7d, 0xdc, 0x2f, 0xd8, 0xe0, 0x9c, 0x56, 0x36, 0x4b, 0xd9, 0xd3, 0x4b, 0x25, - 0x03, 0x44, 0x2f, 0xf5, 0xc0, 0xec, 0xf5, 0x65, 0x97, 0xa2, 0x5f, 0x68, 0x14, 0x8f, 0xc4, 0x3e, - 0x6a, 0x95, 0x8f, 0xfc, 0x5e, 0xbc, 0xe5, 0x29, 0x1e, 0x18, 0x1b, 0xcf, 0x50, 0x54, 0xb7, 0x9b, - 0x36, 0xc6, 0x96, 0x02, 0xbd, 0x07, 0xc5, 0x6e, 0xef, 0xb9, 0x18, 0x70, 0xf2, 0x16, 0x36, 0x6a, - 0x5f, 0x9c, 0x89, 0x48, 0xb7, 0xf9, 0xd5, 0xa9, 0xf4, 0x31, 0xa3, 0xa7, 0x3f, 0x58, 0x9a, 0xfd, - 0x02, 0x46, 0x45, 0xb9, 0x76, 0x54, 0xcf, 0xcf, 0x4c, 0x1c, 0xc4, 0x99, 0x56, 0x93, 0x0e, 0xd4, - 0x1c, 0x7f, 0x38, 0x8a, 0xdb, 0xe2, 0x6b, 0xd7, 0x77, 0x63, 0x37, 0xf0, 0xa3, 0x7a, 0x51, 0xba, - 0xdc, 0xc8, 0x2e, 0x3d, 0x61, 0xc1, 0x66, 0x5c, 0xe8, 0x77, 0x16, 0x5c, 0x9d, 0x02, 0x2f, 0xe0, - 0x95, 0x5b, 0xce, 0xeb, 0xdd, 0x64, 0x64, 0xda, 0xd2, 0xb0, 0xb1, 0x90, 0xcd, 0xe4, 0x04, 0xfd, - 0xc5, 0x82, 0x6b, 0xf3, 0x0c, 0xe6, 0xb2, 0x69, 0x00, 0x3c, 0x09, 0xdd, 0x01, 0x0f, 0xc7, 0x9f, - 0x88, 0xb1, 0xbe, 0x3d, 0x32, 0x08, 0xf9, 0x1c, 0xd6, 0xa7, 0x62, 0x7d, 0xd4, 0x53, 0x29, 0x52, - 0xa4, 0x6e, 0x2d, 0x24, 0xa5, 0xec, 0xd8, 0x02, 0x77, 0xfa, 0xa7, 0x05, 0xd7, 0xe7, 0xaa, 0xd2, - 0xee, 0xb3, 0xb2, 0x8d, 0x7e, 0x07, 0x6a, 0xcf, 0x70, 0x30, 0xb4, 0x45, 0x14, 0xbb, 0x3e, 0x47, - 0x4b, 0xdd, 0x9e, 0x33, 0x38, 0x71, 0xa0, 0x24, 0xb1, 0x3d, 0x3e, 0xd4, 0x34, 0xdf, 0xbe, 0x80, - 0xe6, 0x96, 0xb1, 0xd7, 0x73, 0xd3, 0x88, 0x48, 0x46, 0xce, 0x71, 0x73, 0x29, 0x48, 0x01, 0x27, - 0xe2, 0x84, 0xc3, 0xa5, 0xa6, 0x5a, 0x00, 0x1b, 0x66, 0x92, 0x4c, 0x30, 0x59, 0x7e, 0x26, 0x3f, - 0x00, 0x48, 0x4d, 0xf5, 0x71, 0x5f, 0xd2, 0x9f, 0x19, 0x63, 0xfa, 0x08, 0x36, 0xcc, 0x98, 0xbb, - 0xc4, 0x82, 0xa6, 0x5b, 0x72, 0x69, 0xb7, 0xd0, 0x0e, 0xd8, 0x4f, 0x99, 0x83, 0x57, 0x9d, 0x3c, - 0xad, 0xa6, 0x44, 0x5a, 0x42, 0x97, 0x47, 0x41, 0x14, 0x1b, 0x17, 0xfc, 0x46, 0xec, 0x49, 0x10, - 0xc6, 0x92, 0x71, 0x95, 0xc9, 0x6f, 0xfa, 0x1e, 0xe4, 0xf7, 0x83, 0xbe, 0x20, 0xab, 0x90, 0x73, - 0xda, 0x3a, 0x46, 0xce, 0x69, 0x93, 0x5b, 0x32, 0xbc, 0x9e, 0x21, 0xd5, 0x74, 0x73, 0x4f, 0x99, - 0xc3, 0x50, 0x43, 0x1f, 0x40, 0x0d, 0x1d, 0xbb, 0x31, 0x8f, 0x93, 0x11, 0xb6, 0x0e, 0x45, 0xc4, - 0x92, 0x40, 0x5a, 0x92, 0x17, 0x02, 0xda, 0x99, 0x21, 0x26, 0x05, 0xfa, 0xa3, 0x05, 0x60, 0x42, - 0x8c, 0x22, 0x42, 0x15, 0x13, 0xe9, 0x5a, 0x69, 0xad, 0xa6, 0x4b, 0x22, 0xca, 0x14, 0xcb, 0x77, - 0x32, 0xd7, 0xf0, 0xec, 0x7c, 0x4b, 0x54, 0x2c, 0x73, 0x59, 0x6f, 0x9a, 0x71, 0xa6, 0x0b, 0x55, - 0x4b, 0xed, 0x15, 0xae, 0x53, 0x86, 0x37, 0x40, 0x75, 0xd7, 0x1b, 0x45, 0xb1, 0x08, 0x35, 0x23, - 0x7c, 0x2e, 0x28, 0x20, 0xd9, 0x51, 0x0a, 0xcc, 0xdf, 0x14, 0x79, 0x03, 0x0a, 0xc8, 0xd4, 0x9c, - 0xc9, 0xe9, 0x6d, 0x28, 0x25, 0xed, 0x42, 0x61, 0xf1, 0x1c, 0x20, 0x90, 0x97, 0x8f, 0x43, 0x5d, - 0x3a, 0xf9, 0x2e, 0xac, 0x81, 0xbd, 0xe7, 0xaa, 0x5e, 0xb3, 0x19, 0x7e, 0x4a, 0x84, 0x9f, 0xc9, - 0xb3, 0x80, 0x08, 0xc7, 0x6b, 0x71, 0x4d, 0x35, 0x33, 0xce, 0xf1, 0x97, 0xb9, 0xc0, 0xcc, 0xfb, - 0xca, 0xce, 0xbc, 0xaf, 0xba, 0xb0, 0xa6, 0x1a, 0xf6, 0xff, 0x0c, 0xfa, 0x73, 0x0e, 0xd6, 0x98, - 0x88, 0xdc, 0x17, 0xc2, 0xf1, 0xa3, 0x38, 0x1c, 0x25, 0xc3, 0xe6, 0xe3, 0xe0, 0x50, 0xa7, 0xda, - 0x66, 0x4a, 0x48, 0xda, 0x22, 0xb7, 0xa4, 0x2d, 0xee, 0xe2, 0x4b, 0x3f, 0x08, 0xfb, 0x38, 0x74, - 0x82, 0x50, 0x17, 0x7a, 0xda, 0x34, 0x6b, 0x42, 0xee, 0xc2, 0x4a, 0x37, 0x18, 0x85, 0xbd, 0xe4, - 0x4a, 0x5a, 0x4f, 0xad, 0x15, 0x33, 0xa5, 0x66, 0xc6, 0x2c, 0xd3, 0x47, 0x85, 0xe5, 0x7d, 0x44, - 0xee, 0x4f, 0xf5, 0x91, 0x7c, 0x84, 0x57, 0x5a, 0xaf, 0xa5, 0x0e, 0x13, 0x6a, 0x36, 0x69, 0x4d, - 0xbf, 0xb7, 0xe0, 0x4a, 0x96, 0xc2, 0xbf, 0x3a, 0x18, 0x49, 0x45, 0x72, 0x73, 0x2b, 0x62, 0xcf, - 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0x64, 0x2b, 0x64, 0x9e, 0x6c, 0xf4, 0x18, 0x6e, 0xcc, 0x94, - 0x69, 0x37, 0x18, 0x0c, 0xb1, 0x1f, 0xfe, 0x43, 0xb9, 0xae, 0x41, 0xa1, 0x13, 0x86, 0xba, 0x50, - 0x65, 0xa6, 0x04, 0xfa, 0x25, 0x5c, 0xef, 0x8a, 0x38, 0x53, 0x24, 0xd3, 0x6d, 0x4d, 0xb0, 0x1f, - 0x7b, 0xfd, 0x05, 0xdb, 0x47, 0x15, 0x5a, 0xec, 0x8b, 0xd3, 0x05, 0x6b, 0xa2, 0x8a, 0xee, 0x40, - 0xe9, 0x20, 0x18, 0x06, 0x5e, 0x70, 0x34, 0xbe, 0xe0, 0x58, 0xd7, 0x61, 0x45, 0x4d, 0x2d, 0xf5, - 0x28, 0x28, 0x33, 0x23, 0xee, 0xd4, 0x7e, 0x3d, 0x6f, 0x58, 0xbf, 0x9d, 0x37, 0xac, 0xdf, 0xcf, - 0x1b, 0xd6, 0x4f, 0x7f, 0x34, 0x5e, 0x39, 0x2c, 0xca, 0xff, 0xcf, 0x7b, 0xff, 0x04, 0x00, 0x00, - 0xff, 0xff, 0x03, 0x63, 0x93, 0xf4, 0x90, 0x0e, 0x00, 0x00, + // 1250 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, + 0x17, 0x7e, 0xd7, 0xbb, 0x76, 0xec, 0xe3, 0x38, 0x71, 0xe6, 0x4d, 0x83, 0x13, 0x45, 0xae, 0x19, + 0x21, 0x1a, 0x2a, 0x11, 0x15, 0x57, 0x02, 0x1a, 0x54, 0xa9, 0x24, 0x76, 0xd5, 0x05, 0x12, 0xca, + 0x38, 0x2d, 0x12, 0x17, 0x48, 0x13, 0x7b, 0x48, 0x57, 0x59, 0xef, 0x9a, 0xdd, 0x71, 0x12, 0xf7, + 0x82, 0x4b, 0x84, 0x84, 0xb8, 0x47, 0xdc, 0xf2, 0x67, 0xb8, 0xe4, 0x27, 0xa0, 0xf0, 0x23, 0x90, + 0xb8, 0x01, 0xcd, 0xd7, 0xee, 0xfa, 0x33, 0xa4, 0x70, 0xb7, 0xe7, 0x39, 0x1f, 0xf3, 0xcc, 0x39, + 0x67, 0xce, 0xcc, 0x42, 0x65, 0x10, 0x79, 0xe7, 0x94, 0xb3, 0xdd, 0x41, 0x14, 0xf2, 0x10, 0x15, + 0xbd, 0x80, 0xb3, 0x28, 0xa0, 0x3e, 0xfe, 0x14, 0x4a, 0x6e, 0xd0, 0x63, 0x97, 0x87, 0x8c, 0x53, + 0xd4, 0x80, 0xf2, 0x41, 0xe8, 0x0f, 0xfb, 0xc1, 0x27, 0xf4, 0x84, 0xf9, 0x35, 0xab, 0x61, 0xed, + 0x94, 0x48, 0x16, 0x12, 0x16, 0xc7, 0x5e, 0x9f, 0x7d, 0x36, 0xa4, 0x01, 0x1f, 0xf6, 0x6b, 0x39, + 0x65, 0x91, 0x81, 0xf0, 0x9f, 0x16, 0x94, 0x1e, 0x47, 0xb4, 0xcf, 0x64, 0xc4, 0x2d, 0x28, 0x92, + 0xf0, 0x22, 0x1b, 0x2e, 0x91, 0xd1, 0x9b, 0xb0, 0xe2, 0x06, 0xe7, 0x2c, 0x8a, 0x59, 0x3b, 0xa0, + 0x27, 0x3e, 0xeb, 0xc9, 0x70, 0x45, 0x32, 0x81, 0xa2, 0x6d, 0x28, 0x1d, 0xd0, 0xee, 0x0b, 0x76, + 0x3c, 0x1a, 0xb0, 0x9a, 0x2d, 0x83, 0xa4, 0x40, 0xa2, 0xed, 0x78, 0x2f, 0x59, 0xcd, 0x69, 0x58, + 0x3b, 0x15, 0x92, 0x02, 0x93, 0x7c, 0xf3, 0x53, 0x7c, 0x11, 0x86, 0x65, 0x42, 0x83, 0xd3, 0x84, + 0x43, 0x41, 0x72, 0x18, 0xc3, 0xd0, 0x1d, 0x28, 0x3c, 0xf6, 0x98, 0xdf, 0x8b, 0x6b, 0x4b, 0x0d, + 0x7b, 0xa7, 0xdc, 0x5c, 0xdd, 0x35, 0xf9, 0xdb, 0x95, 0x38, 0xd1, 0x6a, 0x8c, 0x61, 0xc5, 0xed, + 0x0f, 0xc2, 0x88, 0x13, 0x16, 0x0f, 0xc2, 0x20, 0x66, 0xa8, 0x0a, 0x76, 0x3b, 0x8a, 0xf4, 0xde, + 0xc5, 0x27, 0xfe, 0x06, 0xaa, 0xfb, 0x7e, 0xd8, 0x3d, 0x6b, 0x51, 0x4e, 0x09, 0xfb, 0x7a, 0xc8, + 0x62, 0x8e, 0xd6, 0x21, 0x2f, 0xab, 0xa0, 0xed, 0x94, 0x20, 0x50, 0x99, 0x49, 0x9d, 0x66, 0x25, + 0x08, 0x54, 0xfa, 0xcb, 0x54, 0x38, 0x44, 0x09, 0x02, 0xed, 0xf8, 0x5e, 0x57, 0xa5, 0xc0, 0x21, + 0x4a, 0x40, 0x08, 0x9c, 0xe7, 0x1e, 0xbb, 0xd0, 0xfb, 0x96, 0xdf, 0xd8, 0x85, 0xb5, 0xcc, 0xfa, + 0x9a, 0xe6, 0x06, 0x14, 0x48, 0x78, 0xe1, 0xb6, 0xe2, 0x9a, 0xd5, 0xb0, 0x77, 0x1c, 0xa2, 0x25, + 0x99, 0x5d, 0x59, 0x7e, 0xa1, 0xca, 0x49, 0x55, 0x0a, 0xe0, 0x4d, 0xc8, 0xcb, 0x54, 0x8b, 0x5d, + 0xa6, 0xbe, 0xe2, 0x13, 0xff, 0x65, 0x41, 0xe9, 0x90, 0x5e, 0x4a, 0x1a, 0x31, 0x7a, 0x08, 0xc5, + 0x0e, 0xa7, 0x41, 0x8f, 0x46, 0x3d, 0x69, 0x54, 0x6e, 0xbe, 0x9e, 0xa6, 0x30, 0x31, 0xdb, 0x35, + 0x36, 0xed, 0x80, 0x47, 0x23, 0x92, 0xb8, 0xa0, 0x3d, 0x58, 0xd2, 0x3d, 0x21, 0x39, 0x94, 0x9b, + 0x8d, 0x59, 0xde, 0x49, 0xdb, 0x08, 0x67, 0xe3, 0xb0, 0xf5, 0x01, 0x54, 0xc6, 0xc2, 0x0a, 0xae, + 0x67, 0x6c, 0x64, 0x2a, 0x72, 0xc6, 0x46, 0x22, 0x77, 0xe7, 0xd4, 0x1f, 0xaa, 0x3c, 0x3b, 0x44, + 0x09, 0x7b, 0xb9, 0xf7, 0xad, 0xad, 0x3d, 0x58, 0xce, 0x46, 0xbd, 0x89, 0x2f, 0xfe, 0x12, 0xd0, + 0x41, 0xc4, 0x28, 0x67, 0x92, 0xde, 0x21, 0x8b, 0x63, 0x7a, 0xca, 0xe6, 0x57, 0x5a, 0x55, 0x2f, + 0x97, 0xad, 0xde, 0x36, 0x94, 0xdc, 0xd8, 0x6c, 0xdc, 0x96, 0x7d, 0x99, 0x02, 0xf8, 0x2e, 0xa0, + 0x16, 0xf3, 0x19, 0x67, 0xfa, 0xfc, 0x2e, 0x88, 0x8f, 0x3b, 0x86, 0xcb, 0xf5, 0xb6, 0xe8, 0x0e, + 0x38, 0xe2, 0xe8, 0x4a, 0x2a, 0xe5, 0xe6, 0xff, 0xd3, 0x4c, 0x27, 0x73, 0x82, 0x48, 0x03, 0xec, + 0x99, 0xa0, 0xfa, 0xb8, 0x5f, 0xb3, 0xc1, 0x19, 0xad, 0x6c, 0x96, 0xb2, 0x27, 0x97, 0x4a, 0x06, + 0x88, 0x5e, 0xea, 0x91, 0xd9, 0xeb, 0xab, 0x2e, 0x85, 0xbf, 0xd0, 0xa8, 0x38, 0x12, 0x47, 0x42, + 0xab, 0x7c, 0xe4, 0xf7, 0xfc, 0x2d, 0x4f, 0xf0, 0x10, 0xb1, 0xc5, 0x19, 0x8a, 0x6b, 0x76, 0xc3, + 0x16, 0xb1, 0xa5, 0x80, 0xef, 0x43, 0xa1, 0xd3, 0x7d, 0xc1, 0xfa, 0x14, 0xbd, 0x25, 0x1a, 0xb5, + 0xc7, 0x2e, 0x59, 0xac, 0xdb, 0x7c, 0x75, 0x22, 0x7d, 0xc4, 0xe8, 0xf1, 0xf7, 0x96, 0x66, 0x3f, + 0x87, 0x51, 0x41, 0xae, 0x1d, 0xd7, 0x9c, 0xa9, 0x89, 0x23, 0x70, 0xa2, 0xd5, 0xa8, 0x0d, 0x55, + 0x37, 0x18, 0x0c, 0x79, 0x8b, 0x7d, 0xe5, 0x05, 0x1e, 0xf7, 0xc2, 0x20, 0xae, 0x15, 0xa4, 0xcb, + 0x66, 0x76, 0xe9, 0x31, 0x0b, 0x32, 0xe5, 0x82, 0xbf, 0xb5, 0x60, 0x75, 0x02, 0xbc, 0x86, 0x57, + 0x6e, 0x31, 0xaf, 0x77, 0x93, 0x91, 0x69, 0x4b, 0xc3, 0xfa, 0x5c, 0x36, 0xe3, 0x13, 0xf4, 0x67, + 0x0b, 0xd6, 0x67, 0x19, 0xcc, 0x64, 0x53, 0x07, 0x78, 0x1a, 0x79, 0x7d, 0x1a, 0x8d, 0x3e, 0x66, + 0x23, 0x7d, 0x7b, 0x64, 0x10, 0xf4, 0x39, 0x6c, 0x4c, 0xc4, 0xfa, 0xb0, 0xab, 0x52, 0xa4, 0x48, + 0xdd, 0x9e, 0x4b, 0x4a, 0xd9, 0x91, 0x39, 0xee, 0xf8, 0x0f, 0x0b, 0x6e, 0xcd, 0x54, 0xa5, 0xdd, + 0x67, 0x65, 0x1b, 0xfd, 0x2e, 0x54, 0x9f, 0x8b, 0xc1, 0xd0, 0x62, 0x31, 0xf7, 0x02, 0x2a, 0x2c, + 0x75, 0x7b, 0x4e, 0xe1, 0xc8, 0x85, 0xa2, 0xc4, 0x0e, 0xe9, 0x40, 0xd3, 0x7c, 0xfb, 0x1a, 0x9a, + 0xbb, 0xc6, 0x5e, 0xcf, 0x4d, 0x23, 0x0a, 0x32, 0x72, 0x8e, 0x9b, 0x4b, 0x41, 0x0a, 0x62, 0x22, + 0x8e, 0x39, 0xdc, 0x68, 0xaa, 0x85, 0xb0, 0x6d, 0x26, 0xc9, 0x18, 0x93, 0xc5, 0x67, 0xf2, 0x01, + 0x40, 0x6a, 0xaa, 0x8f, 0xfb, 0x82, 0xfe, 0xcc, 0x18, 0xe3, 0x27, 0xb0, 0x6d, 0xc6, 0xdc, 0x0d, + 0x16, 0x34, 0xdd, 0x92, 0x4b, 0xbb, 0x05, 0xb7, 0xc1, 0x7e, 0x46, 0x5c, 0x71, 0xd5, 0xc9, 0xd3, + 0x6a, 0x4a, 0xa4, 0x25, 0xe1, 0xf2, 0x24, 0x8c, 0xb9, 0x71, 0x11, 0xdf, 0x02, 0x7b, 0x1a, 0x46, + 0x5c, 0x32, 0xae, 0x10, 0xf9, 0x8d, 0xdf, 0x03, 0xe7, 0x28, 0xec, 0x31, 0xb4, 0x02, 0x39, 0xb7, + 0xa5, 0x63, 0xe4, 0xdc, 0x16, 0xba, 0x2d, 0xc3, 0xeb, 0x19, 0x52, 0x49, 0x37, 0xf7, 0x8c, 0xb8, + 0x44, 0x68, 0xf0, 0x23, 0xa8, 0x0a, 0xc7, 0x0e, 0xa7, 0x3c, 0x19, 0x61, 0x1b, 0x50, 0x10, 0x58, + 0x12, 0x48, 0x4b, 0xf2, 0x42, 0x10, 0x76, 0x66, 0x88, 0x49, 0x01, 0xff, 0x60, 0x01, 0x98, 0x10, + 0xc3, 0x18, 0x61, 0xc5, 0x44, 0xba, 0x96, 0x9b, 0x2b, 0xe9, 0x92, 0x02, 0x25, 0x8a, 0xe5, 0x3b, + 0x99, 0x6b, 0x78, 0x7a, 0xbe, 0x25, 0x2a, 0x92, 0xb9, 0xac, 0x77, 0xcc, 0x38, 0xd3, 0x85, 0xaa, + 0xa6, 0xf6, 0x0a, 0xd7, 0x29, 0x13, 0x37, 0x40, 0xe5, 0xc0, 0x1f, 0xc6, 0x9c, 0x45, 0x9a, 0x91, + 0x78, 0x2e, 0x28, 0x20, 0xd9, 0x51, 0x0a, 0xcc, 0xde, 0x14, 0x7a, 0x03, 0xf2, 0x82, 0xa9, 0x39, + 0x93, 0x93, 0xdb, 0x50, 0x4a, 0xdc, 0x81, 0xfc, 0xfc, 0x39, 0x80, 0xc0, 0x91, 0x8f, 0x43, 0x5d, + 0x3a, 0xf9, 0x2e, 0xac, 0x82, 0x7d, 0xe8, 0xa9, 0x5e, 0xb3, 0x89, 0xf8, 0x94, 0x08, 0xbd, 0x94, + 0x67, 0x41, 0x20, 0x54, 0x5c, 0x8b, 0x6b, 0xaa, 0x99, 0xc5, 0x1c, 0x7f, 0x95, 0x0b, 0xcc, 0xbc, + 0xaf, 0xec, 0xcc, 0xfb, 0xaa, 0x03, 0x6b, 0xaa, 0x61, 0xff, 0xcb, 0xa0, 0x3f, 0xe5, 0x60, 0x8d, + 0xb0, 0xd8, 0x7b, 0xc9, 0xdc, 0x20, 0xe6, 0xd1, 0x30, 0x19, 0x36, 0x1f, 0x85, 0x27, 0x3a, 0xd5, + 0x36, 0x51, 0x42, 0xd2, 0x16, 0xb9, 0x05, 0x6d, 0x71, 0x4f, 0xbc, 0xf4, 0xc3, 0xa8, 0x27, 0x86, + 0x4e, 0x18, 0xe9, 0x42, 0x4f, 0x9a, 0x66, 0x4d, 0xd0, 0x3d, 0x58, 0xea, 0x84, 0xc3, 0xa8, 0x9b, + 0x5c, 0x49, 0x1b, 0xa9, 0xb5, 0x62, 0xa6, 0xd4, 0xc4, 0x98, 0x65, 0xfa, 0x28, 0xbf, 0xb8, 0x8f, + 0xd0, 0xc3, 0x89, 0x3e, 0x92, 0x8f, 0xf0, 0x72, 0xf3, 0xb5, 0xd4, 0x61, 0x4c, 0x4d, 0xc6, 0xad, + 0xf1, 0x77, 0x16, 0x2c, 0x67, 0x29, 0xfc, 0xa3, 0x83, 0x91, 0x54, 0x24, 0x37, 0xb3, 0x22, 0xf6, + 0xac, 0x8a, 0x38, 0x69, 0x45, 0xd2, 0x27, 0x5b, 0x3e, 0xf3, 0x64, 0xc3, 0x67, 0xb0, 0x39, 0x55, + 0xa6, 0x83, 0xb0, 0x3f, 0x10, 0xfd, 0xf0, 0x2f, 0xca, 0xb5, 0x0e, 0xf9, 0x76, 0x14, 0xe9, 0x42, + 0x95, 0x88, 0x12, 0xf0, 0x03, 0xb8, 0xd5, 0x61, 0x3c, 0x53, 0x24, 0xd3, 0x6d, 0x0d, 0xb0, 0x8f, + 0xd8, 0xc5, 0x9c, 0xed, 0x0b, 0x15, 0xde, 0x87, 0xe2, 0x71, 0x38, 0x08, 0xfd, 0xf0, 0x74, 0x74, + 0xcd, 0xa1, 0xad, 0xc1, 0x92, 0x9a, 0x49, 0xea, 0xca, 0x2f, 0x11, 0x23, 0xee, 0x57, 0x7f, 0xb9, + 0xaa, 0x5b, 0xbf, 0x5e, 0xd5, 0xad, 0xdf, 0xae, 0xea, 0xd6, 0x8f, 0xbf, 0xd7, 0xff, 0x77, 0x52, + 0x90, 0x7f, 0x97, 0xf7, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x5d, 0xc5, 0x2e, 0x66, 0x6e, 0x0e, + 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 1686b3be2..4e3261079 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -186,8 +186,7 @@ message ResizeInstructionComplete { } message SetCoordinatorMessage { - Node Old = 1; - Node New = 2; + Node New = 1; } message Topology { diff --git a/server.go b/server.go index 7652a6c5b..63c14a88e 100644 --- a/server.go +++ b/server.go @@ -442,7 +442,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.SetCoordinatorMessage: - s.Cluster.SetCoordinator(DecodeNode(obj.Old), DecodeNode(obj.New)) + s.Cluster.SetCoordinator(DecodeNode(obj.New)) case *internal.NodeStateMessage: err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) if err != nil { From 346e92a91dd2e9d7449953e4d6edf1e9004c0833 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 2 Feb 2018 15:58:08 -0600 Subject: [PATCH 066/311] return 0 values for errors. panic on unmarshal node meta data --- cluster.go | 6 +++--- gossip/gossip.go | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index d531c90d6..76642107b 100644 --- a/cluster.go +++ b/cluster.go @@ -633,12 +633,12 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) lenTo := len(other.Nodes) // Determine if a node is being added or removed. if lenFrom == lenTo { - return action, nodeID, errors.New("clusters are the same size") + return "", "", errors.New("clusters are the same size") } if lenFrom < lenTo { // Adding a node. if lenTo-lenFrom > 1 { - return action, nodeID, errors.New("adding more than one node at a time is not supported") + return "", "", errors.New("adding more than one node at a time is not supported") } action = ResizeJobActionAdd // Determine the node ID that is being added. @@ -651,7 +651,7 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) } else if lenFrom > lenTo { // Removing a node. if lenFrom-lenTo > 1 { - return action, nodeID, errors.New("removing more than one node at a time is not supported") + return "", "", errors.New("removing more than one node at a time is not supported") } action = ResizeJobActionRemove // Determine the node ID that is being removed. diff --git a/gossip/gossip.go b/gossip/gossip.go index d0868be92..acfad66a1 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -384,8 +384,7 @@ func (g *GossipEventReceiver) listen() { // Get the node from the event.Node meta data. var n internal.Node if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { - // TODO: consider logging error - continue + panic("failed to unmarshal event node meta data") } node := pilosa.DecodeNode(&n) From eca01e4b4fa8e8418dc58a75a175e5a4cb92f91d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 6 Feb 2018 08:51:38 -0600 Subject: [PATCH 067/311] replace sha1 hash with faster xxhash --- Gopkg.lock | 16 ++++++++++++++-- attr.go | 6 ++++-- fragment.go | 7 ++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 7837cadff..76fcccbd4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,6 +37,12 @@ revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" +[[projects]] + name = "github.com/cespare/xxhash" + packages = ["."] + revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" + version = "v1.0.0" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -189,10 +195,16 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["host","internal/common","mem","process"] + packages = ["cpu","host","internal/common","mem","net","process"] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -268,6 +280,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56" + inputs-digest = "53ae0cdcbfa8419b233a0544d7ebb80614af3dd04bebdc0c8f2d3620a87efa91" solver-name = "gps-cdcl" solver-version = 1 diff --git a/attr.go b/attr.go index ab29fc9c0..437cab433 100644 --- a/attr.go +++ b/attr.go @@ -16,13 +16,15 @@ package pilosa import ( "bytes" - "crypto/sha1" + "encoding/binary" "fmt" "sort" "sync" "time" + "github.com/cespare/xxhash" + "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -242,7 +244,7 @@ func (s *AttrStore) Blocks() ([]AttrBlock, error) { block := AttrBlock{ID: cur.blockID()} // Compute checksum of every key/value in block. - h := sha1.New() + h := xxhash.New() for k, v := cur.next(); k != nil; k, v = cur.next() { h.Write(k) h.Write(v) diff --git a/fragment.go b/fragment.go index 930c8e02b..95bcb7f2d 100644 --- a/fragment.go +++ b/fragment.go @@ -20,7 +20,6 @@ import ( "bytes" "container/heap" "context" - "crypto/sha1" "encoding/binary" "errors" "fmt" @@ -36,6 +35,8 @@ import ( "time" "unsafe" + "github.com/cespare/xxhash" + "math" "github.com/gogo/protobuf/proto" @@ -1020,7 +1021,7 @@ type TopOptions struct { // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { - h := sha1.New() + h := xxhash.New() for _, block := range f.Blocks() { h.Write(block.Checksum) } @@ -1660,7 +1661,7 @@ type blockHasher struct { func newBlockHasher() blockHasher { return blockHasher{ blockID: -1, - hash: sha1.New(), + hash: xxhash.New(), } } func (h *blockHasher) Reset() { From a01a64aec5d28a71174537ec8b5c34a209388c82 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 6 Feb 2018 10:51:03 -0600 Subject: [PATCH 068/311] remove unneeded dep --- Gopkg.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 76fcccbd4..ed0469fc0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -199,12 +199,6 @@ revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" -[[projects]] - branch = "master" - name = "github.com/shirou/w32" - packages = ["."] - revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" - [[projects]] name = "github.com/sony/gobreaker" packages = ["."] From 5529040514fa0aa5499add048d69c774e58e33b6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 7 Feb 2018 11:29:58 -0600 Subject: [PATCH 069/311] clarify the options in configuration for metrics.service --- docs/administration.md | 6 +++--- docs/configuration.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 795d6f1c4..b77d05262 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -150,9 +150,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: - - Host to receive events - - Polling interval for runtime metrics - - Metric type (StatsD, Expvar). + - [Host](../configuration#metrics-host): specify host that receives metric events + - [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics + - [Service](../configuration#metrics-service): declare type StatsD or Expvar ##### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: diff --git a/docs/configuration.md b/docs/configuration.md index 43b70c8f1..2385f2494 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -220,7 +220,7 @@ Any flag that has a value that is a comma separated list on the command line bec cpu-time = "30s" ``` ##### Metric Service -* Description: Which stats service to use (StatsD or ExpVar). +* Description: Which stats service to use. Choose from [statsd, expvar]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' * Config: From 9085c03ce813f083fbe64dd7d803914c826f983c Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 8 Feb 2018 10:48:06 -0600 Subject: [PATCH 070/311] Changes configuration cluster.type (string) to cluster.disabled (bool) --- cmd/server_test.go | 6 +-- config.go | 41 +++---------------- config_test.go | 24 +++-------- ctl/server.go | 2 +- pilosa.go | 3 +- server/cluster_test.go | 20 +++++----- server/server.go | 82 +++++++++++++++++++------------------- server/server_test.go | 4 +- test/pilosa.go | 90 +++++++++++++++++++----------------------- test/pilosa_test.go | 4 +- 10 files changed, 112 insertions(+), 164 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index c9b900bde..b1f39e921 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -52,7 +52,7 @@ func TestServerConfig(t *testing.T) { max-writes-per-request = 3000 [cluster] - type = "static" + disabled = true replicas = 2 hosts = [ "localhost:19444", @@ -78,7 +78,7 @@ func TestServerConfig(t *testing.T) { bind = "localhost:0" data-dir = "` + actualDataDir + `" [cluster] - type = "static" + disabled = true hosts = [ "localhost:19444", ] @@ -92,7 +92,7 @@ func TestServerConfig(t *testing.T) { }, // TEST 2 { - args: []string{"server", "--log-path", logFile.Name(), "--cluster.type", "static"}, + args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"}, env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"}, cfgFileContent: ` bind = "localhost:19444" diff --git a/config.go b/config.go index a43963f93..05f132db2 100644 --- a/config.go +++ b/config.go @@ -35,8 +35,8 @@ const ( // DefaultPort is the default port to use with the hostname. DefaultPort = "10101" - // DefaultClusterType sets the node intercommunication method. - DefaultClusterType = ClusterGossip + // DefaultClusterDisabled sets the node intercommunication method. + DefaultClusterDisabled = false // DefaultMetrics sets the internal metrics to no-op. DefaultMetrics = "nop" @@ -113,9 +113,6 @@ const ( DefaultMetricPollInterval = 0 * time.Minute ) -// ClusterTypes set of cluster types. -var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterGossip} - // TLSConfig contains TLS configuration type TLSConfig struct { // CertificatePath contains the path to the certificate (.crt or .pem file) @@ -145,9 +142,9 @@ type Config struct { TLS TLSConfig Cluster struct { + Disabled bool `toml:"disabled"` Coordinator string `toml:"coordinator"` ReplicaN int `toml:"replicas"` - Type string `toml:"type"` Hosts []string `toml:"hosts"` LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` @@ -189,9 +186,9 @@ func NewConfig() *Config { } // Cluster config. + c.Cluster.Disabled = DefaultClusterDisabled // c.Cluster.Coordinator = "" c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.Type = DefaultClusterType c.Cluster.Hosts = []string{} c.Cluster.LongQueryTime = Duration(time.Minute) @@ -222,38 +219,12 @@ func NewConfig() *Config { // Validate that all configuration permutations are compatible with each other. func (c *Config) Validate() error { - if !StringInSlice(c.Cluster.Type, ClusterTypes) { - return ErrConfigClusterTypeInvalid + if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 { + return ErrConfigClusterEnabledHosts } - - if c.Cluster.Type == ClusterGossip { - if len(c.Cluster.Hosts) > 0 { - bindWithDefaults, err := AddressWithDefaults(c.Bind) - if err != nil { - return err - } - if !c.foundHost(bindWithDefaults) { - return ErrConfigHostsMissing - } - } - } - return nil } -func (c *Config) foundHost(host *URI) bool { - for _, clusterHost := range c.Cluster.Hosts { - uri, err := NewURIFromAddress(clusterHost) - if err != nil { - continue - } - if host.Equals(uri) { - return true - } - } - return false -} - // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/config_test.go b/config_test.go index c409fd80f..38a3fc640 100644 --- a/config_test.go +++ b/config_test.go @@ -11,27 +11,15 @@ import ( func Test_NewConfig(t *testing.T) { c := pilosa.NewConfig() + if c.Cluster.Disabled != pilosa.DefaultClusterDisabled { + t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled) + } + + // Ensure that hosts can't be specificed on a non-disabled cluster. c.Cluster.Hosts = []string{c.Bind, "localhost:10102"} // Change cluster type from the default (gossip) to an invalid string. - c.Cluster.Type = "invalid-type" - if err := c.Validate(); err != pilosa.ErrConfigClusterTypeInvalid { - t.Fatal(err) - } - - // Change cluster type back to gossip. - c.Cluster.Type = pilosa.ClusterGossip - - // Check for bind address in cluster hosts. - c.Bind = "localhost:1" - if err := c.Validate(); err != pilosa.ErrConfigHostsMissing { - t.Fatal(err) - } - - c.Bind = "localhost:10101" - c.Cluster.ReplicaN = 2 - c.GossipSeed = "localhost:14000" - if err := c.Validate(); err != nil { + if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts { t.Fatal(err) } } diff --git a/ctl/server.go b/ctl/server.go index 1dddeb0d7..3c5b1a058 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -35,9 +35,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) // Cluster + flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") - flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") diff --git a/pilosa.go b/pilosa.go index 6e7777804..98c67203f 100644 --- a/pilosa.go +++ b/pilosa.go @@ -73,8 +73,7 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") - ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") + ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed") ) // Regular expression to validate index and frame names. diff --git a/server/cluster_test.go b/server/cluster_test.go index f8789f169..114a25b0c 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -217,7 +217,7 @@ func TestClusterResize_EmptyNode(t *testing.T) { // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { // Configure node0 - m0 := test.NewMain() + m0 := test.NewMainWithCluster() defer m0.Close() gossipHost := "localhost" @@ -228,7 +228,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { } // Configure node1 - m1 := test.NewMain() + m1 := test.NewMainWithCluster() defer m1.Close() seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, seed, &coord) @@ -247,7 +247,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { // Configure node0 - m0 := test.NewMain() + m0 := test.NewMainWithCluster() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -256,7 +256,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMain() + m1 := test.NewMainWithCluster() defer m1.Close() var eg errgroup.Group @@ -281,7 +281,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := test.NewMain() + m0 := test.NewMainWithCluster() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -300,7 +300,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMain() + m1 := test.NewMainWithCluster() defer m1.Close() var eg errgroup.Group @@ -327,7 +327,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Run("ContinuousSlices", func(t *testing.T) { // Configure node0 - m0 := test.NewMain() + m0 := test.NewMainWithCluster() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -355,7 +355,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMain() + m1 := test.NewMainWithCluster() defer m1.Close() var eg errgroup.Group @@ -382,7 +382,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Run("SkippedSlice", func(t *testing.T) { // Configure node0 - m0 := test.NewMain() + m0 := test.NewMainWithCluster() defer m0.Close() seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) @@ -410,7 +410,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMain() + m1 := test.NewMainWithCluster() defer m1.Close() var eg errgroup.Group diff --git a/server/server.go b/server/server.go index 8379ef581..ef07ee381 100644 --- a/server/server.go +++ b/server/server.go @@ -211,46 +211,7 @@ func (m *Command) SetupServer() error { // SetupNetworking sets up internode communication based on the configuration. func (m *Command) SetupNetworking() error { - switch m.Config.Cluster.Type { - case pilosa.ClusterGossip: - // Set internal port (string). - gossipPortStr := pilosa.DefaultGossipPort - // Config.GossipPort is deprecated, so Config.Gossip.Port has priority - if m.Config.Gossip.Port != "" { - gossipPortStr = m.Config.Gossip.Port - } else if m.Config.GossipPort != "" { - gossipPortStr = m.Config.GossipPort - } - - gossipPort, err := strconv.Atoi(gossipPortStr) - if err != nil { - return err - } - - // get the host portion of addr to use for binding - gossipHost := m.Server.URI.Host() - var transport *gossip.Transport - if m.GossipTransport != nil { - transport = m.GossipTransport - } else { - transport, err = gossip.NewTransport(gossipHost, gossipPort) - if err != nil { - return err - } - } - - m.Server.NodeID = m.Server.LoadNodeID() - - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() - gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) - if err != nil { - return err - } - m.Server.Cluster.MemberSet = gossipMemberSet - m.Server.Broadcaster = m.Server - m.Server.BroadcastReceiver = gossipMemberSet - m.Server.Gossiper = gossipMemberSet - case pilosa.ClusterStatic, pilosa.ClusterNone: + if m.Config.Cluster.Disabled { m.Server.Cluster.Static = true for _, address := range m.Config.Cluster.Hosts { uri, err := pilosa.NewURIFromAddress(address) @@ -270,9 +231,46 @@ func (m *Command) SetupNetworking() error { if err != nil { return err } - default: - return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type) + return nil } + + // Set internal port (string). + gossipPortStr := pilosa.DefaultGossipPort + // Config.GossipPort is deprecated, so Config.Gossip.Port has priority + if m.Config.Gossip.Port != "" { + gossipPortStr = m.Config.Gossip.Port + } else if m.Config.GossipPort != "" { + gossipPortStr = m.Config.GossipPort + } + + gossipPort, err := strconv.Atoi(gossipPortStr) + if err != nil { + return err + } + + // get the host portion of addr to use for binding + gossipHost := m.Server.URI.Host() + var transport *gossip.Transport + if m.GossipTransport != nil { + transport = m.GossipTransport + } else { + transport, err = gossip.NewTransport(gossipHost, gossipPort) + if err != nil { + return err + } + } + + m.Server.NodeID = m.Server.LoadNodeID() + + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) + if err != nil { + return err + } + m.Server.Cluster.MemberSet = gossipMemberSet + m.Server.Broadcaster = m.Server + m.Server.BroadcastReceiver = gossipMemberSet + m.Server.Gossiper = gossipMemberSet return nil } diff --git a/server/server_test.go b/server/server_test.go index 352fc6121..c821ccdf8 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -269,7 +269,7 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { - mains1 := test.NewMainArrayWithCluster(2) + mains1 := test.MustRunMainWithCluster(t, 2) m10 := mains1[0] m11 := mains1[1] @@ -303,7 +303,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Start second cluster. - mains2 := test.NewMainArrayWithCluster(2) + mains2 := test.MustRunMainWithCluster(t, 2) m20 := mains2[0] defer m20.Close() m21 := mains2[1] diff --git a/test/pilosa.go b/test/pilosa.go index cbe259040..a467a2f5c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -37,7 +37,7 @@ func NewMain() *Main { m.Server.Network = *Network m.Config.DataDir = path m.Config.Bind = "localhost:0" - m.Config.Cluster.Type = "static" + m.Config.Cluster.Disabled = true m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr @@ -50,16 +50,50 @@ func NewMain() *Main { return m } -func NewMainArrayWithCluster(size int) []*Main { - cluster, err := NewServerCluster(size) +// NewMainWithCluster returns a new instance of Main with clustering enabled. +func NewMainWithCluster() *Main { + m := NewMain() + m.Config.Cluster.Disabled = false + return m +} + +// MustRunMainWithCluster ruturns a running array of *Main where +// all nodes are joined via memberlist (i.e. clustering enabled). +func MustRunMainWithCluster(t *testing.T, size int) []*Main { + ma, err := runMainWithCluster(size) if err != nil { - panic(err) + t.Fatalf("new main array with cluster: %v", err) } - mainArray := make([]*Main, size) + return ma +} + +// runMainWithCluster runs an array of *Main where all nodes are +// joined via memberlist (i.e. clustering enabled). +func runMainWithCluster(size int) ([]*Main, error) { + if size == 0 { + return nil, errors.New("cluster must contain at least one node") + } + + mains := make([]*Main, size) + + gossipHost := "localhost" + gossipPort := 0 + var err error + var gossipSeed string + var coordinator pilosa.URI + for i := 0; i < size; i++ { - mainArray[i] = cluster.Servers[i] + m := NewMainWithCluster() + + gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, &coordinator) + if err != nil { + return nil, errors.Wrap(err, "RunWithTransport") + } + + mains[i] = m } - return mainArray + + return mains, nil } // MustRunMain returns a new, running Main. Panic on error. @@ -101,8 +135,6 @@ func (m *Main) Reopen() error { func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) { defer close(m.Started) - m.Config.Cluster.Type = "gossip" - /* TEST: - SetupServer (just static settings from config) @@ -202,46 +234,6 @@ func (m *Main) CreateDefinition(index, def, query string) (string, error) { //////////////////////////////////////////////////////////////////////////////////// -type Cluster struct { - Servers []*Main -} - -func MustNewServerCluster(t *testing.T, size int) *Cluster { - cluster, err := NewServerCluster(size) - if err != nil { - t.Fatalf("new cluster: %v", err) - } - return cluster -} - -func NewServerCluster(size int) (cluster *Cluster, err error) { - if size == 0 { - return nil, errors.New("cluster must contain at least one node") - } - - cluster = &Cluster{ - Servers: make([]*Main, size), - } - - gossipHost := "localhost" - gossipPort := 0 - var gossipSeed string - var coordinator pilosa.URI - - for i := 0; i < size; i++ { - m := NewMain() - - gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, &coordinator) - if err != nil { - return nil, errors.Wrap(err, "RunWithTransport") - } - - cluster.Servers[i] = m - } - - return cluster, nil -} - // MustDo executes http.Do() with an http.NewRequest(). Panic on error. func MustDo(method, urlStr string, body string) *httpResponse { req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 8478e6bba..0d0f168d5 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -10,9 +10,9 @@ import ( ) func TestNewCluster(t *testing.T) { - cluster := test.MustNewServerCluster(t, 3) + cluster := test.MustRunMainWithCluster(t, 3) - response, err := http.Get("http://" + cluster.Servers[0].Server.Addr().String() + "/status") + response, err := http.Get("http://" + cluster[0].Server.Addr().String() + "/status") if err != nil { t.Fatalf("getting schema: %v", err) } From 01c2ced8aeee4b86d99466f967b4a9cf51f060fe Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Feb 2018 13:09:00 -0600 Subject: [PATCH 071/311] Add back missing WebUI handler --- handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/handler.go b/handler.go index dd438e977..270b9c072 100644 --- a/handler.go +++ b/handler.go @@ -124,6 +124,8 @@ func (h *Handler) SetRestricted() { } func loadCommon(router *mux.Router, handler *Handler) { + router.HandleFunc("/", handler.handleWebUI).Methods("GET") + router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") @@ -143,7 +145,6 @@ func loadRestricted(router *mux.Router, handler *Handler) { } func loadNormal(router *mux.Router, handler *Handler) { - router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") From ab1a1c7e82e986a72b006f42b263e3e19f19befb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Feb 2018 13:25:14 -0600 Subject: [PATCH 072/311] Add test for WebUI --- handler_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/handler_test.go b/handler_test.go index 0f2b2dc00..9c8ce2bf4 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1839,3 +1839,30 @@ func TestHandler_RecalculateCaches(t *testing.T) { } } + +func TestHandler_WebUI(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.NewHandler() + h.Holder = hldr.Holder + h.Cluster = test.NewCluster(1) + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + if !strings.Contains(w.Body.String(), "Pilosa WebUI") { + t.Fatalf("WebUI is not being served correctly.") + } + + // If curl is the client, the response should be different + w = httptest.NewRecorder() + req := test.MustNewHTTPRequest("GET", "/", nil) + req.Header.Add("User-Agent", "curl/7.54.0") + h.ServeHTTP(w, req) + if !strings.Contains(w.Body.String(), "try the WebUI") { + t.Fatalf("WebUI is not being served correctly.") + } +} From da2ab31945cd59a9660eb2cf20395d1c63622105 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Feb 2018 13:47:57 -0600 Subject: [PATCH 073/311] Add generate-statik to CI job so WebUI test passes --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 308f84877..6efa61e43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ env: - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" install: - - make vendor + - make vendor generate-statik script: - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 From 1085be6582a4f24a2e9c144ae1ed193e97c493f7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 8 Feb 2018 15:07:04 -0600 Subject: [PATCH 074/311] fix bug in NewServerCluster where each host was its own coordinator --- server/cluster_test.go | 20 ++++++++++---------- test/pilosa.go | 13 ++++--------- test/pilosa_test.go | 9 ++++++++- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/server/cluster_test.go b/server/cluster_test.go index 114a25b0c..23284bb00 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -222,7 +222,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { gossipHost := "localhost" gossipPort := 0 - seed, coord, err := m0.RunWithTransport(gossipHost, gossipPort, "", nil) + seed, coord, err := m0.RunWithTransport(gossipHost, gossipPort, "", pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -231,7 +231,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { m1 := test.NewMainWithCluster() defer m1.Close() - seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, seed, &coord) + seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, seed, coord) if err != nil { t.Fatal(err) } @@ -250,7 +250,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -261,7 +261,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) if err != nil { return err } @@ -284,7 +284,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) if err != nil { return err } @@ -330,7 +330,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -360,7 +360,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) if err != nil { return err } @@ -385,7 +385,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", nil) + seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -415,7 +415,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, &coord) + _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) if err != nil { return err } diff --git a/test/pilosa.go b/test/pilosa.go index a467a2f5c..b5d745fe0 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -85,7 +85,7 @@ func runMainWithCluster(size int) ([]*Main, error) { for i := 0; i < size; i++ { m := NewMainWithCluster() - gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, &coordinator) + gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, coordinator) if err != nil { return nil, errors.Wrap(err, "RunWithTransport") } @@ -132,7 +132,7 @@ func (m *Main) Reopen() error { } // RunWithTransport runs Main and returns the dynamically allocated gossip port. -func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) { +func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator pilosa.URI) (seed string, coord pilosa.URI, err error) { defer close(m.Started) /* @@ -185,12 +185,7 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coor return seed, coord, err } - if coordinator != nil { - coord = *coordinator - } else { - coord = m.Server.URI - } - m.Server.Cluster.Coordinator = coord + m.Server.Cluster.Coordinator = coordinator m.Server.Cluster.Static = false // Initialize server. @@ -199,7 +194,7 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coor return seed, coord, err } - return seed, coord, nil + return seed, m.Server.Cluster.Coordinator, nil } // URL returns the base URL string for accessing the running program. diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 0d0f168d5..c4842bf8b 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -10,7 +10,14 @@ import ( ) func TestNewCluster(t *testing.T) { - cluster := test.MustRunMainWithCluster(t, 3) + numNodes := 3 + cluster := test.MustRunMainWithCluster(t, numNodes) + coordinator := cluster[0].Server.Cluster.Coordinator + for i := 1; i < numNodes; i++ { + if coordi := cluster[i].Server.Cluster.Coordinator; coordi != coordinator { + t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator) + } + } response, err := http.Get("http://" + cluster[0].Server.Addr().String() + "/status") if err != nil { From b3532a617f4f5280d6db83d21b28ee5ff0bca0b3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Feb 2018 16:50:44 -0600 Subject: [PATCH 075/311] Fix error message (panics since err == nil) --- handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler.go b/handler.go index dd438e977..f865bb9ac 100644 --- a/handler.go +++ b/handler.go @@ -1966,7 +1966,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r newNode := h.Cluster.nodeByID(req.ID) if newNode == nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, "Node with provided ID does not exist", http.StatusBadRequest) return } From f799d98564198424a76db1ca221427583b9516c7 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 9 Feb 2018 17:33:07 +0300 Subject: [PATCH 076/311] Fixes #1092 --- Gopkg.lock | 8 +++++++- server.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index ed0469fc0..a8071c915 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -199,6 +199,12 @@ revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -274,6 +280,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "53ae0cdcbfa8419b233a0544d7ebb80614af3dd04bebdc0c8f2d3620a87efa91" + inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/server.go b/server.go index c8e536c50..83d417abd 100644 --- a/server.go +++ b/server.go @@ -15,6 +15,7 @@ package pilosa import ( + "context" "crypto/tls" "errors" "fmt" @@ -35,7 +36,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) From a2580915ba110238f9dcfe310a870d0791e8e857 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 9 Feb 2018 17:34:18 +0300 Subject: [PATCH 077/311] remove golang/org context from the lock --- Gopkg.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a8071c915..f33a9ec4f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -247,12 +247,6 @@ revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" version = "v1.0.0" -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = ["context"] - revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" - [[projects]] branch = "master" name = "golang.org/x/sync" From ac4010bce0951c60ffe1f648d830a9dfa0d0a6b9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 9 Feb 2018 11:35:13 -0600 Subject: [PATCH 078/311] avoid overflow bug in differenceRunArray which was appending a full run to the container --- roaring/roaring.go | 8 ++++++++ roaring/roaring_internal_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index a685e479c..efdce69a0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2504,6 +2504,8 @@ func differenceRunArray(a, b *container) *container { bidx := 0 vb := b.array[bidx] + +RUNLOOP: for _, run := range a.runs { start := run.start for vb < run.start { @@ -2515,6 +2517,9 @@ func differenceRunArray(a, b *container) *container { } for vb >= run.start && vb <= run.last { if vb == start { + if vb == 65535 { // overflow + break RUNLOOP + } start++ bidx++ if bidx >= len(b.array) { @@ -2525,6 +2530,9 @@ func differenceRunArray(a, b *container) *container { } output.runs = append(output.runs, interval16{start: start, last: vb - 1}) output.n += int(vb - start) + if vb == 65535 { // overflow + break RUNLOOP + } start = vb + 1 bidx++ if bidx >= len(b.array) { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index a2fbb7d64..f79f53053 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1479,6 +1479,16 @@ func TestDifferenceRunArray(t *testing.T) { array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19, 25, 27}, exp: []interval16{{start: 1, last: 8}, {start: 15, last: 16}, {start: 20, last: 24}, {start: 26, last: 26}, {start: 28, last: 28}}, }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65533, last: 65535}}, + array: []uint16{65533, 65534, 65535}, + exp: []interval16{{start: 0, last: 20}}, + }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65530, last: 65535}}, + array: []uint16{37, 65535}, + exp: []interval16{{start: 0, last: 20}, {start: 65530, last: 65534}}, + }, } for i, test := range tests { a.runs = test.runs From db7e88f6d55af32374e5b03d9cab44b0d6edfa7f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 9 Feb 2018 15:14:26 -0600 Subject: [PATCH 079/311] add request/response tags to the docs --- docs/api-reference.md | 116 +++++++++++++--------------------------- docs/getting-started.md | 112 ++++++++++++++++++++++++++++++++------ docs/query-language.md | 5 +- docs/tutorials.md | 113 +++++++++++++++++++++++--------------- 4 files changed, 205 insertions(+), 141 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 206c30f55..52ea2c369 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -13,13 +13,10 @@ nav = [] Returns the schema of all indexes in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index ``` - -Response: -``` +``` response {"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} ``` @@ -29,13 +26,10 @@ Response: Returns the schema of the specified index in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user ``` - -Response: -``` +``` response {"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} ``` @@ -45,13 +39,10 @@ Response: Creates an index with the given name. -Request: -``` +``` request curl -XPOST localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -61,13 +52,10 @@ Response: Removes the given index. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -77,15 +65,12 @@ Response: Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. -Request: -``` +``` request curl localhost:10101/index/user/query \ -X POST \ -d 'Bitmap(frame="language", rowID=5)' ``` - -Response: -``` +``` response {"results":[{"attrs":{},"bits":[100]}]} ``` @@ -95,14 +80,12 @@ The response doesn't include column attributes by default. To return them, set t The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. -Request: -``` +``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ -d 'Bitmap(frame="language", rowID=5)' ``` -Response: -``` +``` response { "results":[{"attrs":{},"bits":[100]}], "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] @@ -134,21 +117,21 @@ Each individual `field` contains the following: Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. -Request: -``` +``` request curl localhost:10101/index/user/frame/language \ -X POST \ -d '{"options": {"inverseEnabled": true}}' ``` - +``` response +{} ``` + +``` request curl localhost:10101/index/repository/frame/stats \ -X POST \ -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` - -Response: -``` +``` response {} ``` @@ -158,13 +141,10 @@ Response: Removes the given frame. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/frame/language ``` - -Response: -``` +``` response {} ``` @@ -188,15 +168,12 @@ The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Va * MDH: month, day and hour * YMDH: year, month, day and hour -Request: -``` +``` request curl localhost:10101/index/user/frame/language/time-quantum \ -X POST \ -d '{"timeQuantum": "YM"}' ``` - -Response: -``` +``` response {} ``` @@ -211,15 +188,12 @@ The request payload is JSON, and it must contain the fields `type`, `min`, `max` * `min` (int): Minimum value allowed for this field. * `max` (int): Maximum value allowed for this field. -Request: -``` +``` request curl localhost:10101/index/repository/frame/stats/field/pullrequests \ -X POST \ -d '{"type": "int", "min": 0, "max": 1000000}' ``` - -Response: -``` +``` response {} ``` @@ -251,8 +225,7 @@ The `action` describes how the field value will be processed. Each `action` may - `mapping`: Map the value to a RowID in the `valueMap`. * `valueMap` (object): string and integer pairs used to map field values to RowID's. -Request: -``` +``` request curl localhost:10101/index/user/input-definition/stargazer-input \ -X POST \ -d '{ @@ -284,9 +257,7 @@ curl localhost:10101/index/user/input-definition/stargazer-input \ ] }' ``` - -Response: -``` +``` response {} ``` @@ -296,13 +267,10 @@ Response: Returns the given input definition as JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user/input-definition/stargazer-input ``` - -Response: -``` +``` response {"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} ``` @@ -312,13 +280,10 @@ Response: Removes the given input definition. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input ``` - -Response: -``` +``` response {} ``` @@ -330,15 +295,12 @@ Processes the JSON payload using the given input definition. The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. -Request: -``` +``` request curl localhost:10101/index/user/input/stargazer-input \ -X POST \ -d '[{"language_id": "Go", "repo_id": 92274475}]' ``` - -Response: -``` +``` response {} ``` @@ -348,13 +310,10 @@ Response: Returns the hosts in the cluster. -Request: -``` +``` request curl -XGET localhost:10101/hosts ``` - -Response: -``` +``` response [{"host":":10101"}] ``` @@ -364,13 +323,10 @@ Response: Returns the version of the Pilosa server. -Request: -``` +``` request curl -XGET localhost:10101/version ``` - -Response: -``` +``` response {"version":"v0.6.0"} ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index 35042d59d..d4802e8ee 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,11 +31,12 @@ docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest ``` Let's make sure Pilosa is running: -``` +``` request curl localhost:10101/status ``` - -Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}` +``` response +{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}} +``` ### Sample Project @@ -45,30 +46,41 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term #### Create the Schema -Note: The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: -``` +``` request curl localhost:10101/schema +``` +``` response {"indexes":null} ``` Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: -``` +``` request curl localhost:10101/index/repository -X POST ``` +``` response +{} +``` Let's create the `stargazer` frame which has user IDs of stargazers as its rows: -``` +``` request curl localhost:10101/index/repository/frame/stargazer \ -X POST \ -d '{"options": {"timeQuantum": "YMD"}}' ``` +``` response +{} +``` Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` frame, which will contain IDs for programming languages: +``` request +curl localhost:10101/index/repository/frame/language \ + -X POST ``` -curl localhost:10101/index/repository/frame/language -X POST +``` response +{} ``` #### Import Data From CSV Files @@ -107,46 +119,112 @@ Alternatively Pilosa can import JSON data using an [Input Definition](../input-d Which repositories did user 14 star: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'Bitmap(frame="stargazer", rowID=14)' ``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] + } + ] +} +``` What are the top 5 languages in the sample data: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'TopN(frame="language", n=5)' ``` +``` response +{ + "results":[ + [ + {"id":5,"count":119}, + {"id":1,"count":50}, + {"id":4,"count":48}, + {"id":9,"count":31}, + {"id":13,"count":25} + ] + ] +} +``` Which repositories were starred by user 14 and 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Intersect( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,3,362,396,416,461,464,466,470,486] + } + ] +} ``` Which repositories were starred by user 14 or 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Union(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Union( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] + } + ] +} ``` Which repositories were starred by user 14 and 19 and also were written in language 1: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19), Bitmap(frame="language", rowID=1))' + -d 'Intersect( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19), + Bitmap(frame="language", rowID=1) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,362,416,461] + } + ] +} ``` Set user 99999 as a stargazer for repository 77777: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)' ``` +``` response +{"results":[true]} +``` ### What's Next? diff --git a/docs/query-language.md b/docs/query-language.md index f28257fe6..0e02351ab 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -34,11 +34,14 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data. The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", columnID=10, rowID=1)' ``` +``` response +{"results":[true]} +``` #### Arguments and Types diff --git a/docs/tutorials.md b/docs/tutorials.md index 9c19f3a67..4d787577c 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -178,38 +178,53 @@ curl -k --ipv4 https://01.pilosa.local:10501/status The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. All nodes should be in the `UP` state: -``` +``` response {"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} ``` #### Running Queries Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' ``` +``` response +{} +``` This will create index `sample-index` with default options. Let's create the frame now: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' ``` +``` response +{} +``` We just created frame `sample-frame` with default options. Let's run a `SetBit` query: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' ``` +``` response +{"results":[true]} +``` Confirm that the bit was indeed set: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' ``` +``` response +{"results":[{"attrs":{},"bits":[100]}]} +``` The same response should be returned when querying other nodes in the cluster: -``` +``` request curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' ``` +``` response +{"results":[{"attrs":{},"bits":[100]}]} +``` #### What's Next? @@ -223,20 +238,16 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: -``` +``` request curl localhost:10101/index/patients \ -X POST ``` - -Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient. -``` -curl localhost:10101/index/patients/frame/measurements \ - -X POST \ - -d '{"options":{"rangeEnabled": true}}' +``` response +{} ``` In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. -``` +``` request curl localhost:10101/index/patients/frame/measurements \ -X POST \ -d '{"options":{ @@ -248,17 +259,23 @@ curl localhost:10101/index/patients/frame/measurements \ ] }}' ``` +``` response +{} +``` If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' ``` +``` response +{"results":[null]} +``` In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. @@ -282,39 +299,38 @@ pilosa import -i patients -f measurements --field age ages.csv Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Range(frame="measurements", age > 40)' ``` -You should get the following results: -``` +``` response {"results":[{"attrs":{},"bits":[2,6,9]}]} ``` You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. To find the average age of all patients, run a `Sum` query: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(frame="measurements", field="age")' ``` -The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -``` +``` response {"results":[{"sum":377,"count":9}]} ``` +The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' ``` -Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. -``` +``` response {"results":[{"sum":191,"count":3}]} ``` +Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. ### Storing Row and Column Attributes @@ -323,20 +339,26 @@ Notice in this case that the count is only `3` because of the `age > 40` filter Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data. First, create an index called `books` to use for this tutorial: -``` +``` request curl localhost:10101/index/books \ -X POST ``` +``` response +{} +``` Next, create a frame in the `books` index called `members` which will represent library members who have read books. -``` +``` request curl localhost:10101/index/books/frame/members \ -X POST \ -d '{}' ``` +``` response +{} +``` Now, let's add some books to our index. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) @@ -345,9 +367,12 @@ curl localhost:10101/index/books/query \ SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' ``` +``` response +{"results":[null,null,null,null,null]} +``` And add some members. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") @@ -356,20 +381,22 @@ curl localhost:10101/index/books/query \ SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' ``` +``` response +{"results":[null,null,null,null,null]} +``` At this point we can query one of the `member` records by querying that row. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` -You should get the following result set: -``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} ``` Now let's add some data to the matrix such that each pair represents a member who has read that book. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetBit(frame="members", rowID=10001, columnID=3) @@ -390,28 +417,28 @@ curl localhost:10101/index/books/query \ SetBit(frame="members", rowID=10005, columnID=4) SetBit(frame="members", rowID=10005, columnID=5)' ``` +``` response +{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} +``` Now pull the record for `Sue Perkins` again. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` -Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. -``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} ``` +Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` - -Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute. - -``` +``` response { "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], "columnAttrs":[ @@ -421,15 +448,15 @@ Here, the `book` attributes will be included in the result set at the `columnAtt ] } ``` +The `book` attributes are included in the result set at the `columnAttrs` attribute. Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members: -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' ``` - -``` +``` response { "results":[{"attrs":{},"bits":[4]}], "columnAttrs":[ From c2d16f44e352c2151c326d6d2f864cac4bc92c33 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 12 Feb 2018 04:12:35 +0300 Subject: [PATCH 080/311] Spread recalculate caches to all nodes. Fixes #1069 --- broadcast.go | 5 + handler.go | 6 + internal/private.pb.go | 244 +++++++++++++++++++++++++++-------------- internal/private.proto | 1 + server.go | 2 + server/server_test.go | 48 ++++++++ test/pilosa.go | 8 ++ 7 files changed, 234 insertions(+), 80 deletions(-) diff --git a/broadcast.go b/broadcast.go index 883b62436..bb32ab0ba 100644 --- a/broadcast.go +++ b/broadcast.go @@ -137,6 +137,7 @@ const ( MessageTypeResizeInstructionComplete MessageTypeSetCoordinator MessageTypeNodeState + MessageTypeRecalculateCaches ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -171,6 +172,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeSetCoordinator case *internal.NodeStateMessage: typ = MessageTypeNodeState + case *internal.RecalculateCaches: + typ = MessageTypeRecalculateCaches default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -215,6 +218,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.SetCoordinatorMessage{} case MessageTypeNodeState: m = &internal.NodeStateMessage{} + case MessageTypeRecalculateCaches: + m = &internal.RecalculateCaches{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/handler.go b/handler.go index dd438e977..ff591a568 100644 --- a/handler.go +++ b/handler.go @@ -2151,6 +2151,12 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { + err := h.Broadcaster.SendSync(&internal.RecalculateCaches{}) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + return + } h.Holder.RecalculateCaches() w.WriteHeader(http.StatusNoContent) } diff --git a/internal/private.pb.go b/internal/private.pb.go index b584a10a9..3ea095687 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -42,6 +42,7 @@ ResizeInstructionComplete SetCoordinatorMessage Topology + RecalculateCaches */ package internal @@ -1068,6 +1069,14 @@ func (m *Topology) GetNodeIDs() []string { return nil } +type RecalculateCaches struct { +} + +func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } +func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } +func (*RecalculateCaches) ProtoMessage() {} +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{33} } + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -1102,6 +1111,7 @@ func init() { proto.RegisterType((*ResizeInstructionComplete)(nil), "internal.ResizeInstructionComplete") proto.RegisterType((*SetCoordinatorMessage)(nil), "internal.SetCoordinatorMessage") proto.RegisterType((*Topology)(nil), "internal.Topology") + proto.RegisterType((*RecalculateCaches)(nil), "internal.RecalculateCaches") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -2449,6 +2459,24 @@ func (m *Topology) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *RecalculateCaches) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -3070,6 +3098,12 @@ func (m *Topology) Size() (n int) { return n } +func (m *RecalculateCaches) Size() (n int) { + var l int + _ = l + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -7663,6 +7697,56 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } return nil } +func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RecalculateCaches: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RecalculateCaches: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -7771,84 +7855,84 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1250 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xd7, 0xbb, 0x76, 0xec, 0xe3, 0x38, 0x71, 0xe6, 0x4d, 0x83, 0x13, 0x45, 0xae, 0x19, - 0x21, 0x1a, 0x2a, 0x11, 0x15, 0x57, 0x02, 0x1a, 0x54, 0xa9, 0x24, 0x76, 0xd5, 0x05, 0x12, 0xca, - 0x38, 0x2d, 0x12, 0x17, 0x48, 0x13, 0x7b, 0x48, 0x57, 0x59, 0xef, 0x9a, 0xdd, 0x71, 0x12, 0xf7, - 0x82, 0x4b, 0x84, 0x84, 0xb8, 0x47, 0xdc, 0xf2, 0x67, 0xb8, 0xe4, 0x27, 0xa0, 0xf0, 0x23, 0x90, - 0xb8, 0x01, 0xcd, 0xd7, 0xee, 0xfa, 0x33, 0xa4, 0x70, 0xb7, 0xe7, 0x39, 0x1f, 0xf3, 0xcc, 0x39, - 0x67, 0xce, 0xcc, 0x42, 0x65, 0x10, 0x79, 0xe7, 0x94, 0xb3, 0xdd, 0x41, 0x14, 0xf2, 0x10, 0x15, - 0xbd, 0x80, 0xb3, 0x28, 0xa0, 0x3e, 0xfe, 0x14, 0x4a, 0x6e, 0xd0, 0x63, 0x97, 0x87, 0x8c, 0x53, - 0xd4, 0x80, 0xf2, 0x41, 0xe8, 0x0f, 0xfb, 0xc1, 0x27, 0xf4, 0x84, 0xf9, 0x35, 0xab, 0x61, 0xed, - 0x94, 0x48, 0x16, 0x12, 0x16, 0xc7, 0x5e, 0x9f, 0x7d, 0x36, 0xa4, 0x01, 0x1f, 0xf6, 0x6b, 0x39, - 0x65, 0x91, 0x81, 0xf0, 0x9f, 0x16, 0x94, 0x1e, 0x47, 0xb4, 0xcf, 0x64, 0xc4, 0x2d, 0x28, 0x92, - 0xf0, 0x22, 0x1b, 0x2e, 0x91, 0xd1, 0x9b, 0xb0, 0xe2, 0x06, 0xe7, 0x2c, 0x8a, 0x59, 0x3b, 0xa0, - 0x27, 0x3e, 0xeb, 0xc9, 0x70, 0x45, 0x32, 0x81, 0xa2, 0x6d, 0x28, 0x1d, 0xd0, 0xee, 0x0b, 0x76, - 0x3c, 0x1a, 0xb0, 0x9a, 0x2d, 0x83, 0xa4, 0x40, 0xa2, 0xed, 0x78, 0x2f, 0x59, 0xcd, 0x69, 0x58, - 0x3b, 0x15, 0x92, 0x02, 0x93, 0x7c, 0xf3, 0x53, 0x7c, 0x11, 0x86, 0x65, 0x42, 0x83, 0xd3, 0x84, - 0x43, 0x41, 0x72, 0x18, 0xc3, 0xd0, 0x1d, 0x28, 0x3c, 0xf6, 0x98, 0xdf, 0x8b, 0x6b, 0x4b, 0x0d, - 0x7b, 0xa7, 0xdc, 0x5c, 0xdd, 0x35, 0xf9, 0xdb, 0x95, 0x38, 0xd1, 0x6a, 0x8c, 0x61, 0xc5, 0xed, - 0x0f, 0xc2, 0x88, 0x13, 0x16, 0x0f, 0xc2, 0x20, 0x66, 0xa8, 0x0a, 0x76, 0x3b, 0x8a, 0xf4, 0xde, - 0xc5, 0x27, 0xfe, 0x06, 0xaa, 0xfb, 0x7e, 0xd8, 0x3d, 0x6b, 0x51, 0x4e, 0x09, 0xfb, 0x7a, 0xc8, - 0x62, 0x8e, 0xd6, 0x21, 0x2f, 0xab, 0xa0, 0xed, 0x94, 0x20, 0x50, 0x99, 0x49, 0x9d, 0x66, 0x25, - 0x08, 0x54, 0xfa, 0xcb, 0x54, 0x38, 0x44, 0x09, 0x02, 0xed, 0xf8, 0x5e, 0x57, 0xa5, 0xc0, 0x21, - 0x4a, 0x40, 0x08, 0x9c, 0xe7, 0x1e, 0xbb, 0xd0, 0xfb, 0x96, 0xdf, 0xd8, 0x85, 0xb5, 0xcc, 0xfa, - 0x9a, 0xe6, 0x06, 0x14, 0x48, 0x78, 0xe1, 0xb6, 0xe2, 0x9a, 0xd5, 0xb0, 0x77, 0x1c, 0xa2, 0x25, - 0x99, 0x5d, 0x59, 0x7e, 0xa1, 0xca, 0x49, 0x55, 0x0a, 0xe0, 0x4d, 0xc8, 0xcb, 0x54, 0x8b, 0x5d, - 0xa6, 0xbe, 0xe2, 0x13, 0xff, 0x65, 0x41, 0xe9, 0x90, 0x5e, 0x4a, 0x1a, 0x31, 0x7a, 0x08, 0xc5, - 0x0e, 0xa7, 0x41, 0x8f, 0x46, 0x3d, 0x69, 0x54, 0x6e, 0xbe, 0x9e, 0xa6, 0x30, 0x31, 0xdb, 0x35, - 0x36, 0xed, 0x80, 0x47, 0x23, 0x92, 0xb8, 0xa0, 0x3d, 0x58, 0xd2, 0x3d, 0x21, 0x39, 0x94, 0x9b, - 0x8d, 0x59, 0xde, 0x49, 0xdb, 0x08, 0x67, 0xe3, 0xb0, 0xf5, 0x01, 0x54, 0xc6, 0xc2, 0x0a, 0xae, - 0x67, 0x6c, 0x64, 0x2a, 0x72, 0xc6, 0x46, 0x22, 0x77, 0xe7, 0xd4, 0x1f, 0xaa, 0x3c, 0x3b, 0x44, - 0x09, 0x7b, 0xb9, 0xf7, 0xad, 0xad, 0x3d, 0x58, 0xce, 0x46, 0xbd, 0x89, 0x2f, 0xfe, 0x12, 0xd0, - 0x41, 0xc4, 0x28, 0x67, 0x92, 0xde, 0x21, 0x8b, 0x63, 0x7a, 0xca, 0xe6, 0x57, 0x5a, 0x55, 0x2f, - 0x97, 0xad, 0xde, 0x36, 0x94, 0xdc, 0xd8, 0x6c, 0xdc, 0x96, 0x7d, 0x99, 0x02, 0xf8, 0x2e, 0xa0, - 0x16, 0xf3, 0x19, 0x67, 0xfa, 0xfc, 0x2e, 0x88, 0x8f, 0x3b, 0x86, 0xcb, 0xf5, 0xb6, 0xe8, 0x0e, - 0x38, 0xe2, 0xe8, 0x4a, 0x2a, 0xe5, 0xe6, 0xff, 0xd3, 0x4c, 0x27, 0x73, 0x82, 0x48, 0x03, 0xec, - 0x99, 0xa0, 0xfa, 0xb8, 0x5f, 0xb3, 0xc1, 0x19, 0xad, 0x6c, 0x96, 0xb2, 0x27, 0x97, 0x4a, 0x06, - 0x88, 0x5e, 0xea, 0x91, 0xd9, 0xeb, 0xab, 0x2e, 0x85, 0xbf, 0xd0, 0xa8, 0x38, 0x12, 0x47, 0x42, - 0xab, 0x7c, 0xe4, 0xf7, 0xfc, 0x2d, 0x4f, 0xf0, 0x10, 0xb1, 0xc5, 0x19, 0x8a, 0x6b, 0x76, 0xc3, - 0x16, 0xb1, 0xa5, 0x80, 0xef, 0x43, 0xa1, 0xd3, 0x7d, 0xc1, 0xfa, 0x14, 0xbd, 0x25, 0x1a, 0xb5, - 0xc7, 0x2e, 0x59, 0xac, 0xdb, 0x7c, 0x75, 0x22, 0x7d, 0xc4, 0xe8, 0xf1, 0xf7, 0x96, 0x66, 0x3f, - 0x87, 0x51, 0x41, 0xae, 0x1d, 0xd7, 0x9c, 0xa9, 0x89, 0x23, 0x70, 0xa2, 0xd5, 0xa8, 0x0d, 0x55, - 0x37, 0x18, 0x0c, 0x79, 0x8b, 0x7d, 0xe5, 0x05, 0x1e, 0xf7, 0xc2, 0x20, 0xae, 0x15, 0xa4, 0xcb, - 0x66, 0x76, 0xe9, 0x31, 0x0b, 0x32, 0xe5, 0x82, 0xbf, 0xb5, 0x60, 0x75, 0x02, 0xbc, 0x86, 0x57, - 0x6e, 0x31, 0xaf, 0x77, 0x93, 0x91, 0x69, 0x4b, 0xc3, 0xfa, 0x5c, 0x36, 0xe3, 0x13, 0xf4, 0x67, - 0x0b, 0xd6, 0x67, 0x19, 0xcc, 0x64, 0x53, 0x07, 0x78, 0x1a, 0x79, 0x7d, 0x1a, 0x8d, 0x3e, 0x66, - 0x23, 0x7d, 0x7b, 0x64, 0x10, 0xf4, 0x39, 0x6c, 0x4c, 0xc4, 0xfa, 0xb0, 0xab, 0x52, 0xa4, 0x48, - 0xdd, 0x9e, 0x4b, 0x4a, 0xd9, 0x91, 0x39, 0xee, 0xf8, 0x0f, 0x0b, 0x6e, 0xcd, 0x54, 0xa5, 0xdd, - 0x67, 0x65, 0x1b, 0xfd, 0x2e, 0x54, 0x9f, 0x8b, 0xc1, 0xd0, 0x62, 0x31, 0xf7, 0x02, 0x2a, 0x2c, - 0x75, 0x7b, 0x4e, 0xe1, 0xc8, 0x85, 0xa2, 0xc4, 0x0e, 0xe9, 0x40, 0xd3, 0x7c, 0xfb, 0x1a, 0x9a, - 0xbb, 0xc6, 0x5e, 0xcf, 0x4d, 0x23, 0x0a, 0x32, 0x72, 0x8e, 0x9b, 0x4b, 0x41, 0x0a, 0x62, 0x22, - 0x8e, 0x39, 0xdc, 0x68, 0xaa, 0x85, 0xb0, 0x6d, 0x26, 0xc9, 0x18, 0x93, 0xc5, 0x67, 0xf2, 0x01, - 0x40, 0x6a, 0xaa, 0x8f, 0xfb, 0x82, 0xfe, 0xcc, 0x18, 0xe3, 0x27, 0xb0, 0x6d, 0xc6, 0xdc, 0x0d, - 0x16, 0x34, 0xdd, 0x92, 0x4b, 0xbb, 0x05, 0xb7, 0xc1, 0x7e, 0x46, 0x5c, 0x71, 0xd5, 0xc9, 0xd3, - 0x6a, 0x4a, 0xa4, 0x25, 0xe1, 0xf2, 0x24, 0x8c, 0xb9, 0x71, 0x11, 0xdf, 0x02, 0x7b, 0x1a, 0x46, - 0x5c, 0x32, 0xae, 0x10, 0xf9, 0x8d, 0xdf, 0x03, 0xe7, 0x28, 0xec, 0x31, 0xb4, 0x02, 0x39, 0xb7, - 0xa5, 0x63, 0xe4, 0xdc, 0x16, 0xba, 0x2d, 0xc3, 0xeb, 0x19, 0x52, 0x49, 0x37, 0xf7, 0x8c, 0xb8, - 0x44, 0x68, 0xf0, 0x23, 0xa8, 0x0a, 0xc7, 0x0e, 0xa7, 0x3c, 0x19, 0x61, 0x1b, 0x50, 0x10, 0x58, - 0x12, 0x48, 0x4b, 0xf2, 0x42, 0x10, 0x76, 0x66, 0x88, 0x49, 0x01, 0xff, 0x60, 0x01, 0x98, 0x10, - 0xc3, 0x18, 0x61, 0xc5, 0x44, 0xba, 0x96, 0x9b, 0x2b, 0xe9, 0x92, 0x02, 0x25, 0x8a, 0xe5, 0x3b, - 0x99, 0x6b, 0x78, 0x7a, 0xbe, 0x25, 0x2a, 0x92, 0xb9, 0xac, 0x77, 0xcc, 0x38, 0xd3, 0x85, 0xaa, - 0xa6, 0xf6, 0x0a, 0xd7, 0x29, 0x13, 0x37, 0x40, 0xe5, 0xc0, 0x1f, 0xc6, 0x9c, 0x45, 0x9a, 0x91, - 0x78, 0x2e, 0x28, 0x20, 0xd9, 0x51, 0x0a, 0xcc, 0xde, 0x14, 0x7a, 0x03, 0xf2, 0x82, 0xa9, 0x39, - 0x93, 0x93, 0xdb, 0x50, 0x4a, 0xdc, 0x81, 0xfc, 0xfc, 0x39, 0x80, 0xc0, 0x91, 0x8f, 0x43, 0x5d, - 0x3a, 0xf9, 0x2e, 0xac, 0x82, 0x7d, 0xe8, 0xa9, 0x5e, 0xb3, 0x89, 0xf8, 0x94, 0x08, 0xbd, 0x94, - 0x67, 0x41, 0x20, 0x54, 0x5c, 0x8b, 0x6b, 0xaa, 0x99, 0xc5, 0x1c, 0x7f, 0x95, 0x0b, 0xcc, 0xbc, - 0xaf, 0xec, 0xcc, 0xfb, 0xaa, 0x03, 0x6b, 0xaa, 0x61, 0xff, 0xcb, 0xa0, 0x3f, 0xe5, 0x60, 0x8d, - 0xb0, 0xd8, 0x7b, 0xc9, 0xdc, 0x20, 0xe6, 0xd1, 0x30, 0x19, 0x36, 0x1f, 0x85, 0x27, 0x3a, 0xd5, - 0x36, 0x51, 0x42, 0xd2, 0x16, 0xb9, 0x05, 0x6d, 0x71, 0x4f, 0xbc, 0xf4, 0xc3, 0xa8, 0x27, 0x86, - 0x4e, 0x18, 0xe9, 0x42, 0x4f, 0x9a, 0x66, 0x4d, 0xd0, 0x3d, 0x58, 0xea, 0x84, 0xc3, 0xa8, 0x9b, - 0x5c, 0x49, 0x1b, 0xa9, 0xb5, 0x62, 0xa6, 0xd4, 0xc4, 0x98, 0x65, 0xfa, 0x28, 0xbf, 0xb8, 0x8f, - 0xd0, 0xc3, 0x89, 0x3e, 0x92, 0x8f, 0xf0, 0x72, 0xf3, 0xb5, 0xd4, 0x61, 0x4c, 0x4d, 0xc6, 0xad, - 0xf1, 0x77, 0x16, 0x2c, 0x67, 0x29, 0xfc, 0xa3, 0x83, 0x91, 0x54, 0x24, 0x37, 0xb3, 0x22, 0xf6, - 0xac, 0x8a, 0x38, 0x69, 0x45, 0xd2, 0x27, 0x5b, 0x3e, 0xf3, 0x64, 0xc3, 0x67, 0xb0, 0x39, 0x55, - 0xa6, 0x83, 0xb0, 0x3f, 0x10, 0xfd, 0xf0, 0x2f, 0xca, 0xb5, 0x0e, 0xf9, 0x76, 0x14, 0xe9, 0x42, - 0x95, 0x88, 0x12, 0xf0, 0x03, 0xb8, 0xd5, 0x61, 0x3c, 0x53, 0x24, 0xd3, 0x6d, 0x0d, 0xb0, 0x8f, - 0xd8, 0xc5, 0x9c, 0xed, 0x0b, 0x15, 0xde, 0x87, 0xe2, 0x71, 0x38, 0x08, 0xfd, 0xf0, 0x74, 0x74, - 0xcd, 0xa1, 0xad, 0xc1, 0x92, 0x9a, 0x49, 0xea, 0xca, 0x2f, 0x11, 0x23, 0xee, 0x57, 0x7f, 0xb9, - 0xaa, 0x5b, 0xbf, 0x5e, 0xd5, 0xad, 0xdf, 0xae, 0xea, 0xd6, 0x8f, 0xbf, 0xd7, 0xff, 0x77, 0x52, - 0x90, 0x7f, 0x97, 0xf7, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x5d, 0xc5, 0x2e, 0x66, 0x6e, 0x0e, - 0x00, 0x00, + // 1263 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0xb6, 0x63, 0x3f, 0xd7, 0x89, 0x33, 0x4d, 0x83, 0x13, 0x45, 0xae, 0x19, 0x21, + 0x1a, 0x2a, 0x11, 0x15, 0x57, 0x02, 0x1a, 0x54, 0xa9, 0x24, 0x76, 0xd5, 0x05, 0x12, 0xca, 0x38, + 0x2d, 0x12, 0x07, 0xa4, 0x89, 0x3d, 0xa4, 0xab, 0xac, 0x77, 0xcd, 0xee, 0x6c, 0x12, 0xf7, 0xc0, + 0x11, 0x21, 0x21, 0xee, 0x88, 0x2b, 0x5f, 0x86, 0x23, 0x1f, 0x01, 0x85, 0x0f, 0x81, 0xc4, 0x05, + 0x34, 0xff, 0x76, 0xd7, 0x7f, 0x43, 0x0a, 0xb7, 0x7d, 0xbf, 0xf7, 0x67, 0x7e, 0xf3, 0xde, 0x9b, + 0x37, 0xb3, 0x50, 0x1d, 0x86, 0xee, 0x19, 0xe5, 0x6c, 0x67, 0x18, 0x06, 0x3c, 0x40, 0x25, 0xd7, + 0xe7, 0x2c, 0xf4, 0xa9, 0x87, 0x3f, 0x83, 0xb2, 0xe3, 0xf7, 0xd9, 0xc5, 0x01, 0xe3, 0x14, 0x35, + 0xa1, 0xb2, 0x1f, 0x78, 0xf1, 0xc0, 0xff, 0x94, 0x1e, 0x33, 0xaf, 0x6e, 0x35, 0xad, 0xed, 0x32, + 0xc9, 0x42, 0xc2, 0xe2, 0xc8, 0x1d, 0xb0, 0xcf, 0x63, 0xea, 0xf3, 0x78, 0x50, 0xcf, 0x29, 0x8b, + 0x0c, 0x84, 0xff, 0xb2, 0xa0, 0xfc, 0x38, 0xa4, 0x03, 0x26, 0x23, 0x6e, 0x42, 0x89, 0x04, 0xe7, + 0xd9, 0x70, 0x89, 0x8c, 0xde, 0x82, 0x65, 0xc7, 0x3f, 0x63, 0x61, 0xc4, 0x3a, 0x3e, 0x3d, 0xf6, + 0x58, 0x5f, 0x86, 0x2b, 0x91, 0x09, 0x14, 0x6d, 0x41, 0x79, 0x9f, 0xf6, 0x5e, 0xb0, 0xa3, 0xd1, + 0x90, 0xd5, 0x6d, 0x19, 0x24, 0x05, 0x12, 0x6d, 0xd7, 0x7d, 0xc9, 0xea, 0xf9, 0xa6, 0xb5, 0x5d, + 0x25, 0x29, 0x30, 0xc9, 0xb7, 0x30, 0xc5, 0x17, 0x61, 0xb8, 0x41, 0xa8, 0x7f, 0x92, 0x70, 0x28, + 0x4a, 0x0e, 0x63, 0x18, 0xba, 0x03, 0xc5, 0xc7, 0x2e, 0xf3, 0xfa, 0x51, 0x7d, 0xa9, 0x69, 0x6f, + 0x57, 0x5a, 0x2b, 0x3b, 0x26, 0x7f, 0x3b, 0x12, 0x27, 0x5a, 0x8d, 0x31, 0x2c, 0x3b, 0x83, 0x61, + 0x10, 0x72, 0xc2, 0xa2, 0x61, 0xe0, 0x47, 0x0c, 0xd5, 0xc0, 0xee, 0x84, 0xa1, 0xde, 0xbb, 0xf8, + 0xc4, 0xdf, 0x42, 0x6d, 0xcf, 0x0b, 0x7a, 0xa7, 0x6d, 0xca, 0x29, 0x61, 0xdf, 0xc4, 0x2c, 0xe2, + 0x68, 0x0d, 0x0a, 0xb2, 0x0a, 0xda, 0x4e, 0x09, 0x02, 0x95, 0x99, 0xd4, 0x69, 0x56, 0x82, 0x40, + 0xa5, 0xbf, 0x4c, 0x45, 0x9e, 0x28, 0x41, 0xa0, 0x5d, 0xcf, 0xed, 0xa9, 0x14, 0xe4, 0x89, 0x12, + 0x10, 0x82, 0xfc, 0x73, 0x97, 0x9d, 0xeb, 0x7d, 0xcb, 0x6f, 0xec, 0xc0, 0x6a, 0x66, 0x7d, 0x4d, + 0x73, 0x1d, 0x8a, 0x24, 0x38, 0x77, 0xda, 0x51, 0xdd, 0x6a, 0xda, 0xdb, 0x79, 0xa2, 0x25, 0x99, + 0x5d, 0x59, 0x7e, 0xa1, 0xca, 0x49, 0x55, 0x0a, 0xe0, 0x0d, 0x28, 0xc8, 0x54, 0x8b, 0x5d, 0xa6, + 0xbe, 0xe2, 0x13, 0xff, 0x6d, 0x41, 0xf9, 0x80, 0x5e, 0x48, 0x1a, 0x11, 0x7a, 0x08, 0xa5, 0x2e, + 0xa7, 0x7e, 0x9f, 0x86, 0x7d, 0x69, 0x54, 0x69, 0xbd, 0x91, 0xa6, 0x30, 0x31, 0xdb, 0x31, 0x36, + 0x1d, 0x9f, 0x87, 0x23, 0x92, 0xb8, 0xa0, 0x5d, 0x58, 0xd2, 0x3d, 0x21, 0x39, 0x54, 0x5a, 0xcd, + 0x59, 0xde, 0x49, 0xdb, 0x08, 0x67, 0xe3, 0xb0, 0xf9, 0x21, 0x54, 0xc7, 0xc2, 0x0a, 0xae, 0xa7, + 0x6c, 0x64, 0x2a, 0x72, 0xca, 0x46, 0x22, 0x77, 0x67, 0xd4, 0x8b, 0x55, 0x9e, 0xf3, 0x44, 0x09, + 0xbb, 0xb9, 0x0f, 0xac, 0xcd, 0x5d, 0xb8, 0x91, 0x8d, 0x7a, 0x1d, 0x5f, 0xfc, 0x15, 0xa0, 0xfd, + 0x90, 0x51, 0xce, 0x24, 0xbd, 0x03, 0x16, 0x45, 0xf4, 0x84, 0xcd, 0xaf, 0xb4, 0xaa, 0x5e, 0x2e, + 0x5b, 0xbd, 0x2d, 0x28, 0x3b, 0x91, 0xd9, 0xb8, 0x2d, 0xfb, 0x32, 0x05, 0xf0, 0x5d, 0x40, 0x6d, + 0xe6, 0x31, 0xce, 0xf4, 0xf9, 0x5d, 0x10, 0x1f, 0x77, 0x0d, 0x97, 0xab, 0x6d, 0xd1, 0x1d, 0xc8, + 0x8b, 0xa3, 0x2b, 0xa9, 0x54, 0x5a, 0x37, 0xd3, 0x4c, 0x27, 0x73, 0x82, 0x48, 0x03, 0xec, 0x9a, + 0xa0, 0xfa, 0xb8, 0x5f, 0xb1, 0xc1, 0x19, 0xad, 0x6c, 0x96, 0xb2, 0x27, 0x97, 0x4a, 0x06, 0x88, + 0x5e, 0xea, 0x91, 0xd9, 0xeb, 0xab, 0x2e, 0x85, 0xbf, 0xd4, 0xa8, 0x38, 0x12, 0x87, 0x42, 0xab, + 0x7c, 0xe4, 0xf7, 0xfc, 0x2d, 0x4f, 0xf0, 0x10, 0xb1, 0xc5, 0x19, 0x8a, 0xea, 0x76, 0xd3, 0x16, + 0xb1, 0xa5, 0x80, 0xef, 0x43, 0xb1, 0xdb, 0x7b, 0xc1, 0x06, 0x14, 0xbd, 0x2d, 0x1a, 0xb5, 0xcf, + 0x2e, 0x58, 0xa4, 0xdb, 0x7c, 0x65, 0x22, 0x7d, 0xc4, 0xe8, 0xf1, 0x0f, 0x96, 0x66, 0x3f, 0x87, + 0x51, 0x51, 0xae, 0x1d, 0xd5, 0xf3, 0x53, 0x13, 0x47, 0xe0, 0x44, 0xab, 0x51, 0x07, 0x6a, 0x8e, + 0x3f, 0x8c, 0x79, 0x9b, 0x7d, 0xed, 0xfa, 0x2e, 0x77, 0x03, 0x3f, 0xaa, 0x17, 0xa5, 0xcb, 0x46, + 0x76, 0xe9, 0x31, 0x0b, 0x32, 0xe5, 0x82, 0xbf, 0xb3, 0x60, 0x65, 0x02, 0xbc, 0x82, 0x57, 0x6e, + 0x31, 0xaf, 0xf7, 0x92, 0x91, 0x69, 0x4b, 0xc3, 0xc6, 0x5c, 0x36, 0xe3, 0x13, 0xf4, 0x17, 0x0b, + 0xd6, 0x66, 0x19, 0xcc, 0x64, 0xd3, 0x00, 0x78, 0x1a, 0xba, 0x03, 0x1a, 0x8e, 0x3e, 0x61, 0x23, + 0x7d, 0x7b, 0x64, 0x10, 0xf4, 0x05, 0xac, 0x4f, 0xc4, 0xfa, 0xa8, 0xa7, 0x52, 0xa4, 0x48, 0xdd, + 0x9e, 0x4b, 0x4a, 0xd9, 0x91, 0x39, 0xee, 0xf8, 0x4f, 0x0b, 0x6e, 0xcd, 0x54, 0xa5, 0xdd, 0x67, + 0x65, 0x1b, 0xfd, 0x2e, 0xd4, 0x9e, 0x8b, 0xc1, 0xd0, 0x66, 0x11, 0x77, 0x7d, 0x2a, 0x2c, 0x75, + 0x7b, 0x4e, 0xe1, 0xc8, 0x81, 0x92, 0xc4, 0x0e, 0xe8, 0x50, 0xd3, 0x7c, 0xe7, 0x0a, 0x9a, 0x3b, + 0xc6, 0x5e, 0xcf, 0x4d, 0x23, 0x0a, 0x32, 0x72, 0x8e, 0x9b, 0x4b, 0x41, 0x0a, 0x62, 0x22, 0x8e, + 0x39, 0x5c, 0x6b, 0xaa, 0x05, 0xb0, 0x65, 0x26, 0xc9, 0x18, 0x93, 0xc5, 0x67, 0xf2, 0x01, 0x40, + 0x6a, 0xaa, 0x8f, 0xfb, 0x82, 0xfe, 0xcc, 0x18, 0xe3, 0x27, 0xb0, 0x65, 0xc6, 0xdc, 0x35, 0x16, + 0x34, 0xdd, 0x92, 0x4b, 0xbb, 0x05, 0x77, 0xc0, 0x7e, 0x46, 0x1c, 0x71, 0xd5, 0xc9, 0xd3, 0x6a, + 0x4a, 0xa4, 0x25, 0xe1, 0xf2, 0x24, 0x88, 0xb8, 0x71, 0x11, 0xdf, 0x02, 0x7b, 0x1a, 0x84, 0x5c, + 0x32, 0xae, 0x12, 0xf9, 0x8d, 0xdf, 0x87, 0xfc, 0x61, 0xd0, 0x67, 0x68, 0x19, 0x72, 0x4e, 0x5b, + 0xc7, 0xc8, 0x39, 0x6d, 0x74, 0x5b, 0x86, 0xd7, 0x33, 0xa4, 0x9a, 0x6e, 0xee, 0x19, 0x71, 0x88, + 0xd0, 0xe0, 0x47, 0x50, 0x13, 0x8e, 0x5d, 0x4e, 0x79, 0x32, 0xc2, 0xd6, 0xa1, 0x28, 0xb0, 0x24, + 0x90, 0x96, 0xe4, 0x85, 0x20, 0xec, 0xcc, 0x10, 0x93, 0x02, 0xfe, 0xd1, 0x02, 0x30, 0x21, 0xe2, + 0x08, 0x61, 0xc5, 0x44, 0xba, 0x56, 0x5a, 0xcb, 0xe9, 0x92, 0x02, 0x25, 0x8a, 0xe5, 0xbb, 0x99, + 0x6b, 0x78, 0x7a, 0xbe, 0x25, 0x2a, 0x92, 0xb9, 0xac, 0xb7, 0xcd, 0x38, 0xd3, 0x85, 0xaa, 0xa5, + 0xf6, 0x0a, 0xd7, 0x29, 0x13, 0x37, 0x40, 0x75, 0xdf, 0x8b, 0x23, 0xce, 0x42, 0xcd, 0x48, 0x3c, + 0x17, 0x14, 0x90, 0xec, 0x28, 0x05, 0x66, 0x6f, 0x0a, 0xbd, 0x09, 0x05, 0xc1, 0xd4, 0x9c, 0xc9, + 0xc9, 0x6d, 0x28, 0x25, 0xee, 0x42, 0x61, 0xfe, 0x1c, 0x40, 0x90, 0x97, 0x8f, 0x43, 0x5d, 0x3a, + 0xf9, 0x2e, 0xac, 0x81, 0x7d, 0xe0, 0xaa, 0x5e, 0xb3, 0x89, 0xf8, 0x94, 0x08, 0xbd, 0x90, 0x67, + 0x41, 0x20, 0x54, 0x5c, 0x8b, 0xab, 0xaa, 0x99, 0xc5, 0x1c, 0x7f, 0x95, 0x0b, 0xcc, 0xbc, 0xaf, + 0xec, 0xcc, 0xfb, 0xaa, 0x0b, 0xab, 0xaa, 0x61, 0xff, 0xcf, 0xa0, 0x3f, 0xe7, 0x60, 0x95, 0xb0, + 0xc8, 0x7d, 0xc9, 0x1c, 0x3f, 0xe2, 0x61, 0x9c, 0x0c, 0x9b, 0x8f, 0x83, 0x63, 0x9d, 0x6a, 0x9b, + 0x28, 0x21, 0x69, 0x8b, 0xdc, 0x82, 0xb6, 0xb8, 0x27, 0x5e, 0xfa, 0x41, 0xd8, 0x17, 0x43, 0x27, + 0x08, 0x75, 0xa1, 0x27, 0x4d, 0xb3, 0x26, 0xe8, 0x1e, 0x2c, 0x75, 0x83, 0x38, 0xec, 0x25, 0x57, + 0xd2, 0x7a, 0x6a, 0xad, 0x98, 0x29, 0x35, 0x31, 0x66, 0x99, 0x3e, 0x2a, 0x2c, 0xee, 0x23, 0xf4, + 0x70, 0xa2, 0x8f, 0xe4, 0x23, 0xbc, 0xd2, 0x7a, 0x3d, 0x75, 0x18, 0x53, 0x93, 0x71, 0x6b, 0xfc, + 0xbd, 0x05, 0x37, 0xb2, 0x14, 0xfe, 0xd5, 0xc1, 0x48, 0x2a, 0x92, 0x9b, 0x59, 0x11, 0x7b, 0x56, + 0x45, 0xf2, 0x69, 0x45, 0xd2, 0x27, 0x5b, 0x21, 0xf3, 0x64, 0xc3, 0xa7, 0xb0, 0x31, 0x55, 0xa6, + 0xfd, 0x60, 0x30, 0x14, 0xfd, 0xf0, 0x1f, 0xca, 0xb5, 0x06, 0x85, 0x4e, 0x18, 0xea, 0x42, 0x95, + 0x89, 0x12, 0xf0, 0x03, 0xb8, 0xd5, 0x65, 0x3c, 0x53, 0x24, 0xd3, 0x6d, 0x4d, 0xb0, 0x0f, 0xd9, + 0xf9, 0x9c, 0xed, 0x0b, 0x15, 0xde, 0x83, 0xd2, 0x51, 0x30, 0x0c, 0xbc, 0xe0, 0x64, 0x74, 0xc5, + 0xa1, 0xad, 0xc3, 0x92, 0x9a, 0x49, 0xea, 0xca, 0x2f, 0x13, 0x23, 0xe2, 0x9b, 0xa2, 0x25, 0x7b, + 0xd4, 0xeb, 0xc5, 0x1e, 0xe5, 0x4c, 0xfe, 0x08, 0x44, 0x7b, 0xb5, 0x5f, 0x2f, 0x1b, 0xd6, 0x6f, + 0x97, 0x0d, 0xeb, 0xf7, 0xcb, 0x86, 0xf5, 0xd3, 0x1f, 0x8d, 0xd7, 0x8e, 0x8b, 0xf2, 0x97, 0xf3, + 0xfe, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4c, 0x2d, 0x85, 0x72, 0x83, 0x0e, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 4e3261079..ccf2b1c74 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -194,3 +194,4 @@ message Topology { repeated string NodeIDs = 2; } +message RecalculateCaches {} diff --git a/server.go b/server.go index 63c14a88e..a3143cbf3 100644 --- a/server.go +++ b/server.go @@ -448,6 +448,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.RecalculateCaches: + s.Holder.RecalculateCaches() } return nil diff --git a/server/server_test.go b/server/server_test.go index c821ccdf8..3852845a6 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -23,6 +23,7 @@ import ( "reflect" "runtime" "sort" + "strings" "testing" "testing/quick" @@ -383,6 +384,47 @@ func TestCountOpenFiles(t *testing.T) { } } +func TestMain_RecalculateHashes(t *testing.T) { + const clusterSize = 5 + cluster := test.MustRunMainWithCluster(t, clusterSize) + + // Create the schema. + client0 := cluster[0].Client() + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal("create index:", err) + } + if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil { + t.Fatal("create frame:", err) + } + + // Set some bits + data := []string{} + for rowID := 1; rowID < 10; rowID++ { + for columnID := 1; columnID < 100; columnID++ { + data = append(data, fmt.Sprintf(`SetBit(rowID=%d, frame="f", columnID=%d)`, rowID, columnID)) + } + } + if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { + t.Fatal("setting bits:", err) + } + + // Calculate caches on the first node + cluster[0].RecalculateCaches() + target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}` + + // Run a TopN query on all nodes. The result should be the same as the target. + for _, m := range cluster { + res, err := m.Query("i", "", `TopN(frame="f")`) + if err != nil { + t.Fatal(err) + } + res = strings.TrimSpace(res) + if sortedString(target) != sortedString(res) { + t.Fatalf("%v != %v", target, res) + } + } +} + // SetCommand represents a command to set a bit. type SetCommand struct { ID uint64 @@ -448,6 +490,12 @@ func MustMarshalJSON(v interface{}) string { return string(buf) } +func sortedString(s string) string { + arr := strings.Split(s, "") + sort.Strings(arr) + return strings.Join(arr, "") +} + // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/test/pilosa.go b/test/pilosa.go index b5d745fe0..7083b07b7 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -227,6 +227,14 @@ func (m *Main) CreateDefinition(index, def, query string) (string, error) { return resp.Body, nil } +func (m *Main) RecalculateCaches() error { + resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "") + if resp.StatusCode != 204 { + return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return nil +} + //////////////////////////////////////////////////////////////////////////////////// // MustDo executes http.Do() with an http.NewRequest(). Panic on error. From e2c2bd11e45408cea366dface8d0489473b6a205 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 12 Feb 2018 17:16:38 +0300 Subject: [PATCH 081/311] put back golang.org/x/net dep, required by golang.org/x/sync/errgroup --- Gopkg.lock | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Gopkg.lock b/Gopkg.lock index f33a9ec4f..a8071c915 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -247,6 +247,12 @@ revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" version = "v1.0.0" +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context"] + revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" + [[projects]] branch = "master" name = "golang.org/x/sync" From 657ff05e88167b4be628a93d91883e780f470cb4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 12 Feb 2018 17:28:49 +0300 Subject: [PATCH 082/311] fix format strings which fail on go/master --- handler_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/handler_test.go b/handler_test.go index 04903c979..585230b15 100644 --- a/handler_test.go +++ b/handler_test.go @@ -357,7 +357,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } @@ -445,7 +445,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { @@ -506,7 +506,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { @@ -579,7 +579,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } From 944a0b5b3d511923d16615453784e30a37aa7d47 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Sun, 11 Feb 2018 21:21:48 -0600 Subject: [PATCH 083/311] fixes a shift logic bug in bitmapZeroRange that was causing an overflow-like condition --- roaring/roaring.go | 2 +- roaring/roaring_internal_test.go | 86 ++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index efdce69a0..b7cdec1c9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2337,7 +2337,7 @@ func (c *container) bitmapZeroRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { c.n -= int(popcnt(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index f79f53053..28b398c96 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1593,14 +1593,69 @@ func TestDifferenceBitmapRun(t *testing.T) { runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}}, exp: []uint64{0xFFFF0000FFFFFF0F}, }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}, {start: 7, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}}, + exp: []uint64{0xFFFFFFFFFFFFFF80}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 63}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}, {start: 67, last: 72}, {start: 126, last: 130}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF, 0x3FFFFFFFFFFFFE04, 0xFFFFFFFFFFFFFFF8}, + }, + { + bitmap: []uint64{0x0000000000000001}, + runs: []interval16{{start: 0, last: 0}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0x8000000000000000}, + runs: []interval16{{start: 63, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xC000000000000000, 0x0000000000000003}, + runs: []interval16{{start: 63, last: 64}}, + exp: []uint64{0x4000000000000000, 0x0000000000000002}, + }, + { + bitmap: []uint64{0x0000000000000000}, + runs: []interval16{{start: 5, last: 7}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: bitmapLast(), + runs: []interval16{{start: 65535, last: 65535}}, + exp: bitmapEmpty(), + }, + { + bitmap: bitmapFull(), + runs: []interval16{{start: 0, last: 65535}}, + exp: bitmapEmpty(), + }, } for i, test := range tests { for i, v := range test.bitmap { a.bitmap[i] = v } - a.n = a.bitmapCountRange(0, 100) + a.n = a.bitmapCountRange(0, 65536) b.runs = test.runs - b.n = b.runCountRange(0, 100) + b.n = b.runCountRange(0, 65536) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) @@ -2438,7 +2493,7 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { } } -func TestSearc64(t *testing.T) { +func TestSearch64(t *testing.T) { tests := []struct { a []uint64 value uint64 @@ -2596,6 +2651,31 @@ func bitmapEvens() []uint64 { return bitmap } +func bitmapLast() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN-1; i++ { + bitmap[i] = 0 + } + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapFull() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapEmpty() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0 + } + return bitmap +} + var containerWidth uint64 = 65536 // rleCont returns a slice of numbers all in the range starting from From 6529866b5759e294de1634f8963eb1d252d7decc Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 14 Feb 2018 13:30:07 -0600 Subject: [PATCH 084/311] templates for testing various container operations --- roaring/roaring_helpers_test.go | 256 ++++++++++++++++++++ roaring/roaring_internal_test.go | 390 +++++++++++++++++++++++++++++-- 2 files changed, 628 insertions(+), 18 deletions(-) create mode 100644 roaring/roaring_helpers_test.go diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go new file mode 100644 index 000000000..de3dd002c --- /dev/null +++ b/roaring/roaring_helpers_test.go @@ -0,0 +1,256 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +/////////////////////////////////////////////////////////////////////////// + +var containerWidth uint64 = 65536 + +////////////////// array +func arrayEmpty() []uint16 { + return make([]uint16, 0) +} + +func arrayFull() []uint16 { + array := make([]uint16, containerWidth) + for i := 0; i < int(containerWidth); i++ { + array[i] = uint16(i) + } + return array +} + +func arrayFirstBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(0)) + return array +} + +func arrayLastBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(65535)) + return array +} + +func arrayFirstBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 1; i < int(containerWidth); i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayLastBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 0; i < int(containerWidth)-1; i++ { + array[i] = uint16(i) + } + return array +} + +func arrayInnerBitsSet() []uint16 { + array := make([]uint16, containerWidth-2) + for i := 1; i < int(containerWidth)-1; i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayOuterBitsSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(0)) + array = append(array, uint16(65535)) + return array +} + +////////////////// bitmap +func bitmapEmpty() []uint64 { + return make([]uint64, bitmapN) +} + +func bitmapFull() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapFirstBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000001 + return bitmap +} + +func bitmapLastBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapFirstBitUnset() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0xFFFFFFFFFFFFFFFE + for i := 1; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapLastBitUnset() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN-1; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapInnerBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0xFFFFFFFFFFFFFFFE + for i := 1; i < bitmapN-1; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapOuterBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000001 + for i := 1; i < bitmapN-1; i++ { + bitmap[i] = 0x0000000000000000 + } + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +////////////////// run +func runEmpty() []interval16 { + return make([]interval16, 0) +} + +func runFull() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65535}) + return run +} + +func runFirstBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + return run +} + +func runLastBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +func runFirstBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65535}) + return run +} + +func runLastBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65534}) + return run +} + +func runInnerBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65534}) + return run +} + +func runOuterBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +/////////////////////////////////////////////////////////////////////////// + +type testOp struct { + f func(a, b *container) *container + x string + y string + exp string +} + +func doContainer(containerType byte, data interface{}) *container { + c := &container{ + containerType: containerType, + } + + switch containerType { + case ContainerArray: + c.array = data.([]uint16) + case ContainerBitmap: + c.bitmap = data.([]uint64) + case ContainerRun: + c.runs = data.([]interval16) + } + c.n = c.count() + + return c +} + +func setupContainerTests() map[byte]map[string]*container { + + cts := make(map[byte]map[string]*container) + + // array containers + cts[ContainerArray] = map[string]*container{ + "empty": doContainer(ContainerArray, arrayEmpty()), + "full": doContainer(ContainerArray, arrayFull()), + "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(ContainerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), + } + + // bitmap containers + cts[ContainerBitmap] = map[string]*container{ + "empty": doContainer(ContainerBitmap, bitmapEmpty()), + "full": doContainer(ContainerBitmap, bitmapFull()), + "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), + } + + // run containers + cts[ContainerRun] = map[string]*container{ + "empty": doContainer(ContainerRun, runEmpty()), + "full": doContainer(ContainerRun, runFull()), + "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), + "lastBitSet": doContainer(ContainerRun, runLastBitSet()), + "firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), + "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), + } + + return cts +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 28b398c96..ce9ab217d 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -18,6 +18,8 @@ import ( "bytes" "fmt" "reflect" + "runtime" + "strings" "testing" ) @@ -2660,24 +2662,6 @@ func bitmapLast() []uint64 { return bitmap } -func bitmapFull() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } - return bitmap -} - -func bitmapEmpty() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0 - } - return bitmap -} - -var containerWidth uint64 = 65536 - // rleCont returns a slice of numbers all in the range starting from // container_width*num, and ending at container_width*(num+1)-1. If left is // true, then the first 100 bits will be set, if mid is true, 100 bits in the @@ -2759,3 +2743,373 @@ func bitmapVariousContainers() *Bitmap { bm.Optimize() return bm } + +/////////////////////////////////////////////////////////////////////////// + +func getFunctionName(i interface{}) string { + x := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() + y := strings.Split(x, ".") + y = y[len(y)-1:] + return y[0] +} + +func TestContainerCombinations(t *testing.T) { + + cts := setupContainerTests() + + containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun} + + // map used for a more descriptive print + cm := map[byte]string{ + ContainerArray: "array", + ContainerBitmap: "bitmap", + ContainerRun: "run", + } + + testOps := []testOp{ + // intersect + {intersect, "empty", "empty", "empty"}, + {intersect, "empty", "full", "empty"}, + {intersect, "empty", "firstBitSet", "empty"}, + {intersect, "empty", "lastBitSet", "empty"}, + {intersect, "empty", "firstBitUnset", "empty"}, + {intersect, "empty", "lastBitUnset", "empty"}, + {intersect, "empty", "innerBitsSet", "empty"}, + {intersect, "empty", "outerBitsSet", "empty"}, + // + {intersect, "full", "empty", "empty"}, + {intersect, "full", "full", "full"}, + {intersect, "full", "firstBitSet", "firstBitSet"}, + {intersect, "full", "lastBitSet", "lastBitSet"}, + {intersect, "full", "firstBitUnset", "firstBitUnset"}, + {intersect, "full", "lastBitUnset", "lastBitUnset"}, + {intersect, "full", "innerBitsSet", "innerBitsSet"}, + {intersect, "full", "outerBitsSet", "outerBitsSet"}, + // + {intersect, "firstBitSet", "empty", "empty"}, + {intersect, "firstBitSet", "full", "firstBitSet"}, + {intersect, "firstBitSet", "firstBitSet", "firstBitSet"}, + {intersect, "firstBitSet", "lastBitSet", "empty"}, + {intersect, "firstBitSet", "firstBitUnset", "empty"}, + {intersect, "firstBitSet", "lastBitUnset", "firstBitSet"}, + {intersect, "firstBitSet", "innerBitsSet", "empty"}, + {intersect, "firstBitSet", "outerBitsSet", "firstBitSet"}, + // + {intersect, "lastBitSet", "empty", "empty"}, + {intersect, "lastBitSet", "full", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitSet", "empty"}, + {intersect, "lastBitSet", "lastBitSet", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitUnset", "lastBitSet"}, + {intersect, "lastBitSet", "lastBitUnset", "empty"}, + {intersect, "lastBitSet", "innerBitsSet", "empty"}, + {intersect, "lastBitSet", "outerBitsSet", "lastBitSet"}, + // + {intersect, "firstBitUnset", "empty", "empty"}, + {intersect, "firstBitUnset", "full", "firstBitUnset"}, + {intersect, "firstBitUnset", "firstBitSet", "empty"}, + {intersect, "firstBitUnset", "lastBitSet", "lastBitSet"}, + {intersect, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"}, + {intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"}, + // + {intersect, "lastBitUnset", "empty", "empty"}, + {intersect, "lastBitUnset", "full", "lastBitUnset"}, + {intersect, "lastBitUnset", "firstBitSet", "firstBitSet"}, + {intersect, "lastBitUnset", "lastBitSet", "empty"}, + {intersect, "lastBitUnset", "firstBitUnset", "innerBitsSet"}, + {intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"}, + // + {intersect, "innerBitsSet", "empty", "empty"}, + {intersect, "innerBitsSet", "full", "innerBitsSet"}, + {intersect, "innerBitsSet", "firstBitSet", "empty"}, + {intersect, "innerBitsSet", "lastBitSet", "empty"}, + {intersect, "innerBitsSet", "firstBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {intersect, "innerBitsSet", "outerBitsSet", "empty"}, + // + {intersect, "outerBitsSet", "empty", "empty"}, + {intersect, "outerBitsSet", "full", "outerBitsSet"}, + {intersect, "outerBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "outerBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "outerBitsSet", "firstBitUnset", "lastBitSet"}, + {intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"}, + {intersect, "outerBitsSet", "innerBitsSet", "empty"}, + {intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + + // union + {union, "empty", "empty", "empty"}, + {union, "empty", "full", "full"}, + {union, "empty", "firstBitSet", "firstBitSet"}, + {union, "empty", "lastBitSet", "lastBitSet"}, + {union, "empty", "firstBitUnset", "firstBitUnset"}, + {union, "empty", "lastBitUnset", "lastBitUnset"}, + {union, "empty", "innerBitsSet", "innerBitsSet"}, + {union, "empty", "outerBitsSet", "outerBitsSet"}, + // + {union, "full", "empty", "full"}, + {union, "full", "full", "full"}, + {union, "full", "firstBitSet", "full"}, + {union, "full", "lastBitSet", "full"}, + {union, "full", "firstBitUnset", "full"}, + {union, "full", "lastBitUnset", "full"}, + {union, "full", "innerBitsSet", "full"}, + {union, "full", "outerBitsSet", "full"}, + // + {union, "firstBitSet", "empty", "firstBitSet"}, + {union, "firstBitSet", "full", "full"}, + {union, "firstBitSet", "firstBitSet", "firstBitSet"}, + {union, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {union, "firstBitSet", "firstBitUnset", "full"}, + {union, "firstBitSet", "lastBitUnset", "lastBitUnset"}, + {union, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {union, "firstBitSet", "outerBitsSet", "outerBitsSet"}, + // + {union, "lastBitSet", "empty", "lastBitSet"}, + {union, "lastBitSet", "full", "full"}, + {union, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {union, "lastBitSet", "lastBitSet", "lastBitSet"}, + {union, "lastBitSet", "firstBitUnset", "firstBitUnset"}, + {union, "lastBitSet", "lastBitUnset", "full"}, + {union, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {union, "lastBitSet", "outerBitsSet", "outerBitsSet"}, + // + {union, "firstBitUnset", "empty", "firstBitUnset"}, + {union, "firstBitUnset", "full", "full"}, + {union, "firstBitUnset", "firstBitSet", "full"}, + {union, "firstBitUnset", "lastBitSet", "firstBitUnset"}, + {union, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {union, "firstBitUnset", "lastBitUnset", "full"}, + {union, "firstBitUnset", "innerBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "outerBitsSet", "full"}, + // + {union, "lastBitUnset", "empty", "lastBitUnset"}, + {union, "lastBitUnset", "full", "full"}, + {union, "lastBitUnset", "firstBitSet", "lastBitUnset"}, + {union, "lastBitUnset", "lastBitSet", "full"}, + {union, "lastBitUnset", "firstBitUnset", "full"}, + {union, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {union, "lastBitUnset", "innerBitsSet", "lastBitUnset"}, + {union, "lastBitUnset", "outerBitsSet", "full"}, + // + {union, "innerBitsSet", "empty", "innerBitsSet"}, + {union, "innerBitsSet", "full", "full"}, + {union, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {union, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {union, "innerBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "innerBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {union, "innerBitsSet", "outerBitsSet", "full"}, + // + {union, "outerBitsSet", "empty", "outerBitsSet"}, + {union, "outerBitsSet", "full", "full"}, + {union, "outerBitsSet", "firstBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "lastBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "firstBitUnset", "full"}, + {union, "outerBitsSet", "lastBitUnset", "full"}, + {union, "outerBitsSet", "innerBitsSet", "full"}, + {union, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + + // difference + {difference, "empty", "empty", "empty"}, + {difference, "empty", "full", "empty"}, + {difference, "empty", "firstBitSet", "empty"}, + {difference, "empty", "lastBitSet", "empty"}, + {difference, "empty", "firstBitUnset", "empty"}, + {difference, "empty", "lastBitUnset", "empty"}, + {difference, "empty", "innerBitsSet", "empty"}, + {difference, "empty", "outerBitsSet", "empty"}, + // + {difference, "full", "empty", "full"}, + {difference, "full", "full", "empty"}, + {difference, "full", "firstBitSet", "firstBitUnset"}, + {difference, "full", "lastBitSet", "lastBitUnset"}, + {difference, "full", "firstBitUnset", "firstBitSet"}, + {difference, "full", "lastBitUnset", "lastBitSet"}, + {difference, "full", "innerBitsSet", "outerBitsSet"}, + {difference, "full", "outerBitsSet", "innerBitsSet"}, + // + {difference, "firstBitSet", "empty", "firstBitSet"}, + {difference, "firstBitSet", "full", "empty"}, + {difference, "firstBitSet", "firstBitSet", "empty"}, + {difference, "firstBitSet", "lastBitSet", "firstBitSet"}, + {difference, "firstBitSet", "firstBitUnset", "firstBitSet"}, + {difference, "firstBitSet", "lastBitUnset", "empty"}, + {difference, "firstBitSet", "innerBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "outerBitsSet", "empty"}, + // + {difference, "lastBitSet", "empty", "lastBitSet"}, + {difference, "lastBitSet", "full", "empty"}, + {difference, "lastBitSet", "firstBitSet", "lastBitSet"}, + {difference, "lastBitSet", "lastBitSet", "empty"}, + {difference, "lastBitSet", "firstBitUnset", "empty"}, + {difference, "lastBitSet", "lastBitUnset", "lastBitSet"}, + {difference, "lastBitSet", "innerBitsSet", "lastBitSet"}, + {difference, "lastBitSet", "outerBitsSet", "empty"}, + // + {difference, "firstBitUnset", "empty", "firstBitUnset"}, + {difference, "firstBitUnset", "full", "empty"}, + {difference, "firstBitUnset", "firstBitSet", "firstBitUnset"}, + {difference, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {difference, "firstBitUnset", "firstBitUnset", "empty"}, + {difference, "firstBitUnset", "lastBitUnset", "lastBitSet"}, + {difference, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + // + {difference, "lastBitUnset", "empty", "lastBitUnset"}, + {difference, "lastBitUnset", "full", "empty"}, + {difference, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {difference, "lastBitUnset", "lastBitSet", "lastBitUnset"}, + {difference, "lastBitUnset", "firstBitUnset", "firstBitSet"}, + {difference, "lastBitUnset", "lastBitUnset", "empty"}, + {difference, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + // + {difference, "innerBitsSet", "empty", "innerBitsSet"}, + {difference, "innerBitsSet", "full", "empty"}, + {difference, "innerBitsSet", "firstBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "lastBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "firstBitUnset", "empty"}, + {difference, "innerBitsSet", "lastBitUnset", "empty"}, + {difference, "innerBitsSet", "innerBitsSet", "empty"}, + {difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + // + {difference, "outerBitsSet", "empty", "outerBitsSet"}, + {difference, "outerBitsSet", "full", "empty"}, + {difference, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {difference, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {difference, "outerBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "outerBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, + {difference, "outerBitsSet", "outerBitsSet", "empty"}, + + // xor + {xor, "empty", "empty", "empty"}, + {xor, "empty", "full", "full"}, + {xor, "empty", "firstBitSet", "firstBitSet"}, + {xor, "empty", "lastBitSet", "lastBitSet"}, + {xor, "empty", "firstBitUnset", "firstBitUnset"}, + {xor, "empty", "lastBitUnset", "lastBitUnset"}, + {xor, "empty", "innerBitsSet", "innerBitsSet"}, + {xor, "empty", "outerBitsSet", "outerBitsSet"}, + // + {xor, "full", "empty", "full"}, + {xor, "full", "full", "empty"}, + {xor, "full", "firstBitSet", "firstBitUnset"}, + {xor, "full", "lastBitSet", "lastBitUnset"}, + {xor, "full", "firstBitUnset", "firstBitSet"}, + {xor, "full", "lastBitUnset", "lastBitSet"}, + {xor, "full", "innerBitsSet", "outerBitsSet"}, + {xor, "full", "outerBitsSet", "innerBitsSet"}, + // + {xor, "firstBitSet", "empty", "firstBitSet"}, + {xor, "firstBitSet", "full", "firstBitUnset"}, + {xor, "firstBitSet", "firstBitSet", "empty"}, + {xor, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {xor, "firstBitSet", "firstBitUnset", "full"}, + {xor, "firstBitSet", "lastBitUnset", "innerBitsSet"}, + {xor, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {xor, "firstBitSet", "outerBitsSet", "lastBitSet"}, + // + {xor, "lastBitSet", "empty", "lastBitSet"}, + {xor, "lastBitSet", "full", "lastBitUnset"}, + {xor, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {xor, "lastBitSet", "lastBitSet", "empty"}, + {xor, "lastBitSet", "firstBitUnset", "innerBitsSet"}, + {xor, "lastBitSet", "lastBitUnset", "full"}, + {xor, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {xor, "lastBitSet", "outerBitsSet", "firstBitSet"}, + // + {xor, "firstBitUnset", "empty", "firstBitUnset"}, + {xor, "firstBitUnset", "full", "firstBitSet"}, + {xor, "firstBitUnset", "firstBitSet", "full"}, + {xor, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {xor, "firstBitUnset", "firstBitUnset", "empty"}, + {xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"}, + {xor, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"}, + // + {xor, "lastBitUnset", "empty", "lastBitUnset"}, + {xor, "lastBitUnset", "full", "lastBitSet"}, + {xor, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {xor, "lastBitUnset", "lastBitSet", "full"}, + {xor, "lastBitUnset", "firstBitUnset", "outerBitsSet"}, + {xor, "lastBitUnset", "lastBitUnset", "empty"}, + {xor, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"}, + // + {xor, "innerBitsSet", "empty", "innerBitsSet"}, + {xor, "innerBitsSet", "full", "outerBitsSet"}, + {xor, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {xor, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {xor, "innerBitsSet", "firstBitUnset", "lastBitSet"}, + {xor, "innerBitsSet", "lastBitUnset", "firstBitSet"}, + {xor, "innerBitsSet", "innerBitsSet", "empty"}, + {xor, "innerBitsSet", "outerBitsSet", "full"}, + // + {xor, "outerBitsSet", "empty", "outerBitsSet"}, + {xor, "outerBitsSet", "full", "innerBitsSet"}, + {xor, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {xor, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {xor, "outerBitsSet", "firstBitUnset", "lastBitUnset"}, + {xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"}, + {xor, "outerBitsSet", "innerBitsSet", "full"}, + {xor, "outerBitsSet", "outerBitsSet", "empty"}, + } + for _, testOp := range testOps { + for _, x := range containerTypes { + for _, y := range containerTypes { + desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) + ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y]) + exp := testOp.exp + + // Convert to all container types and check result. + for _, ct := range containerTypes { + clone := ret.clone() + if ct == ContainerArray { + if clone.isBitmap() { + clone.bitmapToArray() + } else if clone.isRun() { + clone.runToArray() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + // Because xorRunRun resulting in an empty container returns an array container with a + // nil slice array, then we need to check len() on array first (look for 0). + if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { + t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) + } + } else if ct == ContainerBitmap { + if clone.isArray() { + clone.arrayToBitmap() + } else if clone.isRun() { + clone.runToBitmap() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { + t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) + } + } else if ct == ContainerRun { + if clone.isArray() { + clone.arrayToRun() + } else if clone.isBitmap() { + clone.bitmapToRun() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.runs, cts[ct][exp].runs) { + t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs, clone.runs) + } + } + } + } + } + } +} From 673232baf9261e85d440238bd1c77e770a3955e7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 14 Feb 2018 13:30:38 -0600 Subject: [PATCH 085/311] bug fixes found by running container operation tests --- roaring/roaring.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b7cdec1c9..b42da1418 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2181,7 +2181,7 @@ func unionArrayArray(a, b *container) *container { // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } output := &container{containerType: ContainerRun} @@ -2238,10 +2238,10 @@ func (c *container) runAppendInterval(v interval16) int { } func unionRunRun(a, b *container) *container { - if a.n == maxContainerVal { + if a.n == maxContainerVal+1 { return a.clone() } - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } na, nb := len(a.runs), len(b.runs) @@ -2272,9 +2272,12 @@ func unionRunRun(a, b *container) *container { } func unionBitmapRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } + if a.n == maxContainerVal+1 { + return a.clone() + } output := a.clone() for j := 0; j < len(b.runs); j++ { output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) @@ -2289,7 +2292,7 @@ func (c *container) bitmapSetRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) xcnt := popcnt(X) ycnt := popcnt(Y) if x == y { @@ -2312,7 +2315,7 @@ func (c *container) bitmapXorRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { cnt := popcnt(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip @@ -2784,7 +2787,9 @@ func xorArrayBitmap(a, b *container) *container { } } - if output.count() < ArrayMaxSize { + // It's possible that output was converted from bitmap to array in output.remove() + // so we only do this conversion if output is still a bitmap container. + if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -3199,7 +3204,7 @@ func xorRunRun(a, b *container) *container { if nb == 0 { return a.clone() } - output := &container{} + output := &container{containerType: ContainerRun} lastI, lastJ := -1, -1 From 911af238ff8b6cf5ff2a5f4af52b0ecafc7ee19a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 14 Feb 2018 15:06:53 -0600 Subject: [PATCH 086/311] clean up roaring container helper functions --- roaring/roaring_helpers_test.go | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index de3dd002c..204b2fe2f 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -68,10 +68,7 @@ func arrayInnerBitsSet() []uint16 { } func arrayOuterBitsSet() []uint16 { - array := make([]uint16, 0) - array = append(array, uint16(0)) - array = append(array, uint16(65535)) - return array + return []uint16{0, 65535} } ////////////////// bitmap @@ -100,39 +97,27 @@ func bitmapLastBitSet() []uint64 { } func bitmapFirstBitUnset() []uint64 { - bitmap := make([]uint64, bitmapN) + bitmap := bitmapFull() bitmap[0] = 0xFFFFFFFFFFFFFFFE - for i := 1; i < bitmapN; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } return bitmap } func bitmapLastBitUnset() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN-1; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } + bitmap := bitmapFull() bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF return bitmap } func bitmapInnerBitsSet() []uint64 { - bitmap := make([]uint64, bitmapN) + bitmap := bitmapFull() bitmap[0] = 0xFFFFFFFFFFFFFFFE - for i := 1; i < bitmapN-1; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF return bitmap } func bitmapOuterBitsSet() []uint64 { - bitmap := make([]uint64, bitmapN) + bitmap := bitmapEmpty() bitmap[0] = 0x0000000000000001 - for i := 1; i < bitmapN-1; i++ { - bitmap[i] = 0x0000000000000000 - } bitmap[bitmapN-1] = 0x8000000000000000 return bitmap } From 685a0075ae52779579adeed6a750be77d59cb263 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 15 Feb 2018 10:20:54 -0600 Subject: [PATCH 087/311] add evenBits/oddBits container tests and related bug fix --- roaring/roaring.go | 16 ++- roaring/roaring_helpers_test.go | 54 +++++++++ roaring/roaring_internal_test.go | 187 ++++++++++++++++++++++++++----- 3 files changed, 223 insertions(+), 34 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b42da1418..8d88b2088 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2476,10 +2476,18 @@ func differenceArrayRun(a, b *container) *container { if i < len(a.array) { // keep all array elements after end of runs - output.array = append(output.array, a.array[i:]...) - // TODO: consider handling container.n mutations in one place - // like we do with container.add(). - output.n += len(a.array[i:]) + // It's possible that output was converted from array to bitmap in output.add() + // so check container type before proceeding. + if output.containerType == ContainerArray { + output.array = append(output.array, a.array[i:]...) + // TODO: consider handling container.n mutations in one place + // like we do with container.add(). + output.n += len(a.array[i:]) + } else { + for _, v := range a.array[i:] { + output.add(v) + } + } } return output } diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 204b2fe2f..ea6d86cd9 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -71,6 +71,22 @@ func arrayOuterBitsSet() []uint16 { return []uint16{0, 65535} } +func arrayOddBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2*i + 1) + } + return array +} + +func arrayEvenBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2 * i) + } + return array +} + ////////////////// bitmap func bitmapEmpty() []uint64 { return make([]uint64, bitmapN) @@ -122,6 +138,22 @@ func bitmapOuterBitsSet() []uint64 { return bitmap } +func bitmapOddBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xAAAAAAAAAAAAAAAA + } + return bitmap +} + +func bitmapEvenBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0x5555555555555555 + } + return bitmap +} + ////////////////// run func runEmpty() []interval16 { return make([]interval16, 0) @@ -170,6 +202,22 @@ func runOuterBitsSet() []interval16 { return run } +func runOddBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2*i + 1), last: uint16(2*i + 1)} + } + return run +} + +func runEvenBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2 * i), last: uint16(2 * i)} + } + return run +} + /////////////////////////////////////////////////////////////////////////// type testOp struct { @@ -211,6 +259,8 @@ func setupContainerTests() map[byte]map[string]*container { "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()), } // bitmap containers @@ -223,6 +273,8 @@ func setupContainerTests() map[byte]map[string]*container { "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()), } // run containers @@ -235,6 +287,8 @@ func setupContainerTests() map[byte]map[string]*container { "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(ContainerRun, runOddBitsSet()), + "evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()), } return cts diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index ce9ab217d..dca413811 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1641,7 +1641,7 @@ func TestDifferenceBitmapRun(t *testing.T) { exp: []uint64{0x0000000000000000}, }, { - bitmap: bitmapLast(), + bitmap: bitmapLastBitSet(), runs: []interval16{{start: 65535, last: 65535}}, exp: bitmapEmpty(), }, @@ -1689,12 +1689,12 @@ func TestDifferenceBitmapArray(t *testing.T) { exp: []uint16{8, 9, 11, 12, 13, 14, 15}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10}, exp: []uint16{9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{63}, exp: []uint16{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61}, }, @@ -2610,12 +2610,12 @@ func TestIntersectArrayBitmap(t *testing.T) { }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), exp: []uint16{1, 63, 543, 639, 65535}, }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapEvens(), + bitmap: bitmapEvenBitsSet(), exp: []uint16{0, 120, 12000, 65534}, }, } @@ -2637,31 +2637,6 @@ func TestIntersectArrayBitmap(t *testing.T) { } } -func bitmapOdds() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0xAAAAAAAAAAAAAAAA - } - return bitmap -} - -func bitmapEvens() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0x5555555555555555 - } - return bitmap -} - -func bitmapLast() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN-1; i++ { - bitmap[i] = 0 - } - bitmap[bitmapN-1] = 0x8000000000000000 - return bitmap -} - // rleCont returns a slice of numbers all in the range starting from // container_width*num, and ending at container_width*(num+1)-1. If left is // true, then the first 100 bits will be set, if mid is true, 100 bits in the @@ -2776,6 +2751,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "empty", "lastBitUnset", "empty"}, {intersect, "empty", "innerBitsSet", "empty"}, {intersect, "empty", "outerBitsSet", "empty"}, + {intersect, "empty", "oddBitsSet", "empty"}, + {intersect, "empty", "evenBitsSet", "empty"}, // {intersect, "full", "empty", "empty"}, {intersect, "full", "full", "full"}, @@ -2785,6 +2762,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "full", "lastBitUnset", "lastBitUnset"}, {intersect, "full", "innerBitsSet", "innerBitsSet"}, {intersect, "full", "outerBitsSet", "outerBitsSet"}, + {intersect, "full", "oddBitsSet", "oddBitsSet"}, + {intersect, "full", "evenBitsSet", "evenBitsSet"}, // {intersect, "firstBitSet", "empty", "empty"}, {intersect, "firstBitSet", "full", "firstBitSet"}, @@ -2794,6 +2773,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "firstBitSet", "lastBitUnset", "firstBitSet"}, {intersect, "firstBitSet", "innerBitsSet", "empty"}, {intersect, "firstBitSet", "outerBitsSet", "firstBitSet"}, + {intersect, "firstBitSet", "oddBitsSet", "empty"}, + {intersect, "firstBitSet", "evenBitsSet", "firstBitSet"}, // {intersect, "lastBitSet", "empty", "empty"}, {intersect, "lastBitSet", "full", "lastBitSet"}, @@ -2803,6 +2784,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "lastBitSet", "lastBitUnset", "empty"}, {intersect, "lastBitSet", "innerBitsSet", "empty"}, {intersect, "lastBitSet", "outerBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "oddBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "evenBitsSet", "empty"}, // {intersect, "firstBitUnset", "empty", "empty"}, {intersect, "firstBitUnset", "full", "firstBitUnset"}, @@ -2812,6 +2795,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"}, {intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"}, {intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"}, + {intersect, "firstBitUnset", "oddBitsSet", "oddBitsSet"}, + //{intersect, "firstBitUnset", "evenBitsSet", ""}, // {intersect, "lastBitUnset", "empty", "empty"}, {intersect, "lastBitUnset", "full", "lastBitUnset"}, @@ -2821,6 +2806,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, {intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"}, {intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"}, + //{intersect, "lastBitUnset", "oddBitsSet", ""}, + {intersect, "lastBitUnset", "evenBitsSet", "evenBitsSet"}, // {intersect, "innerBitsSet", "empty", "empty"}, {intersect, "innerBitsSet", "full", "innerBitsSet"}, @@ -2830,6 +2817,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"}, {intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, {intersect, "innerBitsSet", "outerBitsSet", "empty"}, + //{intersect, "innerBitsSet", "oddBitsSet", ""}, + //{intersect, "innerBitsSet", "evenBitsSet", ""}, // {intersect, "outerBitsSet", "empty", "empty"}, {intersect, "outerBitsSet", "full", "outerBitsSet"}, @@ -2839,6 +2828,30 @@ func TestContainerCombinations(t *testing.T) { {intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"}, {intersect, "outerBitsSet", "innerBitsSet", "empty"}, {intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + {intersect, "outerBitsSet", "oddBitsSet", "lastBitSet"}, + {intersect, "outerBitsSet", "evenBitsSet", "firstBitSet"}, + // + {intersect, "oddBitsSet", "empty", "empty"}, + {intersect, "oddBitsSet", "full", "oddBitsSet"}, + {intersect, "oddBitsSet", "firstBitSet", "empty"}, + {intersect, "oddBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "oddBitsSet", "firstBitUnset", "oddBitsSet"}, + //{intersect, "oddBitsSet", "lastBitUnset", ""}, + //{intersect, "oddBitsSet", "innerBitsSet", ""}, + {intersect, "oddBitsSet", "outerBitsSet", "lastBitSet"}, + {intersect, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {intersect, "oddBitsSet", "evenBitsSet", "empty"}, + // + {intersect, "evenBitsSet", "empty", "empty"}, + {intersect, "evenBitsSet", "full", "evenBitsSet"}, + {intersect, "evenBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "evenBitsSet", "lastBitSet", "empty"}, + //{intersect, "evenBitsSet", "firstBitUnset", ""}, + {intersect, "evenBitsSet", "lastBitUnset", "evenBitsSet"}, + //{intersect, "evenBitsSet", "innerBitsSet", ""}, + {intersect, "evenBitsSet", "outerBitsSet", "firstBitSet"}, + {intersect, "evenBitsSet", "oddBitsSet", "empty"}, + {intersect, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, // union {union, "empty", "empty", "empty"}, @@ -2849,6 +2862,8 @@ func TestContainerCombinations(t *testing.T) { {union, "empty", "lastBitUnset", "lastBitUnset"}, {union, "empty", "innerBitsSet", "innerBitsSet"}, {union, "empty", "outerBitsSet", "outerBitsSet"}, + {union, "empty", "oddBitsSet", "oddBitsSet"}, + {union, "empty", "evenBitsSet", "evenBitsSet"}, // {union, "full", "empty", "full"}, {union, "full", "full", "full"}, @@ -2858,6 +2873,8 @@ func TestContainerCombinations(t *testing.T) { {union, "full", "lastBitUnset", "full"}, {union, "full", "innerBitsSet", "full"}, {union, "full", "outerBitsSet", "full"}, + {union, "full", "oddBitsSet", "full"}, + {union, "full", "evenBitsSet", "full"}, // {union, "firstBitSet", "empty", "firstBitSet"}, {union, "firstBitSet", "full", "full"}, @@ -2867,6 +2884,8 @@ func TestContainerCombinations(t *testing.T) { {union, "firstBitSet", "lastBitUnset", "lastBitUnset"}, {union, "firstBitSet", "innerBitsSet", "lastBitUnset"}, {union, "firstBitSet", "outerBitsSet", "outerBitsSet"}, + //{union, "firstBitSet", "oddBitsSet", ""}, + {union, "firstBitSet", "evenBitsSet", "evenBitsSet"}, // {union, "lastBitSet", "empty", "lastBitSet"}, {union, "lastBitSet", "full", "full"}, @@ -2876,6 +2895,8 @@ func TestContainerCombinations(t *testing.T) { {union, "lastBitSet", "lastBitUnset", "full"}, {union, "lastBitSet", "innerBitsSet", "firstBitUnset"}, {union, "lastBitSet", "outerBitsSet", "outerBitsSet"}, + {union, "lastBitSet", "oddBitsSet", "oddBitsSet"}, + //{union, "lastBitSet", "evenBitsSet", ""}, // {union, "firstBitUnset", "empty", "firstBitUnset"}, {union, "firstBitUnset", "full", "full"}, @@ -2885,6 +2906,8 @@ func TestContainerCombinations(t *testing.T) { {union, "firstBitUnset", "lastBitUnset", "full"}, {union, "firstBitUnset", "innerBitsSet", "firstBitUnset"}, {union, "firstBitUnset", "outerBitsSet", "full"}, + {union, "firstBitUnset", "oddBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "evenBitsSet", "full"}, // {union, "lastBitUnset", "empty", "lastBitUnset"}, {union, "lastBitUnset", "full", "full"}, @@ -2894,6 +2917,8 @@ func TestContainerCombinations(t *testing.T) { {union, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, {union, "lastBitUnset", "innerBitsSet", "lastBitUnset"}, {union, "lastBitUnset", "outerBitsSet", "full"}, + {union, "lastBitUnset", "oddBitsSet", "full"}, + {union, "lastBitUnset", "evenBitsSet", "lastBitUnset"}, // {union, "innerBitsSet", "empty", "innerBitsSet"}, {union, "innerBitsSet", "full", "full"}, @@ -2903,6 +2928,8 @@ func TestContainerCombinations(t *testing.T) { {union, "innerBitsSet", "lastBitUnset", "lastBitUnset"}, {union, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, {union, "innerBitsSet", "outerBitsSet", "full"}, + {union, "innerBitsSet", "oddBitsSet", "firstBitUnset"}, + {union, "innerBitsSet", "evenBitsSet", "lastBitUnset"}, // {union, "outerBitsSet", "empty", "outerBitsSet"}, {union, "outerBitsSet", "full", "full"}, @@ -2912,6 +2939,30 @@ func TestContainerCombinations(t *testing.T) { {union, "outerBitsSet", "lastBitUnset", "full"}, {union, "outerBitsSet", "innerBitsSet", "full"}, {union, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + //{union, "outerBitsSet", "oddBitsSet", ""}, + //{union, "outerBitsSet", "evenBitsSet", ""}, + // + {union, "oddBitsSet", "empty", "oddBitsSet"}, + {union, "oddBitsSet", "full", "full"}, + //{union, "oddBitsSet", "firstBitSet", ""}, + {union, "oddBitsSet", "lastBitSet", "oddBitsSet"}, + {union, "oddBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "oddBitsSet", "lastBitUnset", "full"}, + {union, "oddBitsSet", "innerBitsSet", "firstBitUnset"}, + //{union, "oddBitsSet", "outerBitsSet", ""}, + {union, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {union, "oddBitsSet", "evenBitsSet", "full"}, + // + {union, "evenBitsSet", "empty", "evenBitsSet"}, + {union, "evenBitsSet", "full", "full"}, + {union, "evenBitsSet", "firstBitSet", "evenBitsSet"}, + //{union, "evenBitsSet", "lastBitSet", ""}, + {union, "evenBitsSet", "firstBitUnset", "full"}, + {union, "evenBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "evenBitsSet", "innerBitsSet", "lastBitUnset"}, + //{union, "evenBitsSet", "outerBitsSet", ""}, + {union, "evenBitsSet", "oddBitsSet", "full"}, + {union, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, // difference {difference, "empty", "empty", "empty"}, @@ -2922,6 +2973,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "empty", "lastBitUnset", "empty"}, {difference, "empty", "innerBitsSet", "empty"}, {difference, "empty", "outerBitsSet", "empty"}, + {difference, "empty", "oddBitsSet", "empty"}, + {difference, "empty", "evenBitsSet", "empty"}, // {difference, "full", "empty", "full"}, {difference, "full", "full", "empty"}, @@ -2931,6 +2984,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "full", "lastBitUnset", "lastBitSet"}, {difference, "full", "innerBitsSet", "outerBitsSet"}, {difference, "full", "outerBitsSet", "innerBitsSet"}, + {difference, "full", "oddBitsSet", "evenBitsSet"}, + {difference, "full", "evenBitsSet", "oddBitsSet"}, // {difference, "firstBitSet", "empty", "firstBitSet"}, {difference, "firstBitSet", "full", "empty"}, @@ -2940,6 +2995,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "firstBitSet", "lastBitUnset", "empty"}, {difference, "firstBitSet", "innerBitsSet", "firstBitSet"}, {difference, "firstBitSet", "outerBitsSet", "empty"}, + {difference, "firstBitSet", "oddBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "evenBitsSet", "empty"}, // {difference, "lastBitSet", "empty", "lastBitSet"}, {difference, "lastBitSet", "full", "empty"}, @@ -2949,6 +3006,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "lastBitSet", "lastBitUnset", "lastBitSet"}, {difference, "lastBitSet", "innerBitsSet", "lastBitSet"}, {difference, "lastBitSet", "outerBitsSet", "empty"}, + {difference, "lastBitSet", "oddBitsSet", "empty"}, + {difference, "lastBitSet", "evenBitsSet", "lastBitSet"}, // {difference, "firstBitUnset", "empty", "firstBitUnset"}, {difference, "firstBitUnset", "full", "empty"}, @@ -2958,6 +3017,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "firstBitUnset", "lastBitUnset", "lastBitSet"}, {difference, "firstBitUnset", "innerBitsSet", "lastBitSet"}, {difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + //{difference, "firstBitUnset", "oddBitsSet", ""}, + {difference, "firstBitUnset", "evenBitsSet", "oddBitsSet"}, // {difference, "lastBitUnset", "empty", "lastBitUnset"}, {difference, "lastBitUnset", "full", "empty"}, @@ -2967,6 +3028,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "lastBitUnset", "lastBitUnset", "empty"}, {difference, "lastBitUnset", "innerBitsSet", "firstBitSet"}, {difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + {difference, "lastBitUnset", "oddBitsSet", "evenBitsSet"}, + //{difference, "lastBitUnset", "evenBitsSet", ""}, // {difference, "innerBitsSet", "empty", "innerBitsSet"}, {difference, "innerBitsSet", "full", "empty"}, @@ -2976,6 +3039,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "innerBitsSet", "lastBitUnset", "empty"}, {difference, "innerBitsSet", "innerBitsSet", "empty"}, {difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + //{difference, "innerBitsSet", "oddBitsSet", ""}, + //{difference, "innerBitsSet", "evenBitsSet", ""}, // {difference, "outerBitsSet", "empty", "outerBitsSet"}, {difference, "outerBitsSet", "full", "empty"}, @@ -2985,6 +3050,30 @@ func TestContainerCombinations(t *testing.T) { {difference, "outerBitsSet", "lastBitUnset", "lastBitSet"}, {difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, {difference, "outerBitsSet", "outerBitsSet", "empty"}, + {difference, "outerBitsSet", "oddBitsSet", "firstBitSet"}, + {difference, "outerBitsSet", "evenBitsSet", "lastBitSet"}, + // + {difference, "oddBitsSet", "empty", "oddBitsSet"}, + {difference, "oddBitsSet", "full", "empty"}, + {difference, "oddBitsSet", "firstBitSet", "oddBitsSet"}, + //{difference, "oddBitsSet", "lastBitSet", ""}, + {difference, "oddBitsSet", "firstBitUnset", "empty"}, + {difference, "oddBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "oddBitsSet", "innerBitsSet", "lastBitSet"}, + //{difference, "oddBitsSet", "outerBitsSet", ""}, + {difference, "oddBitsSet", "oddBitsSet", "empty"}, + {difference, "oddBitsSet", "evenBitsSet", "oddBitsSet"}, + // + {difference, "evenBitsSet", "empty", "evenBitsSet"}, + {difference, "evenBitsSet", "full", "empty"}, + //{difference, "evenBitsSet", "firstBitSet", ""}, + {difference, "evenBitsSet", "lastBitSet", "evenBitsSet"}, + {difference, "evenBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "evenBitsSet", "lastBitUnset", "empty"}, + {difference, "evenBitsSet", "innerBitsSet", "firstBitSet"}, + //{difference, "evenBitsSet", "outerBitsSet", ""}, + {difference, "evenBitsSet", "oddBitsSet", "evenBitsSet"}, + {difference, "evenBitsSet", "evenBitsSet", "empty"}, // xor {xor, "empty", "empty", "empty"}, @@ -2995,6 +3084,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "empty", "lastBitUnset", "lastBitUnset"}, {xor, "empty", "innerBitsSet", "innerBitsSet"}, {xor, "empty", "outerBitsSet", "outerBitsSet"}, + {xor, "empty", "oddBitsSet", "oddBitsSet"}, + {xor, "empty", "evenBitsSet", "evenBitsSet"}, // {xor, "full", "empty", "full"}, {xor, "full", "full", "empty"}, @@ -3004,6 +3095,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "full", "lastBitUnset", "lastBitSet"}, {xor, "full", "innerBitsSet", "outerBitsSet"}, {xor, "full", "outerBitsSet", "innerBitsSet"}, + {xor, "full", "oddBitsSet", "evenBitsSet"}, + {xor, "full", "evenBitsSet", "oddBitsSet"}, // {xor, "firstBitSet", "empty", "firstBitSet"}, {xor, "firstBitSet", "full", "firstBitUnset"}, @@ -3013,6 +3106,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "firstBitSet", "lastBitUnset", "innerBitsSet"}, {xor, "firstBitSet", "innerBitsSet", "lastBitUnset"}, {xor, "firstBitSet", "outerBitsSet", "lastBitSet"}, + //{xor, "firstBitSet", "oddBitsSet", ""}, + //{xor, "firstBitSet", "evenBitsSet", ""}, // {xor, "lastBitSet", "empty", "lastBitSet"}, {xor, "lastBitSet", "full", "lastBitUnset"}, @@ -3022,6 +3117,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "lastBitSet", "lastBitUnset", "full"}, {xor, "lastBitSet", "innerBitsSet", "firstBitUnset"}, {xor, "lastBitSet", "outerBitsSet", "firstBitSet"}, + //{xor, "lastBitSet", "oddBitsSet", ""}, + //{xor, "lastBitSet", "evenBitsSet", ""}, // {xor, "firstBitUnset", "empty", "firstBitUnset"}, {xor, "firstBitUnset", "full", "firstBitSet"}, @@ -3031,6 +3128,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"}, {xor, "firstBitUnset", "innerBitsSet", "lastBitSet"}, {xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"}, + //{xor, "firstBitUnset", "oddBitsSet", ""}, + //{xor, "firstBitUnset", "evenBitsSet", ""}, // {xor, "lastBitUnset", "empty", "lastBitUnset"}, {xor, "lastBitUnset", "full", "lastBitSet"}, @@ -3040,6 +3139,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "lastBitUnset", "lastBitUnset", "empty"}, {xor, "lastBitUnset", "innerBitsSet", "firstBitSet"}, {xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"}, + //{xor, "lastBitUnset", "oddBitsSet", ""}, + //{xor, "lastBitUnset", "evenBitsSet", ""}, // {xor, "innerBitsSet", "empty", "innerBitsSet"}, {xor, "innerBitsSet", "full", "outerBitsSet"}, @@ -3049,6 +3150,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "innerBitsSet", "lastBitUnset", "firstBitSet"}, {xor, "innerBitsSet", "innerBitsSet", "empty"}, {xor, "innerBitsSet", "outerBitsSet", "full"}, + //{xor, "innerBitsSet", "oddBitsSet", ""}, + //{xor, "innerBitsSet", "evenBitsSet", ""}, // {xor, "outerBitsSet", "empty", "outerBitsSet"}, {xor, "outerBitsSet", "full", "innerBitsSet"}, @@ -3058,6 +3161,30 @@ func TestContainerCombinations(t *testing.T) { {xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"}, {xor, "outerBitsSet", "innerBitsSet", "full"}, {xor, "outerBitsSet", "outerBitsSet", "empty"}, + //{xor, "outerBitsSet", "oddBitsSet", ""}, + //{xor, "outerBitsSet", "evenBitsSet", ""}, + // + {xor, "oddBitsSet", "empty", "oddBitsSet"}, + {xor, "oddBitsSet", "full", "evenBitsSet"}, + //{xor, "oddBitsSet", "firstBitSet", ""}, + //{xor, "oddBitsSet", "lastBitSet", ""}, + //{xor, "oddBitsSet", "firstBitUnset", ""}, + //{xor, "oddBitsSet", "lastBitUnset", ""}, + //{xor, "oddBitsSet", "innerBitsSet", ""}, + //{xor, "oddBitsSet", "outerBitsSet", ""}, + {xor, "oddBitsSet", "oddBitsSet", "empty"}, + {xor, "oddBitsSet", "evenBitsSet", "full"}, + // + {xor, "evenBitsSet", "empty", "evenBitsSet"}, + {xor, "evenBitsSet", "full", "oddBitsSet"}, + //{xor, "evenBitsSet", "firstBitSet", ""}, + //{xor, "evenBitsSet", "lastBitSet", ""}, + //{xor, "evenBitsSet", "firstBitUnset", ""}, + //{xor, "evenBitsSet", "lastBitUnset", ""}, + //{xor, "evenBitsSet", "innerBitsSet", ""}, + //{xor, "evenBitsSet", "outerBitsSet", ""}, + {xor, "evenBitsSet", "oddBitsSet", "full"}, + {xor, "evenBitsSet", "evenBitsSet", "empty"}, } for _, testOp := range testOps { for _, x := range containerTypes { From 9558df2829c23176feca4282710a007a4da84763 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Feb 2018 10:50:03 -0600 Subject: [PATCH 088/311] Make pre-release builds for all branches --- .travis.yml | 2 +- Makefile | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6efa61e43..b54f4fc8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ deploy: script: make prerelease-upload skip_cleanup: true on: - branch: master + all_branches: true matrix: allow_failures: - go: master diff --git a/Makefile b/Makefile index 48c0075a8..8be692b6f 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,8 @@ PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" DOCKER_GOLANG_IMAGE=golang:latest +BRANCH=$(shell git rev-parse --abbrev-ref HEAD) +BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH) default: test pilosa @@ -75,16 +77,16 @@ else endif prerelease-build: vendor - make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa" - cp LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH) - tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/ - @echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz" + make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa" + cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER) + tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/ + @echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz" prerelease: make prerelease-build GOOS=linux GOARCH=amd64 prerelease-upload: prerelease - aws s3 cp build/pilosa-master-linux-amd64.tar.gz s3://build.pilosa.com/pilosa-master-linux-amd64.tar.gz --acl public-read + aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read install: vendor go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From 666fe22ecbd50e8b42ea9e5bc6c51da664b2390b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 15 Feb 2018 12:02:06 -0600 Subject: [PATCH 089/311] replace swallowed error with log entry --- cluster.go | 2 ++ gossip/gossip.go | 17 ++++++++++++++--- server/cluster_test.go | 4 ++-- server/server.go | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cluster.go b/cluster.go index 76642107b..667a3d348 100644 --- a/cluster.go +++ b/cluster.go @@ -405,6 +405,8 @@ func (c *Cluster) setState(state string) { if c.state == ClusterStateResizing { doCleanup = true } + default: + panic(fmt.Sprintf("invalid cluster state: %s", state)) } c.state = state diff --git a/gossip/gossip.go b/gossip/gossip.go index acfad66a1..19905f2b3 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -338,12 +338,16 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { type GossipEventReceiver struct { ch chan memberlist.NodeEvent eventHandler pilosa.EventHandler + + // The writer for any logging. + LogOutput io.Writer } // NewGossipEventReceiver returns a new instance of GossipEventReceiver. -func NewGossipEventReceiver() *GossipEventReceiver { +func NewGossipEventReceiver(logOutput io.Writer) *GossipEventReceiver { return &GossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), + ch: make(chan memberlist.NodeEvent, 1), + LogOutput: logOutput, } } @@ -366,6 +370,11 @@ func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error { return nil } +// logger returns a logger for the GossipEventReceiver. +func (g *GossipEventReceiver) logger() *log.Logger { + return log.New(g.LogOutput, "", log.LstdFlags) +} + func (g *GossipEventReceiver) listen() { var nodeEventType pilosa.NodeEventType for { @@ -392,7 +401,9 @@ func (g *GossipEventReceiver) listen() { Event: nodeEventType, Node: node, } - _ = g.eventHandler.ReceiveEvent(ne) + if err := g.eventHandler.ReceiveEvent(ne); err != nil { + g.logger().Printf("receive event error: %s", err) + } } } diff --git a/server/cluster_test.go b/server/cluster_test.go index 23284bb00..46be3d45a 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -51,7 +51,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.Coordinator = m0.Server.URI m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} - m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput) gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) if err != nil { t.Fatal(err) @@ -78,7 +78,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Config.Gossip.Seed = gossipMemberSet0.Seed() m1.Server.Cluster.Coordinator = m0.Server.URI - m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput) gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) if err != nil { t.Fatal(err) diff --git a/server/server.go b/server/server.go index ef07ee381..cf73f1b0a 100644 --- a/server/server.go +++ b/server/server.go @@ -262,7 +262,7 @@ func (m *Command) SetupNetworking() error { m.Server.NodeID = m.Server.LoadNodeID() - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput) gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) if err != nil { return err From b8d4f7233a678aecc2ea23cf7608d1b8ba5599ca Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Feb 2018 13:59:28 -0600 Subject: [PATCH 090/311] Only deploy (prerelease upload) if using latest Go --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b54f4fc8a..f3679c2ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ deploy: skip_cleanup: true on: all_branches: true + go: 1.9 matrix: allow_failures: - go: master From f9c0c1548ea68178ab8e8f96dca8040c27ba71b1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Feb 2018 16:32:08 -0600 Subject: [PATCH 091/311] Fix bug with prerelease upload --- .travis.yml | 2 +- Makefile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f3679c2ec..a3e21f83c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ before_deploy: - pip install awscli --user `whoami` deploy: - provider: script - script: make prerelease-upload + script: make prerelease-upload GOOS=linux GOARCH=amd64 skip_cleanup: true on: all_branches: true diff --git a/Makefile b/Makefile index 8be692b6f..b1b545042 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) BUILD_TIME=`date -u +%FT%T%z` LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" DOCKER_GOLANG_IMAGE=golang:latest -BRANCH=$(shell git rev-parse --abbrev-ref HEAD) +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH)) BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH) default: test pilosa From ea210cd525dd5e17be0e565e51e1ddbf92aa6eb6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 16 Feb 2018 11:21:47 -0600 Subject: [PATCH 092/311] remove extra context import --- server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server.go b/server.go index b8ee6f70a..f41b84908 100644 --- a/server.go +++ b/server.go @@ -36,7 +36,6 @@ import ( "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) From 8e6041d0fe36fb9d3e30812f425d76efb062d8f9 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 19 Feb 2018 13:55:19 +0300 Subject: [PATCH 093/311] Validate /fragment/nodes arguments. Fixes #652 --- handler.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++- handler_test.go | 9 ++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/handler.go b/handler.go index e5cce4648..f874b4103 100644 --- a/handler.go +++ b/handler.go @@ -27,6 +27,7 @@ import ( "io/ioutil" "log" "net/http" + "net/url" // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" "os" @@ -70,6 +71,9 @@ type Handler struct { // The writer for any logging. LogOutput io.Writer + + // Keeps the query argument validators for each handler + validators map[string]queryValidationSpec } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -91,9 +95,19 @@ func NewHandler() *Handler { LogOutput: os.Stderr, } handler.Router = NewRouter(handler) + handler.populateValidators() return handler } +func (h *Handler) populateValidators() { + h.validators = map[string]queryValidationSpec{} + // GetFragmentNodes validator + validator := NewQueryValidationSpec() + validator.Required("slice") + validator.Optional("index") + h.validators["GetFragmentNodes"] = validator +} + // NewRouter creates a Gorilla Mux http router. func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() @@ -1342,12 +1356,18 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // handleGetFragmentNodes handles /fragment/nodes requests. func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() + validator := h.validators["GetFragmentNodes"] + err := validator.validate(q) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } index := q.Get("index") // Read slice parameter. slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) + http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) return } @@ -2057,3 +2077,41 @@ func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { } type defaultClusterMessageResponse struct{} + +type queryValidationSpec struct { + required []string + args map[string]bool +} + +func NewQueryValidationSpec() queryValidationSpec { + return queryValidationSpec{ + args: map[string]bool{}, + } +} + +func (s *queryValidationSpec) Required(args ...string) { + s.required = args + for _, arg := range args { + s.args[arg] = true + } +} + +func (s *queryValidationSpec) Optional(args ...string) { + for _, arg := range args { + s.args[arg] = true + } +} + +func (s queryValidationSpec) validate(query url.Values) error { + for _, req := range s.required { + if query.Get(req) == "" { + return errors.New(fmt.Sprintf("%s is required", req)) + } + } + for k, _ := range query { + if _, ok := s.args[k]; !ok { + return errors.New(fmt.Sprintf("%s is not a valid argument", k)) + } + } + return nil +} diff --git a/handler_test.go b/handler_test.go index 585230b15..636665022 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1198,6 +1198,15 @@ func TestHandler_Fragment_Nodes(t *testing.T) { } else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } + + // invalid argument should return BadRequest + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + } // Ensure the handler can return expvars without panicking. From a8e1e93c34aba5a7a2952156d1e820ae1ed5c9f0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 19 Feb 2018 10:34:47 -0600 Subject: [PATCH 094/311] Go 1.10 --- .travis.yml | 1 + Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ddff4f4b..73ab4adf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: go go: - 1.8 - 1.9 + - 1.10 - master env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY diff --git a/Dockerfile b/Dockerfile index 6d86de974..50e88985c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9.2 as builder +FROM golang:1.10 as builder ARG ldflags='' From cf7faa224146b3c381985bd17bdcc53332843a82 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 19 Feb 2018 10:48:02 -0600 Subject: [PATCH 095/311] Use string instead of float for go 1.10 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 73ab4adf0..1e5ff0976 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.8 - 1.9 - - 1.10 + - "1.10" # Use string, as 1.10==1.1 if interpreted as float. - master env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY From c1f9dd4e593e0229c196fd545b09d017abe7f43a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 20 Feb 2018 15:38:58 +0300 Subject: [PATCH 096/311] Added query arg validator middleware; updated Gorilla/mux --- Gopkg.lock | 4 ++-- handler.go | 55 ++++++++++++++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a8071c915..c77dc951a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -88,8 +88,8 @@ [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "7f08801859139f86dfafd1c296e2cba9a80d292e" - version = "v1.6.0" + revision = "53c1911da2b537f792e7cafcb446b05ffe33b996" + version = "v1.6.1" [[projects]] branch = "master" diff --git a/handler.go b/handler.go index f874b4103..895141135 100644 --- a/handler.go +++ b/handler.go @@ -73,7 +73,7 @@ type Handler struct { LogOutput io.Writer // Keeps the query argument validators for each handler - validators map[string]queryValidationSpec + validators map[string]*queryValidationSpec } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -100,12 +100,23 @@ func NewHandler() *Handler { } func (h *Handler) populateValidators() { - h.validators = map[string]queryValidationSpec{} - // GetFragmentNodes validator - validator := NewQueryValidationSpec() - validator.Required("slice") - validator.Optional("index") - h.validators["GetFragmentNodes"] = validator + h.validators = map[string]*queryValidationSpec{} + h.validators["GET /fragment/nodes"] = QueryValidationSpecRequired("slice").Optional("index") +} + +func (h *Handler) queryArgValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := fmt.Sprintf("%s %s", r.Method, r.URL.Path) + if validator, ok := h.validators[key]; ok { + q := r.URL.Query() + err := validator.validate(q) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + next.ServeHTTP(w, r) + }) } // NewRouter creates a Gorilla Mux http router. @@ -160,6 +171,8 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + router.Use(handler.queryArgValidator) + return router } @@ -1356,12 +1369,6 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // handleGetFragmentNodes handles /fragment/nodes requests. func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - validator := h.validators["GetFragmentNodes"] - err := validator.validate(q) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } index := q.Get("index") // Read slice parameter. @@ -2083,23 +2090,23 @@ type queryValidationSpec struct { args map[string]bool } -func NewQueryValidationSpec() queryValidationSpec { - return queryValidationSpec{ - args: map[string]bool{}, +func QueryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { + args := map[string]bool{} + for _, arg := range requiredArgs { + args[arg] = true + } + + return &queryValidationSpec{ + required: requiredArgs, + args: args, } } -func (s *queryValidationSpec) Required(args ...string) { - s.required = args - for _, arg := range args { - s.args[arg] = true - } -} - -func (s *queryValidationSpec) Optional(args ...string) { +func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { for _, arg := range args { s.args[arg] = true } + return s } func (s queryValidationSpec) validate(query url.Values) error { From 2d458b98a1018a51bb8e1ac36be19e7a8b061942 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 20 Feb 2018 10:41:02 -0600 Subject: [PATCH 097/311] merge conflicts --- roaring/assembly.go | 68 ------------------ roaring/assembly_amd64.s | 122 ------------------------------- roaring/assembly_asm.go | 87 ---------------------- roaring/assembly_generic.go | 27 ------- roaring/assembly_test.go | 139 ------------------------------------ roaring/roaring.go | 129 ++++++++++++++++++--------------- 6 files changed, 73 insertions(+), 499 deletions(-) delete mode 100644 roaring/assembly.go delete mode 100644 roaring/assembly_amd64.s delete mode 100644 roaring/assembly_asm.go delete mode 100644 roaring/assembly_generic.go delete mode 100644 roaring/assembly_test.go diff --git a/roaring/assembly.go b/roaring/assembly.go deleted file mode 100644 index ea67cf66e..000000000 --- a/roaring/assembly.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package roaring - - -// bit population count, take from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -func popcntGo(x uint64) (n uint64) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} - -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) - for _, x := range s { - cnt += popcntGo(x) - } - return cnt -} - -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] &^ m[i]) - } - return cnt -} - -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] & m[i]) - } - return cnt -} - -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] | m[i]) - } - return cnt -} - -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] ^ m[i]) - } - return cnt -} diff --git a/roaring/assembly_amd64.s b/roaring/assembly_amd64.s deleted file mode 100644 index 5b65e0635..000000000 --- a/roaring/assembly_amd64.s +++ /dev/null @@ -1,122 +0,0 @@ -#include "textflag.h" - -TEXT ·hasAsm(SB),4,$0-1 - MOVQ $1, AX - CPUID - SHRQ $23, CX - ANDQ $1, CX - MOVB CX, ret+0(FP) - RET - -TEXT ·POPCNTQ(SB),NOSPLIT,$0-16 - MOVQ memory+0(FP), BP - POPCNTQ BP, BX - MOVQ BX, ret+8(FP) - RET - -TEXT ·BSFQ(SB),NOSPLIT,$0-16 - MOVQ memory+0(FP), BP - BSFQ BP, BX - MOVQ BX, ret+8(FP) - RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET - - -TEXT ·popcntAsm(SB),4,$0-16 - MOVQ x+0(FP), DX - POPCNTQ_DX_DX - MOVQ DX, ret+8(FP) - RET diff --git a/roaring/assembly_asm.go b/roaring/assembly_asm.go deleted file mode 100644 index 3eb182cce..000000000 --- a/roaring/assembly_asm.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package roaring - -func hasAsm() bool - -func BSFQ(memory uint64) int - -func POPCNTQ(memory uint64) int - - -//go:noescape -var useAsm = hasAsm() - -//go:noescape -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntXorSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntAsm(x uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} - -func popcnt(x uint64) uint64 { - if useAsm { - return popcntAsm(x) - } - return popcntGo(x) -} diff --git a/roaring/assembly_generic.go b/roaring/assembly_generic.go deleted file mode 100644 index 997bfdaa3..000000000 --- a/roaring/assembly_generic.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !amd64 - -package roaring - -func hasAsm() bool {return false} - - -func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) } -func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) } -func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) } -func popcntOrSlice(s, m []uint64) uint64 { return popcntOrSliceGo(s, m) } -func popcntXorSlice(s, m []uint64) uint64 { return popcntXorSliceGo(s, m) } -func popcnt(s uint64) uint64 { return popcntGo(s) } diff --git a/roaring/assembly_test.go b/roaring/assembly_test.go deleted file mode 100644 index f18aa65ec..000000000 --- a/roaring/assembly_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package roaring - -import "testing" - -func TestBSFQ(t *testing.T) { - result := BSFQ(2) - if result != 1 { - t.Fatalf("BSF INCORRECT: %d", result) - } -} - -func TestBSFQ_CompareGo(t *testing.T) { - v := uint64(1) - for i := 0; i < 64; i++ { - if BSFQ(v) != trailingZeroN(v) { - t.Fatalf("BSF INCORRECT: %d %d", BSFQ(v), trailingZeroN(v)) - } - if v == 0 { - v = 1 - } else { - v *= 2 - } - } - /* - if bsfq(0) != trailingZeroN(0) { - fmt.Println(bsfq(0)) - t.Fatalf("BSF INCORRECT") - } - */ -} -func BenchmarkBSF(b *testing.B) { - for i := 0; i < b.N; i++ { - BSFQ(uint64(i)) - } -} - -func BenchmarkTrailingZeroN(b *testing.B) { - for i := 0; i < b.N; i++ { - trailingZeroN(uint64(i)) - } -} - -func BenchmarkPOPCNTQ(b *testing.B) { - for i := 0; i < b.N; i++ { - POPCNTQ(uint64(i)) - } -} - -func BenchmarkPopcount(b *testing.B) { - for i := 0; i < b.N; i++ { - popcount(uint64(i)) - } -} - -func BenchmarkPopcntAsm(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - popcntAsm(0xdeadbeef) - } -} - -func BenchmarkPopcntGo(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - popcntGo(0xdeadbeef) - } -} - -func getData() []uint64 { - return []uint64{ - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - } -} - -func BenchmarkPopcntSliceGo(b *testing.B) { - d := getData() - for n := 0; n < b.N; n++ { - popcntSliceGo(d) - } -} - -func BenchmarkPopcntSliceAsm(b *testing.B) { - d := getData() - for n := 0; n < b.N; n++ { - popcntSliceAsm(d) - } -} - -func BenchmarkPopcntSlice(b *testing.B) { - d := getData() - for n := 0; n < b.N; n++ { - popcntSlice(d) - } -} diff --git a/roaring/roaring.go b/roaring/roaring.go index 8d88b2088..8ea82af83 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -21,6 +21,7 @@ import ( "fmt" "hash/fnv" "io" + "math/bits" "sort" "unsafe" ) @@ -1274,10 +1275,10 @@ func (c *container) contains(v uint16) bool { func (c *container) bitmapCountRuns() (r int) { for i := 0; i < 1023; i++ { v, v1 := c.bitmap[i], c.bitmap[i+1] - r = r + int(popcnt((v<<1)&^v)+((v>>63)&^v1)) + r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1)) } vl := c.bitmap[len(c.bitmap)-1] - r = r + int(popcnt((vl<<1)&^vl)+vl>>63) + r = r + int(popcount((vl<<1)&^vl)+vl>>63) return r } @@ -1918,7 +1919,7 @@ func intersectionCountArrayBitmap(a, b *container) (n int) { } func intersectionCountBitmapBitmap(a, b *container) (n int) { - return int(popcntAndSlice(a.bitmap, b.bitmap)) + return int(popcountAndSlice(a.bitmap, b.bitmap)) } func intersect(a, b *container) *container { @@ -2061,22 +2062,22 @@ func intersectBitmapRun(a, b *container) *container { for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b output.bitmap[i] = a.bitmap[i] - output.n += int(popcnt(a.bitmap[i])) + output.n += int(popcount(a.bitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) bits := a.bitmap[i] & mask output.bitmap[i] |= bits - output.n += int(popcnt(bits)) + output.n += int(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) bits := (a.bitmap[i] >> offset) << offset output.bitmap[i] |= bits - output.n += int(popcnt(bits)) + output.n += int(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) bits := (a.bitmap[i] << offset) >> offset output.bitmap[i] |= bits - output.n += int(popcnt(bits)) + output.n += int(popcount(bits)) } // update loop vars i++ @@ -2293,19 +2294,19 @@ func (c *container) bitmapSetRange(i, j uint64) { y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) - xcnt := popcnt(X) - ycnt := popcnt(Y) + xcnt := popcount(X) + ycnt := popcount(Y) if x == y { - c.n += int((j - i) - popcnt(c.bitmap[x]&(X&Y))) + c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y))) c.bitmap[x] |= (X & Y) } else { - c.n += int(xcnt - popcnt(c.bitmap[x]&X)) + c.n += int(xcnt - popcount(c.bitmap[x]&X)) c.bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int(64 - popcnt(c.bitmap[i])) + c.n += int(64 - popcount(c.bitmap[i])) c.bitmap[i] = maxBitmap } - c.n += int(ycnt - popcnt(c.bitmap[y]&Y)) + c.n += int(ycnt - popcount(c.bitmap[y]&Y)) c.bitmap[y] |= Y } } @@ -2317,21 +2318,21 @@ func (c *container) bitmapXorRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { - cnt := popcnt(c.bitmap[x]) + cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip - c.n += int(popcnt(c.bitmap[x]) - cnt) + c.n += int(popcount(c.bitmap[x]) - cnt) } else { - cnt := popcnt(c.bitmap[x]) + cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= X - c.n += int(popcnt(c.bitmap[x]) - cnt) + c.n += int(popcount(c.bitmap[x]) - cnt) for i := x + 1; i < y; i++ { - cnt = popcnt(c.bitmap[i]) + cnt = popcount(c.bitmap[i]) c.bitmap[i] ^= maxBitmap - c.n += int(popcnt(c.bitmap[i]) - cnt) + c.n += int(popcount(c.bitmap[i]) - cnt) } - cnt = popcnt(c.bitmap[y]) + cnt = popcount(c.bitmap[y]) c.bitmap[y] ^= Y - c.n += int(popcnt(c.bitmap[y]) - cnt) + c.n += int(popcount(c.bitmap[y]) - cnt) } } @@ -2342,16 +2343,16 @@ func (c *container) bitmapZeroRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { - c.n -= int(popcnt(c.bitmap[x] & (X & Y))) + c.n -= int(popcount(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) } else { - c.n -= int(popcnt(c.bitmap[x] & X)) + c.n -= int(popcount(c.bitmap[x] & X)) c.bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int(popcnt(c.bitmap[i])) + c.n -= int(popcount(c.bitmap[i])) c.bitmap[i] = 0 } - c.n -= int(popcnt(c.bitmap[y] & Y)) + c.n -= int(popcount(c.bitmap[y] & Y)) c.bitmap[y] &= ^Y } } @@ -2376,7 +2377,7 @@ func unionBitmapBitmap(a, b *container) *container { for i := 0; i < bitmapN; i++ { v := a.bitmap[i] | b.bitmap[i] output.bitmap[i] = v - output.n += int(popcnt(v)) + output.n += int(popcount(v)) } return output @@ -2812,7 +2813,7 @@ func xorBitmapBitmap(a, b *container) *container { for i := 0; i < bitmapN; i++ { v := a.bitmap[i] ^ b.bitmap[i] output.bitmap[i] = v - output.n += int(popcnt(v)) + output.n += int(popcount(v)) } if output.count() < ArrayMaxSize { @@ -2975,35 +2976,7 @@ func search64(a []uint64, value uint64) int { // trailingZeroN returns the number of trailing zeros in v. // v must be greater than zero. func trailingZeroN(v uint64) int { - n := int64(63) - if y := v << 32; y != 0 { - n, v = n-32, y - } - if y := v << 16; y != 0 { - n, v = n-16, y - } - if y := v << 8; y != 0 { - n, v = n-8, y - } - if y := v << 4; y != 0 { - n, v = n-4, y - } - if y := v << 2; y != 0 { - n, v = n-2, y - } - return int(n - int64(v<<1>>63)) -} - -// bit population count, taken from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -func popcount(x uint64) (n uint64) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 + return bits.TrailingZeros64(v) } // ErrorList represents a list of errors. @@ -3265,3 +3238,47 @@ func xorBitmapRun(a, b *container) *container { } return output } + +func popcount(x uint64) uint64 { + return uint64(bits.OnesCount64(x)) +} + +func popcountSlice(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcountMaskSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcountAndSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcountOrSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcountXorSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} From e01edb7e859d3ce988470c7d824159af8eb5b23c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 29 Aug 2017 15:38:49 -0500 Subject: [PATCH 098/311] removed 1.8 from support; bits require 1.9 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ddff4f4b..16acb0803 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.8 - 1.9 - master env: From 82592adf64d141cbc73048a9151a20c6a7905849 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 20 Feb 2018 11:50:06 -0600 Subject: [PATCH 099/311] add MustRunMainWithCluster test func to have api compatibility with cluster resize this will allow some pdk code which only works with cluster-resize to work against master (i think) --- test/pilosa.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/pilosa.go b/test/pilosa.go index e14b2d032..97c4321c9 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -101,6 +101,27 @@ func MustNewServerCluster(t *testing.T, size int) *Cluster { return cluster } +// **** below exists to have interface compatibility with cluster-resize, +// **** we can remove when cluster resize gets merged *****************// + +type M struct { + *server.Command + Stdin bytes.Buffer + Stdout bytes.Buffer + Stderr bytes.Buffer +} + +func MustRunMainWithCluster(t *testing.T, size int) []*M { + cluster := MustNewServerCluster(t, size) + mains := make([]*M, 0) + for _, s := range cluster.Servers { + mains = append(mains, &M{Command: s}) + } + return mains +} + +// ***********************************************************************************// + func NewServerCluster(size int) (cluster *Cluster, err error) { cluster = &Cluster{ Servers: make([]*server.Command, size), From 0417439afaf2f5d3554dd318f027bc804afb750c Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 21 Feb 2018 18:03:05 +0300 Subject: [PATCH 100/311] Validate all relevant endpoints; Changed how validation key is composed --- handler.go | 46 +++++++++++++++++++++++++++++----------------- handler_test.go | 2 +- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/handler.go b/handler.go index 895141135..024c222a7 100644 --- a/handler.go +++ b/handler.go @@ -89,6 +89,10 @@ var externalPrefixFlag = map[string]bool{ "version": true, } +type errorResponse struct { + Error string `json:"error"` +} + // NewHandler returns a new instance of Handler with a default logger. func NewHandler() *Handler { handler := &Handler{ @@ -101,17 +105,31 @@ func NewHandler() *Handler { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GET /fragment/nodes"] = QueryValidationSpecRequired("slice").Optional("index") + h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index") + h.validators["GetSliceMax"] = QueryValidationSpecRequired().Optional("inverse") + h.validators["PostQuery"] = QueryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") + h.validators["GetExport"] = QueryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentBlocks"] = QueryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFrameRestore"] = QueryValidationSpecRequired("host") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - key := fmt.Sprintf("%s %s", r.Method, r.URL.Path) + key := mux.CurrentRoute(r).GetName() if validator, ok := h.validators[key]; ok { q := r.URL.Query() err := validator.validate(q) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + // TODO: Return the response depending on the Accept header + response := errorResponse{Error: err.Error()} + body, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, string(body), http.StatusBadRequest) return } } @@ -126,12 +144,12 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") - router.HandleFunc("/export", handler.handleGetExport).Methods("GET") + router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET") - router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET") - router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET") + router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") + router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData") + router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") @@ -143,7 +161,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore") router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") @@ -154,11 +172,11 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") + router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET").Name("GetSliceMax") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -1097,12 +1115,6 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro // readURLQueryRequest parses query parameters from URL parameters from r. func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { q := r.URL.Query() - validQuery := validOptions(QueryRequest{}) - for key := range q { - if _, ok := validQuery[key]; !ok { - return nil, errors.New("invalid query params") - } - } // Parse query string. buf, err := ioutil.ReadAll(r.Body) diff --git a/handler_test.go b/handler_test.go index 636665022..f1c533691 100644 --- a/handler_test.go +++ b/handler_test.go @@ -306,7 +306,7 @@ func TestHandler_Query_Params_Err(t *testing.T) { test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) } From 3e770a5eca9cd6794d40701e76b6f1bb491481c1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 21 Feb 2018 13:47:55 -0600 Subject: [PATCH 101/311] remove cluster type http from docs --- docs/configuration.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2385f2494..33a78018e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -183,9 +183,8 @@ Any flag that has a value that is a comma separated list on the command line bec #### Cluster Type -* Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]. +* Description: Determine how the cluster handles membership and state sharing. Choose from [static, gossip]. * static - Messaging between nodes is disabled. This is primarily used for testing. - * http - Messages are transmitted over HTTP. * gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip. * Flag: `cluster.type="gossip"` * Env: `PILOSA_CLUSTER_TYPE="gossip"` From da125e323c630b1fa2679fca36e02b24d2a5ee5e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 21 Feb 2018 14:56:38 -0600 Subject: [PATCH 102/311] broadcast.SendSync field creation and deletion to all nodes --- broadcast.go | 10 + handler.go | 31 +- internal/private.pb.go | 634 +++++++++++++++++++++++++++++++++++------ internal/private.proto | 12 + server.go | 11 + 5 files changed, 607 insertions(+), 91 deletions(-) diff --git a/broadcast.go b/broadcast.go index 38eb53b96..bc3593a2c 100644 --- a/broadcast.go +++ b/broadcast.go @@ -132,6 +132,8 @@ const ( MessageTypeCreateInputDefinition = 6 MessageTypeDeleteInputDefinition = 7 MessageTypeDeleteView = 8 + MessageTypeCreateField = 9 + MessageTypeDeleteField = 10 ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -148,6 +150,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateFrame case *internal.DeleteFrameMessage: typ = MessageTypeDeleteFrame + case *internal.CreateFieldMessage: + typ = MessageTypeCreateField + case *internal.DeleteFieldMessage: + typ = MessageTypeDeleteField case *internal.CreateInputDefinitionMessage: typ = MessageTypeCreateInputDefinition case *internal.DeleteInputDefinitionMessage: @@ -180,6 +186,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateFrameMessage{} case MessageTypeDeleteFrame: m = &internal.DeleteFrameMessage{} + case MessageTypeCreateField: + m = &internal.CreateFieldMessage{} + case MessageTypeDeleteField: + m = &internal.DeleteFieldMessage{} case MessageTypeCreateInputDefinition: m = &internal.CreateInputDefinitionMessage{} case MessageTypeDeleteInputDefinition: diff --git a/handler.go b/handler.go index e5cce4648..1a67d79be 100644 --- a/handler.go +++ b/handler.go @@ -648,7 +648,6 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) - } type _postFrameRequest postFrameRequest @@ -800,17 +799,30 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { return } - // Create new field. - if err := f.CreateField(&Field{ + field := &Field{ Name: fieldName, Type: req.Type, Min: req.Min, Max: req.Max, - }); err != nil { + } + + // Create new field. + if err := f.CreateField(field); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } + // Send the create field message to all nodes. + err := h.Broadcaster.SendSync( + &internal.CreateFieldMessage{ + Index: indexName, + Frame: frameName, + Field: encodeField(field), + }) + if err != nil { + h.logger().Printf("problem sending CreateField message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) @@ -844,6 +856,17 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) return } + // Send the delete field message to all nodes. + err := h.Broadcaster.SendSync( + &internal.DeleteFieldMessage{ + Index: indexName, + Frame: frameName, + Field: fieldName, + }) + if err != nil { + h.logger().Printf("problem sending DeleteField message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/internal/private.pb.go b/internal/private.pb.go index 1c26aa307..63f2ca581 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -21,6 +21,8 @@ CreateIndexMessage CreateFrameMessage DeleteFrameMessage + CreateFieldMessage + DeleteFieldMessage Frame Index InputDefinition @@ -389,6 +391,70 @@ func (m *DeleteFrameMessage) GetFrame() string { return "" } +type CreateFieldMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field *Field `protobuf:"bytes,3,opt,name=Field" json:"Field,omitempty"` +} + +func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } +func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFieldMessage) ProtoMessage() {} +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } + +func (m *CreateFieldMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateFieldMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *CreateFieldMessage) GetField() *Field { + if m != nil { + return m.Field + } + return nil +} + +type DeleteFieldMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` +} + +func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } +func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFieldMessage) ProtoMessage() {} +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } + +func (m *DeleteFieldMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteFieldMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *DeleteFieldMessage) GetField() string { + if m != nil { + return m.Field + } + return "" +} + type Frame struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` @@ -397,7 +463,7 @@ type Frame struct { func (m *Frame) Reset() { *m = Frame{} } func (m *Frame) String() string { return proto.CompactTextString(m) } func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Frame) GetName() string { if m != nil { @@ -425,7 +491,7 @@ type Index struct { func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *Index) GetName() string { if m != nil { @@ -478,7 +544,7 @@ type InputDefinition struct { func (m *InputDefinition) Reset() { *m = InputDefinition{} } func (m *InputDefinition) String() string { return proto.CompactTextString(m) } func (*InputDefinition) ProtoMessage() {} -func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *InputDefinition) GetName() string { if m != nil { @@ -510,7 +576,7 @@ type InputDefinitionField struct { func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } func (m *InputDefinitionField) String() string { return proto.CompactTextString(m) } func (*InputDefinitionField) ProtoMessage() {} -func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *InputDefinitionField) GetName() string { if m != nil { @@ -543,7 +609,7 @@ type InputDefinitionAction struct { func (m *InputDefinitionAction) Reset() { *m = InputDefinitionAction{} } func (m *InputDefinitionAction) String() string { return proto.CompactTextString(m) } func (*InputDefinitionAction) ProtoMessage() {} -func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *InputDefinitionAction) GetFrame() string { if m != nil { @@ -582,7 +648,7 @@ func (m *CreateInputDefinitionMessage) Reset() { *m = CreateInputDefinit func (m *CreateInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*CreateInputDefinitionMessage) ProtoMessage() {} func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{17} + return fileDescriptorPrivate, []int{19} } func (m *CreateInputDefinitionMessage) GetIndex() string { @@ -608,7 +674,7 @@ func (m *DeleteInputDefinitionMessage) Reset() { *m = DeleteInputDefinit func (m *DeleteInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*DeleteInputDefinitionMessage) ProtoMessage() {} func (*DeleteInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{18} + return fileDescriptorPrivate, []int{20} } func (m *DeleteInputDefinitionMessage) GetIndex() string { @@ -635,7 +701,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } func (m *NodeStatus) GetHost() string { if m != nil { @@ -672,7 +738,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *ClusterStatus) GetNodes() []*NodeStatus { if m != nil { @@ -688,7 +754,7 @@ type FrameSchema struct { func (m *FrameSchema) Reset() { *m = FrameSchema{} } func (m *FrameSchema) String() string { return proto.CompactTextString(m) } func (*FrameSchema) ProtoMessage() {} -func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *FrameSchema) GetFields() []*Field { if m != nil { @@ -707,7 +773,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *Field) GetName() string { if m != nil { @@ -746,7 +812,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -782,6 +848,8 @@ func init() { proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") + proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") + proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") @@ -1256,6 +1324,82 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Field != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Field.Size())) + n9, err := m.Field.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 + } + return i, nil +} + +func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.Field) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + return i, nil +} + func (m *Frame) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1281,11 +1425,11 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n9, err := m.Meta.MarshalTo(dAtA[i:]) + n10, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 } return i, nil } @@ -1315,11 +1459,11 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n10, err := m.Meta.MarshalTo(dAtA[i:]) + n11, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 } if m.MaxSlice != 0 { dAtA[i] = 0x18 @@ -1339,21 +1483,21 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { } } if len(m.Slices) > 0 { - dAtA12 := make([]byte, len(m.Slices)*10) - var j11 int + dAtA13 := make([]byte, len(m.Slices)*10) + var j12 int for _, num := range m.Slices { for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j11++ + j12++ } - dAtA12[j11] = uint8(num) - j11++ + dAtA13[j12] = uint8(num) + j12++ } dAtA[i] = 0x2a i++ - i = encodeVarintPrivate(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) + i = encodeVarintPrivate(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) } if len(m.InputDefinitions) > 0 { for _, msg := range m.InputDefinitions { @@ -1540,11 +1684,11 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Definition.Size())) - n13, err := m.Definition.MarshalTo(dAtA[i:]) + n14, err := m.Definition.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 } return i, nil } @@ -1990,6 +2134,42 @@ func (m *DeleteFrameMessage) Size() (n int) { return n } +func (m *CreateFieldMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Field != nil { + l = m.Field.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *DeleteFieldMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *Frame) Size() (n int) { var l int _ = l @@ -3822,6 +4002,284 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateFieldMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Field == nil { + m.Field = &Field{} + } + if err := m.Field.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteFieldMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Frame) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5634,65 +6092,67 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 948 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, - 0x10, 0x65, 0x3c, 0x63, 0xaf, 0x5d, 0x26, 0x1b, 0xa7, 0x09, 0x2b, 0x6f, 0x14, 0x19, 0xab, 0x0f, - 0x6c, 0x88, 0x44, 0x0e, 0x41, 0x5a, 0x01, 0xcb, 0x01, 0x36, 0xce, 0x2a, 0x16, 0x78, 0x81, 0xf6, - 0x6a, 0xb9, 0x21, 0x75, 0x9c, 0x62, 0x77, 0x94, 0xf1, 0x8c, 0x99, 0x69, 0x27, 0x31, 0x07, 0x8e, - 0x7c, 0x03, 0x12, 0x47, 0x7e, 0x86, 0x23, 0x9f, 0x80, 0xc2, 0x85, 0x3f, 0x40, 0xe2, 0x84, 0xba, - 0xba, 0x7b, 0x66, 0x6c, 0xc7, 0x8e, 0xb2, 0xb7, 0xae, 0x57, 0xd5, 0x55, 0xaf, 0xab, 0xab, 0xaa, - 0x1b, 0x36, 0x26, 0x69, 0x78, 0x21, 0x15, 0x1e, 0x4c, 0xd2, 0x44, 0x25, 0xac, 0x1e, 0xc6, 0x0a, - 0xd3, 0x58, 0x46, 0xfc, 0x6b, 0x68, 0xf4, 0xe3, 0x33, 0xbc, 0x1a, 0xa0, 0x92, 0xac, 0x0b, 0xcd, - 0xa3, 0x24, 0x9a, 0x8e, 0xe3, 0xaf, 0xe4, 0x29, 0x46, 0x6d, 0xaf, 0xeb, 0xed, 0x35, 0x44, 0x19, - 0xd2, 0x16, 0x2f, 0xc2, 0x31, 0x7e, 0x3b, 0x95, 0xb1, 0x9a, 0x8e, 0xdb, 0x15, 0x63, 0x51, 0x82, - 0xf8, 0x7f, 0x1e, 0x34, 0x9e, 0xa5, 0x72, 0x8c, 0xe4, 0x71, 0x07, 0xea, 0x22, 0xb9, 0x2c, 0xbb, - 0xcb, 0x65, 0xf6, 0x3e, 0xdc, 0xef, 0xc7, 0x17, 0x98, 0x66, 0x78, 0x1c, 0xcb, 0xd3, 0x08, 0xcf, - 0xc8, 0x5d, 0x5d, 0x2c, 0xa0, 0x6c, 0x17, 0x1a, 0x47, 0x72, 0xf4, 0x1a, 0x5f, 0xcc, 0x26, 0xd8, - 0xf6, 0xc9, 0x49, 0x01, 0xe4, 0xda, 0x61, 0xf8, 0x13, 0xb6, 0x83, 0xae, 0xb7, 0xb7, 0x21, 0x0a, - 0x60, 0x91, 0x6f, 0x75, 0x89, 0x2f, 0xe3, 0xf0, 0xb6, 0x90, 0xf1, 0xab, 0x9c, 0x43, 0x8d, 0x38, - 0xcc, 0x61, 0xec, 0x11, 0xd4, 0x9e, 0x85, 0x18, 0x9d, 0x65, 0xed, 0x7b, 0x5d, 0x7f, 0xaf, 0x79, - 0xb8, 0x79, 0xe0, 0xf2, 0x77, 0x40, 0xb8, 0xb0, 0x6a, 0xce, 0xe1, 0x7e, 0x7f, 0x3c, 0x49, 0x52, - 0x25, 0x30, 0x9b, 0x24, 0x71, 0x86, 0xac, 0x05, 0xfe, 0x71, 0x9a, 0xda, 0xb3, 0xeb, 0x25, 0xff, - 0x19, 0x5a, 0x4f, 0xa3, 0x64, 0x74, 0xde, 0x93, 0x4a, 0x0a, 0xfc, 0x71, 0x8a, 0x99, 0x62, 0xdb, - 0x50, 0xa5, 0x5b, 0xb0, 0x76, 0x46, 0xd0, 0x28, 0x65, 0xd2, 0xa6, 0xd9, 0x08, 0x1a, 0xa5, 0xfd, - 0x94, 0x8a, 0x40, 0x18, 0x41, 0xa3, 0xc3, 0x28, 0x1c, 0x99, 0x14, 0x04, 0xc2, 0x08, 0x8c, 0x41, - 0xf0, 0x32, 0xc4, 0x4b, 0x7b, 0x6e, 0x5a, 0xf3, 0x3e, 0x6c, 0x95, 0xe2, 0x5b, 0x9a, 0x0f, 0xa0, - 0x26, 0x92, 0xcb, 0x7e, 0x2f, 0x6b, 0x7b, 0x5d, 0x7f, 0x2f, 0x10, 0x56, 0xa2, 0xec, 0xd2, 0xf5, - 0x6b, 0x55, 0x85, 0x54, 0x05, 0xc0, 0x1f, 0x42, 0x95, 0x52, 0xad, 0x4f, 0x59, 0xec, 0xd5, 0x4b, - 0xfe, 0x9b, 0x07, 0x5b, 0x03, 0x79, 0x45, 0x34, 0xb2, 0x3c, 0xcc, 0x09, 0x34, 0x72, 0x90, 0xac, - 0x9b, 0x87, 0xfb, 0x45, 0x2e, 0x97, 0xec, 0x0b, 0xe4, 0x38, 0x56, 0xe9, 0x4c, 0x14, 0x9b, 0x77, - 0x3e, 0x83, 0xfb, 0xf3, 0x4a, 0xcd, 0xe1, 0x1c, 0x67, 0x2e, 0xd3, 0xe7, 0x38, 0xd3, 0x39, 0xb9, - 0x90, 0xd1, 0xd4, 0xe4, 0x2f, 0x10, 0x46, 0xf8, 0xb4, 0xf2, 0xb1, 0xc7, 0xbf, 0x07, 0x76, 0x94, - 0xa2, 0x54, 0x48, 0x0e, 0x06, 0x98, 0x65, 0xf2, 0x15, 0xae, 0xbe, 0x05, 0x93, 0xd9, 0x4a, 0x39, - 0xb3, 0xbb, 0xd0, 0xe8, 0x67, 0xb6, 0x50, 0xe9, 0x26, 0xea, 0xa2, 0x00, 0xf8, 0x3e, 0xb0, 0x1e, - 0x46, 0xa8, 0xd0, 0xf6, 0xd6, 0x1a, 0xff, 0x7c, 0xe8, 0xb8, 0xdc, 0x6e, 0xcb, 0x1e, 0x41, 0xa0, - 0xdb, 0x8a, 0xa8, 0x34, 0x0f, 0xdf, 0x29, 0x52, 0x97, 0xf7, 0xb0, 0x20, 0x03, 0x1e, 0x3a, 0xa7, - 0xb6, 0x15, 0x6f, 0x39, 0xe0, 0x0d, 0x65, 0xe6, 0x42, 0xf9, 0x8b, 0xa1, 0xf2, 0xe6, 0xb6, 0xa1, - 0x3e, 0x77, 0x67, 0x7d, 0xd3, 0x50, 0xbc, 0x67, 0x51, 0x5d, 0xae, 0xcf, 0xb5, 0xd6, 0xec, 0xa1, - 0xf5, 0xea, 0x23, 0x2f, 0xf2, 0xf8, 0xc7, 0xb3, 0x21, 0xef, 0xe6, 0x66, 0x21, 0x73, 0x7a, 0x62, - 0xb9, 0xc2, 0xb2, 0x1d, 0x96, 0xcb, 0x34, 0x07, 0x74, 0xd4, 0xac, 0x1d, 0x2c, 0xcd, 0x01, 0x8d, - 0x0b, 0xab, 0xd6, 0xed, 0x64, 0x8b, 0xbc, 0x6a, 0xda, 0xc9, 0x48, 0xec, 0x18, 0x5a, 0xfd, 0x78, - 0x32, 0x55, 0x3d, 0xfc, 0x21, 0x8c, 0x43, 0x15, 0x26, 0x71, 0xd6, 0xae, 0x91, 0xab, 0x87, 0x65, - 0x46, 0x73, 0x16, 0x62, 0x69, 0x0b, 0xff, 0xc5, 0x83, 0xcd, 0x05, 0x70, 0xc5, 0xa1, 0x1d, 0xdf, - 0xca, 0x7a, 0xbe, 0x8f, 0xf3, 0x01, 0xe7, 0x93, 0x61, 0x67, 0x25, 0x9b, 0xf9, 0x79, 0xf7, 0xbb, - 0x07, 0xdb, 0x37, 0x19, 0xdc, 0xc8, 0xa6, 0x03, 0xf0, 0x4d, 0x1a, 0x8e, 0x65, 0x3a, 0xfb, 0x12, - 0x67, 0x76, 0xd6, 0x97, 0x10, 0xf6, 0x1d, 0x3c, 0x58, 0xf0, 0xf5, 0xc5, 0xc8, 0xa4, 0xc8, 0x90, - 0x7a, 0x6f, 0x25, 0x29, 0x63, 0x27, 0x56, 0x6c, 0xe7, 0xff, 0x7a, 0xf0, 0xee, 0x8d, 0xaa, 0xa2, - 0x1e, 0xbd, 0x72, 0xe9, 0xef, 0x43, 0xeb, 0xa5, 0x1e, 0x15, 0x3d, 0xcc, 0x54, 0x18, 0x4b, 0x6d, - 0x69, 0x0b, 0x76, 0x09, 0x67, 0x7d, 0xa8, 0x13, 0x36, 0x90, 0x13, 0x4b, 0xf3, 0xc3, 0x5b, 0x68, - 0x1e, 0x38, 0x7b, 0x33, 0xd3, 0xf2, 0xed, 0x9a, 0x0c, 0x4d, 0x5d, 0x37, 0xc2, 0x49, 0xd8, 0x79, - 0x02, 0x1b, 0x73, 0x1b, 0xee, 0x34, 0xe7, 0x12, 0xd8, 0x75, 0xb3, 0x65, 0x8e, 0xc9, 0xfa, 0x2e, - 0xfd, 0x04, 0xa0, 0x30, 0xb5, 0x03, 0x60, 0x4d, 0x7d, 0x96, 0x8c, 0xf9, 0x09, 0xec, 0xba, 0xc1, - 0x77, 0x87, 0x80, 0xae, 0x5a, 0x2a, 0x45, 0xb5, 0xf0, 0x19, 0xc0, 0xf3, 0xe4, 0x0c, 0x87, 0x4a, - 0xaa, 0x69, 0xa6, 0x2d, 0x4e, 0x92, 0x4c, 0xb9, 0x7a, 0xd2, 0x6b, 0x1a, 0xcc, 0x4a, 0xaa, 0x7c, - 0x98, 0x90, 0xc0, 0x3e, 0x80, 0x7b, 0xe4, 0x14, 0x5d, 0xd9, 0x6c, 0x2e, 0xf4, 0xba, 0x70, 0x7a, - 0xea, 0xd2, 0xd1, 0x6b, 0x1c, 0x9b, 0x47, 0xb3, 0x21, 0xac, 0xc4, 0x9f, 0xc0, 0xc6, 0x51, 0x34, - 0xcd, 0x14, 0xa6, 0x36, 0xfa, 0x3e, 0x54, 0x35, 0x17, 0xf7, 0x64, 0x6d, 0x17, 0x1e, 0x0b, 0x8a, - 0xc2, 0x98, 0xf0, 0xc7, 0xd0, 0xa4, 0x2a, 0x22, 0x5f, 0xb2, 0xf4, 0x75, 0xf0, 0xd6, 0x7f, 0x1d, - 0x86, 0x50, 0x5d, 0xdd, 0x3a, 0x0c, 0x02, 0xfa, 0xfd, 0xd8, 0x04, 0xd1, 0xc7, 0xa7, 0x05, 0xfe, - 0x20, 0x34, 0xd7, 0xe3, 0x0b, 0xbd, 0x24, 0x44, 0x5e, 0xd1, 0x61, 0x34, 0x22, 0xf5, 0xdb, 0xb2, - 0x65, 0xae, 0x43, 0xbf, 0xfc, 0x6f, 0xf2, 0x0a, 0xb8, 0x0f, 0x84, 0x5f, 0x7c, 0x20, 0x9e, 0xb6, - 0xfe, 0xb8, 0xee, 0x78, 0x7f, 0x5e, 0x77, 0xbc, 0xbf, 0xae, 0x3b, 0xde, 0xaf, 0x7f, 0x77, 0xde, - 0x3a, 0xad, 0xd1, 0xaf, 0xf2, 0xa3, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x47, 0xdd, 0xdd, 0x8e, - 0x66, 0x0a, 0x00, 0x00, + // 978 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0x66, 0xfc, 0xb7, 0x76, 0x99, 0x24, 0x4e, 0x13, 0x56, 0xde, 0x28, 0x32, 0x56, 0x4b, 0xb0, + 0x21, 0x12, 0x39, 0x04, 0x69, 0x05, 0x2c, 0x07, 0xd8, 0x38, 0xab, 0x58, 0xe0, 0x05, 0xda, 0xab, + 0x70, 0x43, 0xea, 0x38, 0x45, 0x32, 0xca, 0x78, 0xc6, 0xcc, 0xb4, 0x93, 0x98, 0x03, 0x47, 0x9e, + 0x01, 0x89, 0x23, 0x2f, 0xc3, 0x91, 0x47, 0x40, 0xe1, 0xc2, 0x1b, 0x20, 0x71, 0x5a, 0x75, 0x75, + 0xf7, 0xcc, 0xf8, 0x37, 0x4a, 0x6e, 0x5d, 0x5f, 0xd7, 0xcf, 0xd7, 0xd5, 0x55, 0xd5, 0x0d, 0x6b, + 0xa3, 0xd8, 0xbf, 0x92, 0x0a, 0xf7, 0x47, 0x71, 0xa4, 0x22, 0x56, 0xf5, 0x43, 0x85, 0x71, 0x28, + 0x03, 0xfe, 0x0d, 0xd4, 0xba, 0xe1, 0x19, 0xde, 0xf4, 0x50, 0x49, 0xd6, 0x86, 0xfa, 0x61, 0x14, + 0x8c, 0x87, 0xe1, 0xd7, 0xf2, 0x14, 0x83, 0xa6, 0xd7, 0xf6, 0x76, 0x6b, 0x22, 0x0f, 0x69, 0x8d, + 0xd7, 0xfe, 0x10, 0xbf, 0x1b, 0xcb, 0x50, 0x8d, 0x87, 0xcd, 0x82, 0xd1, 0xc8, 0x41, 0xfc, 0x7f, + 0x0f, 0x6a, 0x2f, 0x63, 0x39, 0x44, 0xf2, 0xb8, 0x0d, 0x55, 0x11, 0x5d, 0xe7, 0xdd, 0xa5, 0x32, + 0xfb, 0x00, 0xd6, 0xbb, 0xe1, 0x15, 0xc6, 0x09, 0x1e, 0x85, 0xf2, 0x34, 0xc0, 0x33, 0x72, 0x57, + 0x15, 0x33, 0x28, 0xdb, 0x81, 0xda, 0xa1, 0x1c, 0x5c, 0xe0, 0xeb, 0xc9, 0x08, 0x9b, 0x45, 0x72, + 0x92, 0x01, 0xe9, 0x6e, 0xdf, 0xff, 0x19, 0x9b, 0xa5, 0xb6, 0xb7, 0xbb, 0x26, 0x32, 0x60, 0x96, + 0x6f, 0x79, 0x8e, 0x2f, 0xe3, 0xf0, 0xb6, 0x90, 0xe1, 0x79, 0xca, 0xa1, 0x42, 0x1c, 0xa6, 0x30, + 0xf6, 0x14, 0x2a, 0x2f, 0x7d, 0x0c, 0xce, 0x92, 0xe6, 0xa3, 0x76, 0x71, 0xb7, 0x7e, 0xb0, 0xb1, + 0xef, 0xf2, 0xb7, 0x4f, 0xb8, 0xb0, 0xdb, 0x9c, 0xc3, 0x7a, 0x77, 0x38, 0x8a, 0x62, 0x25, 0x30, + 0x19, 0x45, 0x61, 0x82, 0xac, 0x01, 0xc5, 0xa3, 0x38, 0xb6, 0x67, 0xd7, 0x4b, 0xfe, 0x0b, 0x34, + 0x5e, 0x04, 0xd1, 0xe0, 0xb2, 0x23, 0x95, 0x14, 0xf8, 0xd3, 0x18, 0x13, 0xc5, 0xb6, 0xa0, 0x4c, + 0xb7, 0x60, 0xf5, 0x8c, 0xa0, 0x51, 0xca, 0xa4, 0x4d, 0xb3, 0x11, 0x34, 0x4a, 0xf6, 0x94, 0x8a, + 0x92, 0x30, 0x82, 0x46, 0xfb, 0x81, 0x3f, 0x30, 0x29, 0x28, 0x09, 0x23, 0x30, 0x06, 0xa5, 0x13, + 0x1f, 0xaf, 0xed, 0xb9, 0x69, 0xcd, 0xbb, 0xb0, 0x99, 0x8b, 0x6f, 0x69, 0x3e, 0x86, 0x8a, 0x88, + 0xae, 0xbb, 0x9d, 0xa4, 0xe9, 0xb5, 0x8b, 0xbb, 0x25, 0x61, 0x25, 0xca, 0x2e, 0x5d, 0xbf, 0xde, + 0x2a, 0xd0, 0x56, 0x06, 0xf0, 0x27, 0x50, 0xa6, 0x54, 0xeb, 0x53, 0x66, 0xb6, 0x7a, 0xc9, 0x7f, + 0xf7, 0x60, 0xb3, 0x27, 0x6f, 0x88, 0x46, 0x92, 0x86, 0x39, 0x86, 0x5a, 0x0a, 0x92, 0x76, 0xfd, + 0x60, 0x2f, 0xcb, 0xe5, 0x9c, 0x7e, 0x86, 0x1c, 0x85, 0x2a, 0x9e, 0x88, 0xcc, 0x78, 0xfb, 0x73, + 0x58, 0x9f, 0xde, 0xd4, 0x1c, 0x2e, 0x71, 0xe2, 0x32, 0x7d, 0x89, 0x13, 0x9d, 0x93, 0x2b, 0x19, + 0x8c, 0x4d, 0xfe, 0x4a, 0xc2, 0x08, 0x9f, 0x15, 0x3e, 0xf1, 0xf8, 0x0f, 0xc0, 0x0e, 0x63, 0x94, + 0x0a, 0xc9, 0x41, 0x0f, 0x93, 0x44, 0x9e, 0xe3, 0xf2, 0x5b, 0x30, 0x99, 0x2d, 0xe4, 0x33, 0xbb, + 0x03, 0xb5, 0x6e, 0x62, 0x0b, 0x95, 0x6e, 0xa2, 0x2a, 0x32, 0x80, 0xef, 0x01, 0xeb, 0x60, 0x80, + 0x0a, 0x6d, 0x6f, 0xad, 0xf0, 0xcf, 0xfb, 0x8e, 0xcb, 0xdd, 0xba, 0xec, 0x29, 0x94, 0x74, 0x5b, + 0x11, 0x95, 0xfa, 0xc1, 0x3b, 0x59, 0xea, 0xd2, 0x1e, 0x16, 0xa4, 0xc0, 0x7d, 0xe7, 0xd4, 0xb6, + 0xe2, 0x1d, 0x07, 0x5c, 0x50, 0x66, 0x2e, 0x54, 0x71, 0x36, 0x54, 0xda, 0xdc, 0x36, 0xd4, 0x17, + 0xee, 0xac, 0x0f, 0x0d, 0xc5, 0xcf, 0x53, 0xb2, 0xba, 0x8b, 0x1e, 0x42, 0xf6, 0x7d, 0x28, 0x93, + 0xad, 0x65, 0x3b, 0xd7, 0x9f, 0x66, 0x97, 0x9f, 0xa4, 0x54, 0x1f, 0x1a, 0x68, 0x2b, 0x1f, 0xa8, + 0xe6, 0xfc, 0x76, 0xac, 0xae, 0xee, 0xb7, 0x57, 0xda, 0xc6, 0x78, 0xa2, 0xf5, 0xf2, 0x3b, 0x9b, + 0x4d, 0xe4, 0xbf, 0x9e, 0x25, 0x72, 0x3f, 0x37, 0x33, 0x57, 0xaf, 0x47, 0xae, 0xeb, 0x0c, 0x3b, + 0x22, 0x52, 0x99, 0x06, 0x99, 0x8e, 0x9a, 0x34, 0x4b, 0x73, 0x83, 0x4c, 0xe3, 0xc2, 0x6e, 0xeb, + 0x79, 0x60, 0xbb, 0xb4, 0x6c, 0xe6, 0x81, 0x91, 0xd8, 0x11, 0x34, 0xba, 0xe1, 0x68, 0xac, 0x3a, + 0xf8, 0xa3, 0x1f, 0xfa, 0xca, 0x8f, 0xc2, 0xa4, 0x59, 0x21, 0x57, 0x4f, 0xf2, 0x8c, 0xa6, 0x34, + 0xc4, 0x9c, 0x09, 0xff, 0xd5, 0x83, 0x8d, 0x19, 0x70, 0xc9, 0xa1, 0x1d, 0xdf, 0xc2, 0x6a, 0xbe, + 0xcf, 0xd2, 0x09, 0x5d, 0x24, 0xc5, 0xd6, 0x52, 0x36, 0xd3, 0x03, 0xfb, 0x0f, 0x0f, 0xb6, 0x16, + 0x29, 0x2c, 0x64, 0xd3, 0x02, 0xf8, 0x36, 0xf6, 0x87, 0x32, 0x9e, 0x7c, 0x85, 0x13, 0xfb, 0x58, + 0xe5, 0x10, 0xf6, 0x3d, 0x3c, 0x9e, 0xf1, 0xf5, 0xe5, 0xc0, 0xa4, 0xc8, 0x90, 0x7a, 0x6f, 0x29, + 0x29, 0xa3, 0x27, 0x96, 0x98, 0xf3, 0xff, 0x3c, 0x78, 0x77, 0xe1, 0x56, 0x56, 0xa5, 0x5e, 0xbe, + 0x4a, 0xf7, 0xa0, 0x71, 0xa2, 0x67, 0x5d, 0x07, 0x13, 0xe5, 0x87, 0x52, 0x6b, 0xda, 0x32, 0x9e, + 0xc3, 0x59, 0x17, 0xaa, 0x84, 0xf5, 0xe4, 0xc8, 0xd2, 0xfc, 0xe8, 0x0e, 0x9a, 0xfb, 0x4e, 0xdf, + 0x0c, 0xe5, 0xd4, 0x5c, 0x93, 0xa1, 0x67, 0xc3, 0xbd, 0x41, 0x24, 0x6c, 0x3f, 0x87, 0xb5, 0x29, + 0x83, 0x7b, 0x0d, 0xea, 0x08, 0x76, 0xdc, 0x70, 0x9c, 0x62, 0xb2, 0xba, 0x77, 0x3f, 0x05, 0xc8, + 0x54, 0xed, 0x4c, 0x58, 0x51, 0x9f, 0x39, 0x65, 0x7e, 0x0c, 0x3b, 0x6e, 0x72, 0xdf, 0x23, 0xa0, + 0xab, 0x96, 0x42, 0x56, 0x2d, 0x7c, 0x02, 0xf0, 0x2a, 0x3a, 0xc3, 0xbe, 0x92, 0x6a, 0x9c, 0x68, + 0x8d, 0xe3, 0x28, 0x51, 0xae, 0x9e, 0xf4, 0x9a, 0x5e, 0x16, 0x25, 0x55, 0x3a, 0x62, 0x48, 0x60, + 0x1f, 0xc2, 0x23, 0x72, 0x8a, 0xae, 0x6c, 0x36, 0x66, 0x7a, 0x5d, 0xb8, 0x7d, 0xea, 0xd2, 0xc1, + 0x05, 0x0e, 0xcd, 0xab, 0x5f, 0x13, 0x56, 0xe2, 0xcf, 0x61, 0xed, 0x30, 0x18, 0x27, 0x0a, 0x63, + 0x1b, 0x7d, 0x0f, 0xca, 0x9a, 0x8b, 0x7b, 0x73, 0xb7, 0x32, 0x8f, 0x19, 0x45, 0x61, 0x54, 0xf8, + 0x33, 0xa8, 0x53, 0x15, 0x91, 0x2f, 0x99, 0xfb, 0xfb, 0x78, 0xab, 0xff, 0x3e, 0x7d, 0x3b, 0x1a, + 0x17, 0xb6, 0x0e, 0x83, 0x12, 0x7d, 0xdf, 0x6c, 0x82, 0xe8, 0xe7, 0xd6, 0x80, 0x62, 0xcf, 0x37, + 0xd7, 0x53, 0x14, 0x7a, 0x49, 0x88, 0xbc, 0xa1, 0xc3, 0x68, 0x44, 0xea, 0xc7, 0x71, 0xd3, 0x5c, + 0x87, 0xfe, 0xba, 0x3c, 0x64, 0x60, 0xbb, 0x1f, 0x50, 0x31, 0xfb, 0x01, 0xbd, 0x68, 0xfc, 0x79, + 0xdb, 0xf2, 0xfe, 0xba, 0x6d, 0x79, 0x7f, 0xdf, 0xb6, 0xbc, 0xdf, 0xfe, 0x69, 0xbd, 0x75, 0x5a, + 0xa1, 0x6f, 0xf1, 0xc7, 0x6f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x93, 0xbf, 0xbc, 0x1f, 0x27, 0x0b, + 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 083316ddd..ed664115a 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -68,6 +68,18 @@ message DeleteFrameMessage { string Frame = 2; } +message CreateFieldMessage { + string Index = 1; + string Frame = 2; + Field Field = 3; +} + +message DeleteFieldMessage { + string Index = 1; + string Frame = 2; + string Field = 3; +} + message Frame { string Name = 1; FrameMeta Meta = 2; diff --git a/server.go b/server.go index 83d417abd..95dff018d 100644 --- a/server.go +++ b/server.go @@ -404,6 +404,17 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err := idx.DeleteFrame(obj.Frame); err != nil { return err } + case *internal.CreateFieldMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + field := decodeField(obj.Field) + if err := f.CreateField(field); err != nil { + return err + } + case *internal.DeleteFieldMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + if err := f.DeleteField(obj.Field); err != nil { + return err + } case *internal.CreateInputDefinitionMessage: idx := s.Holder.Index(obj.Index) if idx == nil { From 73904db391e548b72cb9277eb2708b4cc94817c4 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 21 Feb 2018 15:42:35 -0600 Subject: [PATCH 103/311] Fix indentation --- docs/administration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index b77d05262..97ee688f5 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -124,7 +124,7 @@ Note: This will only work when the replication factor is >= 2 - Restart the cluster - Wait for the 1st sync (10 minutes) to validate Index connections -#### Diagnostics +### Diagnostics Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. @@ -145,7 +145,7 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. -#### Metrics +### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: @@ -154,7 +154,7 @@ The metrics configuration options are: - [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics - [Service](../configuration#metrics-service): declare type StatsD or Expvar -##### Tags +#### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - NodeID @@ -163,7 +163,7 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - View - Slice -##### Events +#### Events We currently track the following events - **Index:** The creation of a new Index. From 59d41a9cb3b3b938aef8ce18f8adc666fca49dc5 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 21 Feb 2018 15:42:46 -0600 Subject: [PATCH 104/311] Add Xor to docs --- docs/query-language.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/query-language.md b/docs/query-language.md index 0e02351ab..b1315d293 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -52,7 +52,7 @@ curl localhost:10101/index/repository/query \ * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. -* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range` +* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations @@ -308,6 +308,28 @@ Return `{"attrs":{},"bits":[30]}` * Bits are repositories that were starred by user 2 BUT NOT user 1 +#### Xor + +**Description:** + +Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it. + +**Result Type:** object with attrs and bits + +attrs will always be empty + +**Examples:** + +Query repositories which have been starred by two users. + +``` +Xor(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2)) +``` + +Returns `{"attrs":{},"bits":[30]}`. + +* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) + #### Count **Spec:** From a433a5ebfc3aea405b97170969adf8b7d53c3319 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 21 Feb 2018 16:42:35 -0600 Subject: [PATCH 105/311] remove column label repo_id from query language docs column labels are deprecated and the docs' usage of columnID vs repo_id was inconsisent. --- docs/query-language.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 0e02351ab..ee3d49a96 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -80,14 +80,14 @@ A return value of `false` indicates that the bit was already set to 1 and nothin **Examples:** ``` -SetBit(frame="stargazer", repo_id=10, rowID=1) +SetBit(frame="stargazer", columnID=10, rowID=1) ``` This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10. SetBit also supports providing a timestamp. To write the date that a user starred a repository. ``` -SetBit(frame="stargazer", repo_id=10, rowID=1, timestamp="2016-01-01T00:00") +SetBit(frame="stargazer", columnID=10, rowID=1, timestamp="2016-01-01T00:00") ``` Setting multiple bits in a single request: @@ -150,7 +150,7 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` -Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", repo_id=10)`. +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", columnID=10)`. ``` SetColumnAttrs(columnID=10, url=null) @@ -184,7 +184,7 @@ A return value of `false` indicates that the bit was already set to 0 and nothin ClearBit(frame="stargazer", columnID=10, rowID=1) ``` -Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame. +Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame. ### Read Operations From 25e244ebdfd90b434fa04eec894e7164b6c60267 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 22 Feb 2018 17:33:41 +0300 Subject: [PATCH 106/311] Updates --- handler.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/handler.go b/handler.go index 024c222a7..34075b3d4 100644 --- a/handler.go +++ b/handler.go @@ -119,9 +119,7 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { key := mux.CurrentRoute(r).GetName() if validator, ok := h.validators[key]; ok { - q := r.URL.Query() - err := validator.validate(q) - if err != nil { + if err := validator.validate(r.URL.Query()); err != nil { // TODO: Return the response depending on the Accept header response := errorResponse{Error: err.Error()} body, err := json.Marshal(response) @@ -2099,13 +2097,13 @@ type defaultClusterMessageResponse struct{} type queryValidationSpec struct { required []string - args map[string]bool + args map[string]struct{} } func QueryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { - args := map[string]bool{} + args := map[string]struct{}{} for _, arg := range requiredArgs { - args[arg] = true + args[arg] = struct{}{} } return &queryValidationSpec{ @@ -2116,7 +2114,7 @@ func QueryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { for _, arg := range args { - s.args[arg] = true + s.args[arg] = struct{}{} } return s } From fc6b0ea0e89ee2b79ddbb9bcbe59148f0af9c3b5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 21 Feb 2018 15:10:52 -0600 Subject: [PATCH 107/311] Add support for comma-separated list of gossip seeds for redundancy. --- docs/configuration.md | 2 +- gossip/gossip.go | 16 +++++++---- server/cluster_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0798d1381..6c20e6ec5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -111,7 +111,7 @@ Any flag that has a value that is a comma separated list on the command line bec #### Gossip Seed -* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. +* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. You may enter multiple seeds by separating them with a comma. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. * Flag: `--gossip.seed="localhost:11101"` * Env: `PILOSA_GOSSIP_SEED="localhost:11101"` * Config: diff --git a/gossip/gossip.go b/gossip/gossip.go index acfad66a1..a61a5b9b7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -92,13 +92,19 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error { RetransmitMult: 3, } - uri, err := pilosa.NewURIFromAddress(g.config.gossipSeed) - if err != nil { - return fmt.Errorf("new uri from address: %s", err) + parts := strings.Split(g.config.gossipSeed, ",") + var uris = make([]*pilosa.URI, len(parts)) + for i, addr := range parts { + uris[i], err = pilosa.NewURIFromAddress(addr) + if err != nil { + return fmt.Errorf("new uri from address: %s", err) + } } - // attach to gossip seed node - nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds + var nodes = make([]*pilosa.Node, len(uris)) + for i, uri := range uris { + nodes[i] = &pilosa.Node{URI: *uri} + } g.mu.RLock() err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings()) diff --git a/server/cluster_test.go b/server/cluster_test.go index 23284bb00..82c54bb96 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -435,3 +435,64 @@ func TestClusterResize_AddNode(t *testing.T) { } }) } + +// Ensure that redundant gossip seeds are used +func TestCluster_GossipMembership(t *testing.T) { + t.Run("Node0Down", func(t *testing.T) { + // Configure node0 + m0 := test.NewMainWithCluster() + defer m0.Close() + + seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster() + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + // Pass invalid seed as first in list + _, _, err = m1.RunWithTransport("localhost", 0, "http://localhost:8765,"+seed, coord) + if err != nil { + return err + } + return nil + }) + + // Configure node2 + m2 := test.NewMainWithCluster() + defer m2.Close() + + eg.Go(func() error { + // Pass invalid seed as last in list + _, _, err = m2.RunWithTransport("localhost", 0, seed+",http://localhost:8765", coord) + if err != nil { + return err + } + return nil + }) + + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State()) + } + + numNodes := len(m0.Server.Cluster.Status().Nodes) + if numNodes != 3 { + t.Fatalf("Expected 3 nodes, got %d", numNodes) + } + }) +} From 52edeb72b4accbc69e9fcafedb9cb913e04619d1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Feb 2018 12:15:27 -0600 Subject: [PATCH 108/311] add validation around node-remove conditions --- cluster.go | 10 +++++++++ handler.go | 16 ++++---------- handler_test.go | 3 --- server/cluster_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/cluster.go b/cluster.go index d9575b58e..c7f5c4193 100644 --- a/cluster.go +++ b/cluster.go @@ -1690,6 +1690,16 @@ func (c *Cluster) NodeLeave(node *Node) error { return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.State()) } + // Ensure that node is in the cluster. + if c.nodeByID(node.ID) == nil { + return fmt.Errorf("Node is not a member of the cluster: %s", node.ID) + } + + // Prevent removing the coordinator node (this node). + if node.ID == c.Node.ID { + return fmt.Errorf("The coordinator node cannot be removed. First, make a different node the new coordinator.") + } + return c.nodeLeave(node) } diff --git a/handler.go b/handler.go index 5d7af3987..fc1771128 100644 --- a/handler.go +++ b/handler.go @@ -2042,20 +2042,13 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht removeNode := h.Cluster.nodeByID(req.ID) if removeNode == nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, fmt.Sprintf("Node is not a member of the cluster: %s", req.ID), http.StatusBadRequest) return } - if err := func() error { - // TODO: prevent removing the coordinator node - // Start the resize process (similar to NodeJoin) - err := h.Cluster.NodeLeave(removeNode) - if err != nil { - return err - } - - return nil - }(); err != nil { + // Start the resize process (similar to NodeJoin) + err = h.Cluster.NodeLeave(removeNode) + if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -2208,7 +2201,6 @@ func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { - fmt.Println("**unsupported media type**") http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) return } diff --git a/handler_test.go b/handler_test.go index c8030a9ea..419cbcecd 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1075,9 +1075,6 @@ func TestHandler_Frame_GetFields(t *testing.T) { t.Fatal(err) } resp, err := http.Get(s.URL + "/index/i/frame/f/fields") - if err != nil { - t.Fatal(err) - } if err != nil { t.Fatal(err) } else if resp.StatusCode != http.StatusOK { diff --git a/server/cluster_test.go b/server/cluster_test.go index 23284bb00..c2af1d3c0 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -16,7 +16,10 @@ package server_test import ( "context" + "fmt" + "net/http" "reflect" + "strings" "testing" "time" @@ -435,3 +438,50 @@ func TestClusterResize_AddNode(t *testing.T) { } }) } + +func TestClusterResize_RemoveNode(t *testing.T) { + cluster := test.MustRunMainWithCluster(t, 3) + m0 := cluster[0] + m1 := cluster[1] + + t.Run("ErrorRemoveInvalidNode", func(t *testing.T) { + resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`) + expBody := "Node is not a member of the cluster: invalid-node-id" + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected StatusCode %d but got %d", http.StatusBadRequest, resp.StatusCode) + } else if strings.TrimSpace(resp.Body) != expBody { + t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) + + t.Run("ErrorRemoveCoordinator", func(t *testing.T) { + resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "") + nodeID := resp.Body + + resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) + + expBody := "The coordinator node cannot be removed. First, make a different node the new coordinator." + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) + } else if strings.TrimSpace(resp.Body) != expBody { + t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) + + t.Run("ErrorRemoveOnNonCoordinator", func(t *testing.T) { + resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "") + coordinatorNodeID := resp.Body + + resp = test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "") + nodeID := resp.Body + + resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) + + expBody := fmt.Sprintf("Node removal requests are only valid on the Coordinator node: %s", coordinatorNodeID) + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) + } else if strings.TrimSpace(resp.Body) != expBody { + t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) +} From a23fe868391779123b9d5e2d7f61e16a138842bc Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 22 Feb 2018 11:40:28 -0600 Subject: [PATCH 109/311] Convert gossip seed string to slice --- config.go | 9 +-------- ctl/server.go | 4 +--- ctl/server_test.go | 3 --- docs/configuration.md | 12 ++++++------ gossip/gossip.go | 21 ++++++++++----------- server/cluster_test.go | 30 +++++++++++++++--------------- server/server.go | 3 --- test/pilosa.go | 15 ++++++++------- 8 files changed, 41 insertions(+), 56 deletions(-) diff --git a/config.go b/config.go index 05f132db2..234334f56 100644 --- a/config.go +++ b/config.go @@ -127,10 +127,6 @@ type TLSConfig struct { type Config struct { DataDir string `toml:"data-dir"` Bind string `toml:"bind"` - // GossipPort DEPRECATED - GossipPort string `toml:"gossip-port"` - // GossipSeed DEPRECATED - GossipSeed string `toml:"gossip-seed"` // Limits the number of mutating commands that can be in a single request to // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. @@ -151,7 +147,7 @@ type Config struct { Gossip struct { Port string `toml:"port"` - Seed string `toml:"seed"` + Seeds []string `toml:"seeds"` Key string `toml:"key"` StreamTimeout Duration `toml:"stream-timeout"` SuspicionMult int `toml:"suspicion-mult"` @@ -193,9 +189,6 @@ func NewConfig() *Config { c.Cluster.LongQueryTime = Duration(time.Minute) // Gossip config. - // c.Gossip.Port = "" - // c.Gossip.Seed = "" - // c.Gossip.Key = "" c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) c.Gossip.SuspicionMult = DefaultGossipSuspicionMult c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) diff --git a/ctl/server.go b/ctl/server.go index 3c5b1a058..18f4f6f93 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -26,8 +26,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") - flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") @@ -43,7 +41,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", srv.Config.Gossip.Seed, "Host with which to seed the gossip membership.") + flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.") flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") diff --git a/ctl/server_test.go b/ctl/server_test.go index dc31426ac..9866b497d 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -28,9 +28,6 @@ func TestBuildServerFlags(t *testing.T) { stdin, stdout, stderr := GetIO(buf) Server := server.NewCommand(stdin, stdout, stderr) BuildServerFlags(cm, Server) - if cm.Flags().Lookup("gossip-port").Name == "" { - t.Fatal("gossip-port flag is required") - } if cm.Flags().Lookup("data-dir").Name == "" { t.Fatal("data-dir flag is required") } diff --git a/docs/configuration.md b/docs/configuration.md index 6c20e6ec5..dfaa01d64 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -109,16 +109,16 @@ Any flag that has a value that is a comma separated list on the command line bec port = 11101 ``` -#### Gossip Seed +#### Gossip Seeds -* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. You may enter multiple seeds by separating them with a comma. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. -* Flag: `--gossip.seed="localhost:11101"` -* Env: `PILOSA_GOSSIP_SEED="localhost:11101"` +* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms. +* Flag: `--gossip.seeds="localhost:11101"` +* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101"` * Config: ```toml [gossip] - seed = "localhost:11101" + seeds = ["localhost:11101"] ``` #### Gossip Key @@ -134,7 +134,7 @@ Any flag that has a value that is a comma separated list on the command line bec #### Cluster Hosts -* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms. +* Description: List of hosts in the cluster. Multiple hosts should be comma-separated in the flag and env forms. * Flag: `--cluster.hosts="localhost:10101"` * Env: `PILOSA_CLUSTER_HOSTS="localhost:10101"` * Config: diff --git a/gossip/gossip.go b/gossip/gossip.go index a61a5b9b7..4b79c41ac 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -62,9 +62,9 @@ func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { return nil } -// Seed returns the gossipSeed determined by the config. -func (g *GossipMemberSet) Seed() string { - return g.config.gossipSeed +// Seeds returns the gossipSeeds determined by the config. +func (g *GossipMemberSet) Seeds() []string { + return g.config.gossipSeeds } // Open implements the MemberSet interface to start network activity. @@ -92,9 +92,8 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error { RetransmitMult: 3, } - parts := strings.Split(g.config.gossipSeed, ",") - var uris = make([]*pilosa.URI, len(parts)) - for i, addr := range parts { + var uris = make([]*pilosa.URI, len(g.config.gossipSeeds)) + for i, addr := range g.config.gossipSeeds { uris[i], err = pilosa.NewURIFromAddress(addr) if err != nil { return fmt.Errorf("new uri from address: %s", err) @@ -148,7 +147,7 @@ func (g *GossipMemberSet) logger() *log.Logger { //////////////////////////////////////////////////////////////// type gossipConfig struct { - gossipSeed string + gossipSeeds []string memberlistConfig *memberlist.Config } @@ -197,14 +196,14 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport g.config = &gossipConfig{ memberlistConfig: conf, - gossipSeed: cfg.Gossip.Seed, + gossipSeeds: cfg.Gossip.Seeds, } g.statusHandler = server - // If no gossipSeed is provided, use local host:port. - if cfg.Gossip.Seed == "" { - g.config.gossipSeed = fmt.Sprintf("%s:%d", host, port) + // If no gossipSeeds is provided, use local host:port. + if len(cfg.Gossip.Seeds) == 0 { + g.config.gossipSeeds = []string{fmt.Sprintf("%s:%d", host, port)} } return g, nil diff --git a/server/cluster_test.go b/server/cluster_test.go index 82c54bb96..bfc3640b9 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -47,7 +47,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // get the host portion of addr to use for binding m0.Config.Gossip.Port = "0" - m0.Config.Gossip.Seed = "" + m0.Config.Gossip.Seeds = []string{} m0.Server.Cluster.Coordinator = m0.Server.URI m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} @@ -75,7 +75,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // get the host portion of addr to use for binding m1.Config.Gossip.Port = "0" - m1.Config.Gossip.Seed = gossipMemberSet0.Seed() + m1.Config.Gossip.Seeds = gossipMemberSet0.Seeds() m1.Server.Cluster.Coordinator = m0.Server.URI m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver() @@ -222,7 +222,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { gossipHost := "localhost" gossipPort := 0 - seed, coord, err := m0.RunWithTransport(gossipHost, gossipPort, "", pilosa.URI{}) + seed, coord, err := m0.RunWithTransport(gossipHost, gossipPort, []string{}, pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -231,7 +231,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { m1 := test.NewMainWithCluster() defer m1.Close() - seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, seed, coord) + seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed}, coord) if err != nil { t.Fatal(err) } @@ -250,7 +250,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -261,7 +261,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) + _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) if err != nil { return err } @@ -284,7 +284,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) + _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) if err != nil { return err } @@ -330,7 +330,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -360,7 +360,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) + _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) if err != nil { return err } @@ -385,7 +385,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -415,7 +415,7 @@ func TestClusterResize_AddNode(t *testing.T) { var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, seed, coord) + _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) if err != nil { return err } @@ -443,7 +443,7 @@ func TestCluster_GossipMembership(t *testing.T) { m0 := test.NewMainWithCluster() defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) if err != nil { t.Fatal(err) } @@ -455,7 +455,7 @@ func TestCluster_GossipMembership(t *testing.T) { var eg errgroup.Group eg.Go(func() error { // Pass invalid seed as first in list - _, _, err = m1.RunWithTransport("localhost", 0, "http://localhost:8765,"+seed, coord) + _, _, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}, coord) if err != nil { return err } @@ -468,7 +468,7 @@ func TestCluster_GossipMembership(t *testing.T) { eg.Go(func() error { // Pass invalid seed as last in list - _, _, err = m2.RunWithTransport("localhost", 0, seed+",http://localhost:8765", coord) + _, _, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}, coord) if err != nil { return err } diff --git a/server/server.go b/server/server.go index 19a08f982..7b623461a 100644 --- a/server/server.go +++ b/server/server.go @@ -237,11 +237,8 @@ func (m *Command) SetupNetworking() error { // Set internal port (string). gossipPortStr := pilosa.DefaultGossipPort - // Config.GossipPort is deprecated, so Config.Gossip.Port has priority if m.Config.Gossip.Port != "" { gossipPortStr = m.Config.Gossip.Port - } else if m.Config.GossipPort != "" { - gossipPortStr = m.Config.GossipPort } gossipPort, err := strconv.Atoi(gossipPortStr) diff --git a/test/pilosa.go b/test/pilosa.go index 3d741af76..de03b1c28 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -93,13 +93,13 @@ func runMainWithCluster(size int) ([]*Main, error) { gossipHost := "localhost" gossipPort := 0 var err error - var gossipSeed string + var gossipSeeds = make([]string, size) var coordinator pilosa.URI for i := 0; i < size; i++ { m := NewMainWithCluster() - gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, coordinator) + gossipSeeds[i], coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i], coordinator) if err != nil { return nil, errors.Wrap(err, "RunWithTransport") } @@ -146,7 +146,7 @@ func (m *Main) Reopen() error { } // RunWithTransport runs Main and returns the dynamically allocated gossip port. -func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator pilosa.URI) (seed string, coord pilosa.URI, err error) { +func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string, coordinator pilosa.URI) (seed string, coord pilosa.URI, err error) { defer close(m.Started) /* @@ -182,12 +182,13 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coor } m.GossipTransport = transport - if joinSeed != "" { - m.Config.Gossip.Seed = joinSeed + if len(joinSeeds) != 0 { + m.Config.Gossip.Seeds = joinSeeds } else { - m.Config.Gossip.Seed = transport.URI.String() + m.Config.Gossip.Seeds = []string{transport.URI.String()} } - seed = m.Config.Gossip.Seed + + seed = transport.URI.String() // SetupNetworking err = m.SetupNetworking() From e454473ec78b6b6be65d7e5b5d9479f0ac841314 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Feb 2018 16:02:43 -0600 Subject: [PATCH 110/311] prevent excessive sendSyce (createView) messages. --- frame.go | 41 +++++++++++++++++++---------------------- server.go | 2 +- view.go | 6 ++++++ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/frame.go b/frame.go index b5807dc71..069e0bc89 100644 --- a/frame.go +++ b/frame.go @@ -22,7 +22,6 @@ import ( "os" "path/filepath" "sort" - "strings" "sync" "time" @@ -574,53 +573,51 @@ func (f *Frame) RecalculateCaches() { // Additionally, a CreateViewMessage is sent to the cluster. func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { - view, err := f.CreateViewIfNotExistsBase(name) + view, created, err := f.createViewIfNotExistsBase(name) if err != nil { return nil, err } - // Broadcast view creation to the cluster. - err = f.broadcaster.SendSync( - &internal.CreateViewMessage{ - Index: f.index, - Frame: f.name, - View: name, - }) - if err != nil { - return nil, err + if created { + // Broadcast view creation to the cluster. + err = f.broadcaster.SendSync( + &internal.CreateViewMessage{ + Index: f.index, + Frame: f.name, + View: name, + }) + if err != nil { + return nil, err + } } return view, nil } -// CreateViewIfNotExistsBase returns the named view, creating it if necessary. -func (f *Frame) CreateViewIfNotExistsBase(name string) (*View, error) { +// createViewIfNotExistsBase returns the named view, creating it if necessary. +// The returned bool indicates whether the view was created or not. +func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { // Don't create inverse views if they are not enabled. if !f.InverseEnabled() && IsInverseView(name) { - return nil, ErrFrameInverseDisabled + return nil, false, ErrFrameInverseDisabled } f.mu.Lock() defer f.mu.Unlock() if view := f.views[name]; view != nil { - return view, nil + return view, false, nil } view := f.newView(f.ViewPath(name), name) - // Never keep a cache for field views. - if strings.HasPrefix(name, ViewFieldPrefix) { - view.cacheType = CacheTypeNone - } - if err := view.Open(); err != nil { - return nil, err + return nil, false, err } view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view - return view, nil + return view, true, nil } func (f *Frame) newView(path, name string) *View { diff --git a/server.go b/server.go index 6a92a7768..3751e28ad 100644 --- a/server.go +++ b/server.go @@ -424,7 +424,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if f == nil { return fmt.Errorf("Local Frame not found: %s", obj.Frame) } - _, err := f.CreateViewIfNotExistsBase(obj.View) + _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { return err } diff --git a/view.go b/view.go index d77d9257a..db2eb97f1 100644 --- a/view.go +++ b/view.go @@ -99,6 +99,12 @@ func (v *View) Path() string { return v.path } // Open opens and initializes the view. func (v *View) Open() error { + + // Never keep a cache for field views. + if strings.HasPrefix(v.name, ViewFieldPrefix) { + v.cacheType = CacheTypeNone + } + if err := func() error { // Ensure the view's path exists. if err := os.MkdirAll(v.path, 0777); err != nil { From 1086c6c9597b05ebfb360a2ae4418c043e858015 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 23 Feb 2018 10:18:53 -0600 Subject: [PATCH 111/311] remove old GossipPort and GossipSeed config options --- config.go | 4 ---- ctl/server.go | 4 +--- ctl/server_test.go | 3 --- server/server.go | 2 -- 4 files changed, 1 insertion(+), 12 deletions(-) diff --git a/config.go b/config.go index 05f132db2..2ed26dbfe 100644 --- a/config.go +++ b/config.go @@ -127,10 +127,6 @@ type TLSConfig struct { type Config struct { DataDir string `toml:"data-dir"` Bind string `toml:"bind"` - // GossipPort DEPRECATED - GossipPort string `toml:"gossip-port"` - // GossipSeed DEPRECATED - GossipSeed string `toml:"gossip-seed"` // Limits the number of mutating commands that can be in a single request to // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. diff --git a/ctl/server.go b/ctl/server.go index 3c5b1a058..2341123dd 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -26,8 +26,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") - flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") @@ -38,7 +36,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") - flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") + flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") // Gossip diff --git a/ctl/server_test.go b/ctl/server_test.go index dc31426ac..9866b497d 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -28,9 +28,6 @@ func TestBuildServerFlags(t *testing.T) { stdin, stdout, stderr := GetIO(buf) Server := server.NewCommand(stdin, stdout, stderr) BuildServerFlags(cm, Server) - if cm.Flags().Lookup("gossip-port").Name == "" { - t.Fatal("gossip-port flag is required") - } if cm.Flags().Lookup("data-dir").Name == "" { t.Fatal("data-dir flag is required") } diff --git a/server/server.go b/server/server.go index 19a08f982..5b6bd573d 100644 --- a/server/server.go +++ b/server/server.go @@ -240,8 +240,6 @@ func (m *Command) SetupNetworking() error { // Config.GossipPort is deprecated, so Config.Gossip.Port has priority if m.Config.Gossip.Port != "" { gossipPortStr = m.Config.Gossip.Port - } else if m.Config.GossipPort != "" { - gossipPortStr = m.Config.GossipPort } gossipPort, err := strconv.Atoi(gossipPortStr) From 3d1538d53c4ebf96f326ba97e99627f135eb4d8f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 23 Feb 2018 10:48:55 -0600 Subject: [PATCH 112/311] handle the scheme correctly in config.Bind --- gossip/gossip.go | 13 ++++++++----- test/pilosa.go | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index acfad66a1..66aee8c13 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -19,7 +19,6 @@ import ( "io" "io/ioutil" "log" - "net" "os" "strconv" "strings" @@ -154,10 +153,12 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport } port := transport.Net.GetAutoBindPort() - host, _, err := net.SplitHostPort(cfg.Bind) + + bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) if err != nil { - return nil, fmt.Errorf("split host port: %s", err) + return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err) } + host := bindURI.Host() var gossipKey []byte if cfg.Gossip.Key != "" { @@ -210,10 +211,12 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) if err != nil { return nil, fmt.Errorf("convert port: %s", err) } - host, _, err := net.SplitHostPort(cfg.Bind) + + bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) if err != nil { - return nil, fmt.Errorf("split host port: %s", err) + return nil, fmt.Errorf("getting uri from bind address: %s", err) } + host := bindURI.Host() // Set up the transport. transport, err := NewTransport(host, port) diff --git a/test/pilosa.go b/test/pilosa.go index 3d741af76..8e5f00268 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -50,7 +50,7 @@ func NewMain() *Main { m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} m.Server.Network = *Network m.Config.DataDir = path - m.Config.Bind = "localhost:0" + m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout From 3bba35236f9037f71f7edd56d57bd447737d904f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 23 Feb 2018 11:11:45 -0600 Subject: [PATCH 113/311] Add back example settings --- config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.go b/config.go index 234334f56..4fa06fc51 100644 --- a/config.go +++ b/config.go @@ -189,6 +189,9 @@ func NewConfig() *Config { c.Cluster.LongQueryTime = Duration(time.Minute) // Gossip config. + // c.Gossip.Port = "" + // c.Gossip.Seeds = []string{} + // c.Gossip.Key = "" c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) c.Gossip.SuspicionMult = DefaultGossipSuspicionMult c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) From 10ebb6ab11cd1cd1cfdab057cadab9974a3c5830 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 26 Feb 2018 10:30:51 -0600 Subject: [PATCH 114/311] put GCNotify behind an inerface --- gc.go | 38 ++++++++++++++++++++++++++++++++++++++ gcnotify/gcnotify.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ server.go | 10 ++++++---- server/server.go | 2 ++ 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 gc.go create mode 100644 gcnotify/gcnotify.go diff --git a/gc.go b/gc.go new file mode 100644 index 000000000..a260cc0b5 --- /dev/null +++ b/gc.go @@ -0,0 +1,38 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +// GCNotifier represents an interface for garbage collection notificationss. +type GCNotifier interface { + Close() + AfterGC() <-chan struct{} +} + +func init() { + NopGCNotifier = &nopGCNotifier{} +} + +// NopGCNotifier represents a GCNotifier that doesn't do anything. +var NopGCNotifier GCNotifier + +type nopGCNotifier struct{} + +// Close is a no-op implemenetation of GCNotifier Close method. +func (n *nopGCNotifier) Close() {} + +// AfterGC is a no-op implemenetation of GCNotifier AfterGC method. +func (c *nopGCNotifier) AfterGC() <-chan struct{} { + return nil +} diff --git a/gcnotify/gcnotify.go b/gcnotify/gcnotify.go new file mode 100644 index 000000000..76953a378 --- /dev/null +++ b/gcnotify/gcnotify.go @@ -0,0 +1,44 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcnotify + +import ( + "github.com/CAFxX/gcnotifier" + "github.com/pilosa/pilosa" +) + +// Ensure ActiveGCNotifier implements interface. +var _ pilosa.GCNotifier = &ActiveGCNotifier{} + +type ActiveGCNotifier struct { + gcn *gcnotifier.GCNotifier +} + +// NewActiveGCNotifier creates an active GCNotifier. +func NewActiveGCNotifier() *ActiveGCNotifier { + return &ActiveGCNotifier{ + gcn: gcnotifier.New(), + } +} + +// Close implements the GCNotifier interface. +func (n *ActiveGCNotifier) Close() { + n.gcn.Close() +} + +// AfterGC implements the GCNotifier interface. +func (n *ActiveGCNotifier) AfterGC() <-chan struct{} { + return n.gcn.AfterGC() +} diff --git a/server.go b/server.go index 3751e28ad..be63a39e3 100644 --- a/server.go +++ b/server.go @@ -31,7 +31,6 @@ import ( "sync" "time" - "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" @@ -73,6 +72,8 @@ type Server struct { Cluster *Cluster diagnostics *diagnostics.Diagnostics + GCNotifier GCNotifier + // Background monitoring intervals. AntiEntropyInterval time.Duration MetricInterval time.Duration @@ -103,6 +104,8 @@ func NewServer() *Server { Network: "tcp", + GCNotifier: NopGCNotifier, + AntiEntropyInterval: DefaultAntiEntropyInterval, MetricInterval: 0, DiagnosticInterval: 0, @@ -652,8 +655,7 @@ func (s *Server) monitorRuntime() { ticker := time.NewTicker(s.MetricInterval) defer ticker.Stop() - gcn := gcnotifier.New() - defer gcn.Close() + defer s.GCNotifier.Close() s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) @@ -662,7 +664,7 @@ func (s *Server) monitorRuntime() { select { case <-s.closing: return - case <-gcn.AfterGC(): + case <-s.GCNotifier.AfterGC(): // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: diff --git a/server/server.go b/server/server.go index 9803f7be2..f0d3314fa 100644 --- a/server/server.go +++ b/server/server.go @@ -33,6 +33,7 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -151,6 +152,7 @@ func (m *Command) SetupServer() error { if m.Config.Metric.Diagnostics { m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval) } + m.Server.GCNotifier = gcnotify.NewActiveGCNotifier() m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) if err != nil { return err From a20235f812d5e6b0d0cc85599c404284b97c6f02 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 26 Feb 2018 15:26:28 -0600 Subject: [PATCH 115/311] Add Xor spec --- docs/query-language.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/query-language.md b/docs/query-language.md index b1315d293..365d7ded6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -310,6 +310,12 @@ Return `{"attrs":{},"bits":[30]}` #### Xor +**Spec:** + +``` +Xor(, [BITMAP_CALL ...]) +``` + **Description:** Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it. From 2c8d9a5ae0c9d1492c771b0305763141dfda8454 Mon Sep 17 00:00:00 2001 From: Ilya Tocar Date: Wed, 28 Feb 2018 16:41:51 -0600 Subject: [PATCH 116/311] roaring: speed-up intersectionCountArrayBitmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While looking at benchmarks as a possible go compiler benchmarks. I've tried some optimizations by hand: Move len(b.bitmap) load out of the loop. Remove some type conversions. Use (x >> off) & 1 to get offs bit, instead of x & (1 << of)f >> off. This produces nice speed-up and passes go test roaring. name old time/op new time/op delta Bitmap_IntersectionCount_ArrayRun-6 2.06µs ± 0% 1.57µs ± 0% -24.04% (p=0.000 n=10+9) Bitmap_IntersectionCount_BitmapRun-6 2.24µs ± 0% 2.24µs ± 0% ~ (p=0.913 n=10+9) Bitmap_IntersectionCount_ArrayBitmap-6 2.06µs ± 0% 1.56µs ± 1% -24.05% (p=0.000 n=9+10) --- roaring/roaring.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 8ea82af83..9e233347e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1907,13 +1907,14 @@ func intersectionCountBitmapRun(a, b *container) (n int) { } func intersectionCountArrayBitmap(a, b *container) (n int) { + ln := len(b.bitmap) for _, val := range a.array { - i := val >> 6 - if i >= uint16(len(b.bitmap)) { + i := int(val >> 6) + if i >= ln { break } off := val % 64 - n += int((b.bitmap[i] & (1 << off)) >> off) + n += int(b.bitmap[i]>>off) & 1 } return n } From dcb411d028705bf92b55d4da9bed07a3d46f6261 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 2 Mar 2018 11:00:46 -0600 Subject: [PATCH 117/311] add warnings about sequential ids --- docs/client-libraries.md | 1 + docs/data-model.md | 2 ++ docs/getting-started.md | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/docs/client-libraries.md b/docs/client-libraries.md index 67e36b5dc..a328318ee 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -10,6 +10,7 @@ nav = [ ## Client Libraries +This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is not wise to use the output of a hash, or randomly distributed ids with Pilosa. ### Go diff --git a/docs/data-model.md b/docs/data-model.md index 5b1741fca..d448a1de8 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -24,6 +24,8 @@ Rows and columns can represent anything (they could even represent the same set Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa also has the ability to categorize rows into different "frames" and quickly retrieve the top rows in a frame sorted by the number of bits set in each row. +Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but if you try to set a bit with column ID 2^63, bad things will start to happen. + ![data model diagram](/img/docs/data-model.svg) ### Index diff --git a/docs/getting-started.md b/docs/getting-started.md index d4802e8ee..f68805f53 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -226,6 +226,11 @@ curl localhost:10101/index/repository/query \ {"results":[true]} ``` +Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number. +Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to poor performance, out of memory errors, and more. + + + ### What's Next? You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). From d04530a5c05afc55396adb3bde2e681694c385ff Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Sat, 3 Mar 2018 11:45:48 -0600 Subject: [PATCH 118/311] make container struct description more accurate --- roaring/roaring.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 8ea82af83..3cc537a08 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1002,12 +1002,16 @@ const ArrayMaxSize = 4096 // RunMaxSize represents the maximum size of run length encoded containers. const RunMaxSize = 2048 -// container represents a container for uint32 integers. +// container represents a container for uint16 integers. // -// These are used for storing the low bits. Containers are separated into three -// types depending on cardinality. For containers with less than 4,096 values, -// an array or RLE container is used, depending on the contents. For containers -// with more than 4,096 values, the values are encoded into bitmaps. +// These are used for storing the low bits of numbers in larger sets of uint64. +// The high bits are stored in a container's key which is tracked by a separate +// data structure. Integers in a container can be encoded in one of three ways - +// the encoding used is usually whichever is most compact, though any container +// type should be able to encode any set of integers safely. For containers with +// less than 4,096 values, an array is often used. Containers with long runs of +// integers would use run length encoding, and more random data usually uses +// bitmap encoding. type container struct { mapped bool // mapped directly to a byte slice when true containerType byte // array, bitmap, or run From 1233226aa0deaa1927d476299e9978857179f089 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 6 Mar 2018 12:18:44 -0600 Subject: [PATCH 119/311] remove Join method from StaticMemberSet struct --- broadcast.go | 12 ++++-------- server/server.go | 6 +----- test/cluster.go | 2 +- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/broadcast.go b/broadcast.go index dd8802f47..62d69e9ff 100644 --- a/broadcast.go +++ b/broadcast.go @@ -35,8 +35,10 @@ type StaticMemberSet struct { } // NewStaticMemberSet creates a statically defined MemberSet. -func NewStaticMemberSet() *StaticMemberSet { - return &StaticMemberSet{} +func NewStaticMemberSet(nodes []*Node) *StaticMemberSet { + return &StaticMemberSet{ + nodes: nodes, + } } // Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing. @@ -44,12 +46,6 @@ func (s *StaticMemberSet) Open(n *Node) error { return nil } -// Join sets the MemberSet nodes to the slice of Nodes passed in. -func (s *StaticMemberSet) Join(nodes []*Node) error { - s.nodes = nodes - return nil -} - // Broadcaster is an interface for broadcasting messages. type Broadcaster interface { SendSync(pb proto.Message) error diff --git a/server/server.go b/server/server.go index f0d3314fa..cd98ce594 100644 --- a/server/server.go +++ b/server/server.go @@ -227,13 +227,9 @@ func (m *Command) SetupNetworking() error { } m.Server.Broadcaster = pilosa.NopBroadcaster - m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet() + m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes) m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver m.Server.Gossiper = pilosa.NopGossiper - err := m.Server.Cluster.MemberSet.(*pilosa.StaticMemberSet).Join(m.Server.Cluster.Nodes) - if err != nil { - return err - } return nil } diff --git a/test/cluster.go b/test/cluster.go index f5e0dc7a9..536641b13 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -260,7 +260,7 @@ func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, err c.Path = path c.Topology = pilosa.NewTopology() c.Holder = h - c.MemberSet = pilosa.NewStaticMemberSet() + c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) c.Node = node c.Coordinator = t.common.Nodes[0].URI // the first node is the coordinator c.Broadcaster = t From d0009206b457b55495e0cdd39e622b9315be08eb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 6 Mar 2018 12:42:29 -0600 Subject: [PATCH 120/311] add comments to exported methods. remove debugging test. --- cluster.go | 6 ++++-- holder_test.go | 31 ------------------------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/cluster.go b/cluster.go index ed5318917..a2d537b11 100644 --- a/cluster.go +++ b/cluster.go @@ -74,7 +74,7 @@ func (n Node) String() string { return fmt.Sprintf("Node: %s", n.ID) } -// EncodeNodes converts a into its internal representation. +// EncodeNodes converts a slice of Nodes into its internal representation. func EncodeNodes(a []*Node) []*internal.Node { other := make([]*internal.Node, len(a)) for i := range a { @@ -83,7 +83,7 @@ func EncodeNodes(a []*Node) []*internal.Node { return other } -// EncodeNode converts n into its internal representation. +// EncodeNode converts a Node into its internal representation. func EncodeNode(n *Node) *internal.Node { return &internal.Node{ ID: n.ID, @@ -91,6 +91,7 @@ func EncodeNode(n *Node) *internal.Node { } } +// DecodeNodes converts a proto message into a slice of Nodes. func DecodeNodes(a []*internal.Node) []*Node { if len(a) == 0 { return nil @@ -102,6 +103,7 @@ func DecodeNodes(a []*internal.Node) []*Node { return other } +// DecodeNode converts a proto message into a Node. func DecodeNode(node *internal.Node) *Node { return &Node{ ID: node.ID, diff --git a/holder_test.go b/holder_test.go index 3422fe081..5a06a9272 100644 --- a/holder_test.go +++ b/holder_test.go @@ -338,37 +338,6 @@ func TestHolder_HasData(t *testing.T) { }) } -/* -func TestHolder_Schema(t *testing.T) { - t.Run("Schema", func(t *testing.T) { - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if frame, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "i", "f", "views", "standard", "fragments", "0"), 0000); err != nil { - t.Fatal(err) - } - fmt.Printf("%v\n", h.Schema()) - defer os.Chmod(filepath.Join(h.Path, "i", "f", "views", "standard", "fragments", "0"), 0666) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - - t.Fatalf("STOPPER") - }) -} -*/ - // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { hldr := test.MustOpenHolder() From d28a30ebd411acaee94ce218a2a2d4cd5a4e726c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 6 Mar 2018 15:11:46 -0600 Subject: [PATCH 121/311] Proper error handling when attempting to remove node when there aren't enough replicas --- cluster.go | 15 +++++++++++++-- server/cluster_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index ed5318917..12d6fba50 100644 --- a/cluster.go +++ b/cluster.go @@ -737,7 +737,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // the fragment. srcNodeID, ok := srcNodesByFrag[frag] if !ok { - return nil, errors.New("not enough data to perform resize") + return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)") } src := &internal.ResizeSource{ @@ -937,7 +937,11 @@ func (c *Cluster) allNodesReady() bool { func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { j, err := c.generateResizeJob(nodeAction) if err != nil { - return err + c.logger().Printf("generateResizeJob error: err=%s", err) + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.logger().Printf("setStateAndBroadcast error: err=%s", err) + } + return c.setStateAndBroadcast(ClusterStateNormal) } // j.Run() runs in a goroutine because in the case where the @@ -1702,6 +1706,13 @@ func (c *Cluster) NodeLeave(node *Node) error { return fmt.Errorf("The coordinator node cannot be removed. First, make a different node the new coordinator.") } + // See if resize job can be generated + _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) + + if err != nil { + return err + } + return c.nodeLeave(node) } diff --git a/server/cluster_test.go b/server/cluster_test.go index 5ec805f81..9c518aecb 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -545,4 +545,35 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) } }) + + t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) { + client0 := m0.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + setBits := "" + for i := 0; i < 20; i++ { + setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) + } + + if _, err := m0.Query("i", "", setBits); err != nil { + t.Fatal(err) + } + + resp := test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "") + nodeID := resp.Body + + resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) + expBody := "not enough data to perform resize" + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) + } else if !strings.Contains(resp.Body, expBody) { + t.Fatalf("expected to contain '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) } From b7b92913d9c0013e21f7e582fcd9c0cb47565212 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 6 Mar 2018 15:13:24 -0600 Subject: [PATCH 122/311] Add comment to listenForJoins --- cluster.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 12d6fba50..dcb9e0ecc 100644 --- a/cluster.go +++ b/cluster.go @@ -1002,7 +1002,13 @@ func (c *Cluster) ListenForJoins() { } func (c *Cluster) listenForJoins() { - var uriJoined bool + // When a cluster starts, the state is STARTING. + // We first want to wait for at least one node to join. + // Then we want to clear out the joiningLeavingNodes queue (buffered channel). + // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. + // We use a bool `setNormal` to indicate when at least one node has joined. + + var setNormal bool for { @@ -1014,13 +1020,13 @@ func (c *Cluster) listenForJoins() { c.logger().Printf("handleNodeAction error: err=%s", err) continue } - uriJoined = true + setNormal = true continue default: } // Only change state to NORMAL if we have successfully added at least one host. - if uriJoined { + if setNormal { // Put the cluster back to state NORMAL and broadcast. if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { c.logger().Printf("setStateAndBroadcast error: err=%s", err) @@ -1037,7 +1043,7 @@ func (c *Cluster) listenForJoins() { c.logger().Printf("handleNodeAction error: err=%s", err) continue } - uriJoined = true + setNormal = true continue } } From e03dee938677c9e008f462ad5cfd2eeb9cf4ff5b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 6 Mar 2018 16:57:45 -0600 Subject: [PATCH 123/311] put Statik behind an interface --- filesystem.go | 42 ++++++++++++++++++++++++++++++++++++++++++ filesystem/statik.go | 32 ++++++++++++++++++++++++++++++++ handler.go | 10 ++++++++-- handler_test.go | 2 ++ server/server.go | 4 ++++ 5 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 filesystem.go create mode 100644 filesystem/statik.go diff --git a/filesystem.go b/filesystem.go new file mode 100644 index 000000000..0b5b27220 --- /dev/null +++ b/filesystem.go @@ -0,0 +1,42 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "net/http" +) + +// Ensure nopStaticFileSystem implements interface. +var _ StaticFileSystem = &nopStaticFileSystem{} + +// StaticFileSystem represents an interface for a static WebUI. +type StaticFileSystem interface { + New() (http.FileSystem, error) +} + +func init() { + NopStaticFileSystem = &nopStaticFileSystem{} +} + +// NopStaticFileSystem represents a StaticFileSystem that returns an error if called. +var NopStaticFileSystem StaticFileSystem + +type nopStaticFileSystem struct{} + +// New is a no-op implementation of StaticFileSystem New method. +func (n *nopStaticFileSystem) New() (http.FileSystem, error) { + return nil, fmt.Errorf("static file system not implemented") +} diff --git a/filesystem/statik.go b/filesystem/statik.go new file mode 100644 index 000000000..405e31a11 --- /dev/null +++ b/filesystem/statik.go @@ -0,0 +1,32 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystem + +import ( + "net/http" + + "github.com/pilosa/pilosa" + "github.com/rakyll/statik/fs" +) + +// Ensure nopStaticFileSystem implements interface. +var _ pilosa.StaticFileSystem = &StatikFS{} + +type StatikFS struct{} + +// New is a statik implementation of StaticFileSystem New method. +func (s *StatikFS) New() (http.FileSystem, error) { + return fs.New() +} diff --git a/handler.go b/handler.go index 38fd24eca..1688cf5a3 100644 --- a/handler.go +++ b/handler.go @@ -47,7 +47,6 @@ import ( // Allow building Pilosa without the web UI. _ "github.com/pilosa/pilosa/statik" - "github.com/rakyll/statik/fs" ) // Handler represents an HTTP handler. @@ -57,6 +56,8 @@ type Handler struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler + StaticFileSystem StaticFileSystem + // Local hostname & cluster configuration. Node *Node Cluster *Cluster @@ -98,6 +99,11 @@ type errorResponse struct { // NewHandler returns a new instance of Handler with a default logger. func NewHandler() *Handler { handler := &Handler{ + Broadcaster: NopBroadcaster, + //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop + //StatusHandler: NopStatusHandler, // TODO: implement the nop + StaticFileSystem: NopStaticFileSystem, + LogOutput: os.Stderr, } BuildRouters(handler) @@ -285,7 +291,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) return } - statikFS, err := fs.New() + statikFS, err := h.StaticFileSystem.New() if err != nil { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") diff --git a/handler_test.go b/handler_test.go index 1caf7bf99..6386f53f9 100644 --- a/handler_test.go +++ b/handler_test.go @@ -29,6 +29,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/filesystem" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -1853,6 +1854,7 @@ func TestHandler_WebUI(t *testing.T) { h := test.NewHandler() h.Holder = hldr.Holder h.Cluster = test.NewCluster(1) + h.StaticFileSystem = &filesystem.StatikFS{} w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) diff --git a/server/server.go b/server/server.go index f0d3314fa..e056bc76d 100644 --- a/server/server.go +++ b/server/server.go @@ -33,6 +33,7 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/filesystem" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" @@ -191,6 +192,9 @@ func (m *Command) SetupServer() error { m.Server.Handler.RemoteClient = c m.Server.Cluster.RemoteClient = c + // Statik file system. + m.Server.Handler.StaticFileSystem = &filesystem.StatikFS{} + // Default coordintor to port 0 when not specified so that coordinator // can be set to the value of server.URI after server binds to a port. // This would only be useful in a one-node cluster. From 2b1a3c84036b8eef27f604953c6e1bc923f51de2 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 6 Mar 2018 17:48:14 -0600 Subject: [PATCH 124/311] Add docs readme --- docs/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..bf4fb3b91 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,5 @@ +Pilosa docs are maintained here, to stay in sync with the codebase. + +Please visit https://www.pilosa.com/docs to view the docs complete with styles, diagrams, and comprehensive search. + +Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request! From e7eb68b1019f9c9564ee99a31816f7eeb118bc62 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 6 Mar 2018 17:48:40 -0600 Subject: [PATCH 125/311] Add cosmosa to new external tutorials section --- docs/tutorials.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/tutorials.md b/docs/tutorials.md index 4d787577c..ffaf51d64 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -8,6 +8,10 @@ nav = [ ] +++ +### External Tutorials + +- [Run Pilosa with Microsoft's Azure Cosmos DB](https://github.com/pilosa/cosmosa) + ## Tutorials ### Setting Up a Secure Cluster From 04b17847ceac18f28f4b9d69950a7a90ef3892fd Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 6 Mar 2018 18:14:22 -0600 Subject: [PATCH 126/311] Clean up glossary --- docs/glossary.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/glossary.md b/docs/glossary.md index 50906a0cf..998bda86b 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -20,9 +20,7 @@ nav = [] Attribute: Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store. -PQL: Pilosa Query Language - -Index: The Index represents a data namespace. +PQL: [Pilosa Query Language](/docs/query-language). Frame: Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame. @@ -38,7 +36,7 @@ nav = [] Anti-entropy: A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies. -Node: An individual running instance of Pilosa server which belongs to a cluster. +Node: An individual running instance of Pilosa server which belongs to a cluster. Cluster: A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. @@ -46,13 +44,12 @@ nav = [] Tanimoto: Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity. -Protobuf:: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. +Protobuf: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. TOML: We use [TOML](https://github.com/toml-lang/toml) for our configuration file format. -Jump Consistent Hash: A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -https://arxiv.org/pdf/1406.2294v1.pdf +Jump Consistent Hash: A fast, minimal memory, [consistent hash algorithm](https://arxiv.org/pdf/1406.2294v1.pdf) that evenly distributes the workload even when the number of buckets changes. -Partition: The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography. +Partition: The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associate data across the new cluster topography. -Replica: A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made). +Replica: A copy of a [fragment](#fragment) on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made). From 42682e12a828639aa953baf06e526cdb500ed5b4 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 7 Mar 2018 09:45:27 -0600 Subject: [PATCH 127/311] Address code review: Fix error handling and add comment --- cluster.go | 2 +- server/cluster_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index dcb9e0ecc..8b56edaaf 100644 --- a/cluster.go +++ b/cluster.go @@ -941,7 +941,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { c.logger().Printf("setStateAndBroadcast error: err=%s", err) } - return c.setStateAndBroadcast(ClusterStateNormal) + return err } // j.Run() runs in a goroutine because in the case where the diff --git a/server/cluster_test.go b/server/cluster_test.go index 9c518aecb..299415bc3 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -556,6 +556,8 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Fatal(err) } + // This is an attempt to ensure there is data on both nodes, but is not guaranteed. + // TODO: Deterministic node IDs would ensure consistent results setBits := "" for i := 0; i < 20; i++ { setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) From d6896ea0a5127489cf800cfa0a3d04cf074f1c3d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 7 Mar 2018 11:12:05 -0600 Subject: [PATCH 128/311] rename StaticFileSystem to FileSystem. re-org statik files --- filesystem.go | 22 +++++++++---------- handler.go | 6 ++--- handler_test.go | 4 ++-- server/server.go | 4 ++-- .../statik.go => statikfs/filesystem.go | 13 ++++++----- 5 files changed, 25 insertions(+), 24 deletions(-) rename filesystem/statik.go => statikfs/filesystem.go (70%) diff --git a/filesystem.go b/filesystem.go index 0b5b27220..5664f0987 100644 --- a/filesystem.go +++ b/filesystem.go @@ -19,24 +19,24 @@ import ( "net/http" ) -// Ensure nopStaticFileSystem implements interface. -var _ StaticFileSystem = &nopStaticFileSystem{} +// Ensure nopFileSystem implements interface. +var _ FileSystem = &nopFileSystem{} -// StaticFileSystem represents an interface for a static WebUI. -type StaticFileSystem interface { +// FileSystem represents an interface for a WebUI file system. +type FileSystem interface { New() (http.FileSystem, error) } func init() { - NopStaticFileSystem = &nopStaticFileSystem{} + NopFileSystem = &nopFileSystem{} } -// NopStaticFileSystem represents a StaticFileSystem that returns an error if called. -var NopStaticFileSystem StaticFileSystem +// NopFileSystem represents a FileSystem that returns an error if called. +var NopFileSystem FileSystem -type nopStaticFileSystem struct{} +type nopFileSystem struct{} -// New is a no-op implementation of StaticFileSystem New method. -func (n *nopStaticFileSystem) New() (http.FileSystem, error) { - return nil, fmt.Errorf("static file system not implemented") +// New is a no-op implementation of FileSystem New method. +func (n *nopFileSystem) New() (http.FileSystem, error) { + return nil, fmt.Errorf("file system not implemented") } diff --git a/handler.go b/handler.go index 1688cf5a3..067219c8f 100644 --- a/handler.go +++ b/handler.go @@ -56,7 +56,7 @@ type Handler struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler - StaticFileSystem StaticFileSystem + FileSystem FileSystem // Local hostname & cluster configuration. Node *Node @@ -102,7 +102,7 @@ func NewHandler() *Handler { Broadcaster: NopBroadcaster, //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop //StatusHandler: NopStatusHandler, // TODO: implement the nop - StaticFileSystem: NopStaticFileSystem, + FileSystem: NopFileSystem, LogOutput: os.Stderr, } @@ -291,7 +291,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) return } - statikFS, err := h.StaticFileSystem.New() + statikFS, err := h.FileSystem.New() if err != nil { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") diff --git a/handler_test.go b/handler_test.go index 6386f53f9..4f3f12f77 100644 --- a/handler_test.go +++ b/handler_test.go @@ -29,9 +29,9 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/filesystem" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/statikfs" "github.com/pilosa/pilosa/test" ) @@ -1854,7 +1854,7 @@ func TestHandler_WebUI(t *testing.T) { h := test.NewHandler() h.Holder = hldr.Holder h.Cluster = test.NewCluster(1) - h.StaticFileSystem = &filesystem.StatikFS{} + h.FileSystem = &statikfs.FileSystem{} w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) diff --git a/server/server.go b/server/server.go index e056bc76d..52df0e52e 100644 --- a/server/server.go +++ b/server/server.go @@ -33,9 +33,9 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/filesystem" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/statikfs" "github.com/pilosa/pilosa/statsd" ) @@ -193,7 +193,7 @@ func (m *Command) SetupServer() error { m.Server.Cluster.RemoteClient = c // Statik file system. - m.Server.Handler.StaticFileSystem = &filesystem.StatikFS{} + m.Server.Handler.FileSystem = &statikfs.FileSystem{} // Default coordintor to port 0 when not specified so that coordinator // can be set to the value of server.URI after server binds to a port. diff --git a/filesystem/statik.go b/statikfs/filesystem.go similarity index 70% rename from filesystem/statik.go rename to statikfs/filesystem.go index 405e31a11..306034a1f 100644 --- a/filesystem/statik.go +++ b/statikfs/filesystem.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package filesystem +package statikfs import ( "net/http" @@ -21,12 +21,13 @@ import ( "github.com/rakyll/statik/fs" ) -// Ensure nopStaticFileSystem implements interface. -var _ pilosa.StaticFileSystem = &StatikFS{} +// Ensure nopFileSystem implements interface. +var _ pilosa.FileSystem = &FileSystem{} -type StatikFS struct{} +// FileSystem represents a static FileSystem. +type FileSystem struct{} -// New is a statik implementation of StaticFileSystem New method. -func (s *StatikFS) New() (http.FileSystem, error) { +// New is a statik implementation of FileSystem New method. +func (s *FileSystem) New() (http.FileSystem, error) { return fs.New() } From b59901b40361093d3bf013f16904ef8e29475954 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 7 Mar 2018 16:13:25 -0600 Subject: [PATCH 129/311] Linkify docs url --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index bf4fb3b91..31b60a622 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,5 @@ Pilosa docs are maintained here, to stay in sync with the codebase. -Please visit https://www.pilosa.com/docs to view the docs complete with styles, diagrams, and comprehensive search. +Please visit [our website](https://www.pilosa.com/docs) to view the docs complete with styles, diagrams, and comprehensive search. Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request! From 64bf4827aa73f99c634139afeb3e1b2db81127ad Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 7 Mar 2018 16:15:18 -0600 Subject: [PATCH 130/311] Minor fixes --- docs/administration.md | 10 +++++----- docs/architecture.md | 3 +++ docs/data-model.md | 10 ++++++++-- docs/examples.md | 2 +- docs/query-language.md | 3 ++- docs/tutorials.md | 2 ++ docs/webui.md | 3 ++- 7 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 97ee688f5..664301286 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -112,17 +112,17 @@ Note: This will only work when the replication factor is >= 2 #### Copying data files manually -- To accomplish this goal you will 1st need: - - List of all Indexes on your cluster - - List of all frames in your Indexes - - Max slice per Index, listed in the /status endpoint +- To accomplish this you will first need: + - List of all indexes on your cluster + - List of all frames in your indexes + - Max slice per index, listed in the /status endpoint - With this information you can query the `/fragment/nodes` endpoint and iterate over each slice - Using the list of slices owned by this node you will then need to manually: - setup a directory structure similar to the other nodes with a path for each Index/Frame - copy each owned slice for an existing node to this new node - Modify the cluster config file to replace the previous node address with the new node address. - Restart the cluster -- Wait for the 1st sync (10 minutes) to validate Index connections +- Wait for the first sync (10 minutes) to validate Index connections ### Diagnostics diff --git a/docs/architecture.md b/docs/architecture.md index 5cc5ff6ee..81a7e54f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,5 +18,8 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B * After the container storage section is an operation log, of unspecified length. ![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png) +*Pilosa Roaring storage format diagram* All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header. + +Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa. diff --git a/docs/data-model.md b/docs/data-model.md index d448a1de8..1fc5f899a 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -26,7 +26,8 @@ Pilosa lays out data first in rows, so queries which get all the set bits in one Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but if you try to set a bit with column ID 2^63, bad things will start to happen. -![data model diagram](/img/docs/data-model.svg) +![basic data model diagram](/img/docs/data-model.svg) +*Basic data model diagram* ### Index @@ -51,12 +52,14 @@ Row attributes are namespaced at the Frame level. Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation. ![ranked frame diagram](/img/docs/frame-ranked.svg) +*Ranked frame diagram* #### LRU The LRU cache maintains the most recently accessed Rows. ![lru frame diagram](/img/docs/frame-lru.svg) +*LRU frame diagram* ### Time Quantum @@ -92,6 +95,7 @@ SetBit(frame="A", rowID=19, columnID=5) ``` ![inverse frame diagram](/img/docs/frame-inverse.svg) +*Inverse frame diagram* #### Time Quantums @@ -103,6 +107,7 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") ``` ![time quantum frame diagram](/img/docs/frame-time-quantum.svg) +*Time quantum frame diagram* #### BSI Range-Encoding @@ -122,4 +127,5 @@ SetFieldValue(col=2, frame="A", field1=1) SetFieldValue(col=3, frame="A", field1=6) ``` -![BSI diagram](/img/docs/frame-bsi.svg) +![BSI frame diagram](/img/docs/frame-bsi.svg) +*BSI frame diagram* diff --git a/docs/examples.md b/docs/examples.md index bf5e76f29..262684615 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -214,7 +214,7 @@ T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. -All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase +All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a [Github repository](https://github.com/pilosa/chem-usecase). #### Data model diff --git a/docs/query-language.md b/docs/query-language.md index 3a5b94d60..83098fe0a 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -13,7 +13,7 @@ nav = [ ### Overview -This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: +This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: ``` {"results":[...]} @@ -380,6 +380,7 @@ have the attribute specified by `field` with one of the values specified in **Result Type:** array of key/count objects **Caveats:** + * Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order. * Frames with cache type lru will maintain an LRU (Least Recently Used) cache, thus a TopN() query on this type of frame will return bitmaps sorted in order of most recently set bit. * The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN() queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. diff --git a/docs/tutorials.md b/docs/tutorials.md index ffaf51d64..2ffb1b41d 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -49,6 +49,7 @@ openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.cr ``` The command above creates two files in the current directory: + * `pilosa.local.crt` is the SSL certificate. * `pilosa.local.key` is the private key file which must be kept as secret. @@ -130,6 +131,7 @@ key = "pilosa.local.gossip32" ``` Here is some explanation of the configuration items: + * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. * `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. diff --git a/docs/webui.md b/docs/webui.md index cef8e570d..86874807b 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -20,7 +20,8 @@ Each query's result will be displayed in the Output section along with the query The Console will keep a record of each query and its result with the latest query on top. -![console](/img/docs/webui-console.png) +![webUI console screenshot](/img/docs/webui-console.png) +*WebUI console screenshot* In addition to standard PQL, the console supports a few special commands, prefixed with `:`. From 2811ed7b09bda8d5fdb2394c2fc0a869e5890308 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 7 Mar 2018 16:15:45 -0600 Subject: [PATCH 131/311] Improve and linkify glossary --- docs/glossary.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/glossary.md b/docs/glossary.md index 998bda86b..4607973e9 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -6,50 +6,50 @@ nav = [] ## Glossary -Index: Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes. +[Index](../data-model/#index): An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes. -Column: Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index. +[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). -Row: Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index. +[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [bitmap][#bitmap]. -Bit: A bit is the intersection of a Row and Column. +[Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column). -Bitmap: The on-disk and in-memory representation of a Row. +[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). -Roaring Bitmap: [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses. +[Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. -Attribute: Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store. +[Attribute](../data-model/#attribute): Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store. -PQL: [Pilosa Query Language](/docs/query-language). +[PQL](../query-language/): Pilosa Query Language. -Frame: Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame. +[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. -View: Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. -Fragment: A Fragment is the intersection of a frame and slice in an index. +Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). -Slice: Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash. +[Slice](../data-model/#slice): [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). -SliceWidth: This is the default number of columns in a slice. +SliceWidth: This is the number of [columns](#column) in a [slice](#slice). By default, 220 or about one million. -MaxSlice: The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries. +MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#columns). This value is important for all [nodes](#node) to efficiently distribute queries. -Anti-entropy: A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies. +[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. -Node: An individual running instance of Pilosa server which belongs to a cluster. +Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). -Cluster: A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. +Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. -TopN: Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row. +TopN: A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). -Tanimoto: Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity. +[Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). -Protobuf: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. +[Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. -TOML: We use [TOML](https://github.com/toml-lang/toml) for our configuration file format. +[TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration). -Jump Consistent Hash: A fast, minimal memory, [consistent hash algorithm](https://arxiv.org/pdf/1406.2294v1.pdf) that evenly distributes the workload even when the number of buckets changes. +[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -Partition: The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associate data across the new cluster topography. +Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions (256 by default). Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. -Replica: A copy of a [fragment](#fragment) on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made). +[Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. From cf41e7c3f732badfbc64f8c677db0fc707b1486e Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 7 Mar 2018 16:17:05 -0600 Subject: [PATCH 132/311] Alphabetize glossary --- docs/glossary.md | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/glossary.md b/docs/glossary.md index 4607973e9..2d56bcbd3 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -6,50 +6,50 @@ nav = [] ## Glossary -[Index](../data-model/#index): An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes. +[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. -[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). - -[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [bitmap][#bitmap]. +[Attribute](../data-model/#attribute): Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store. [Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column). [Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). -[Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. +Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. -[Attribute](../data-model/#attribute): Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store. +[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). -[PQL](../query-language/): Pilosa Query Language. +Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). [Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. -[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[Index](../data-model/#index): An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes. -Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). +[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. + +MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#columns). This value is important for all [nodes](#node) to efficiently distribute queries. + +Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). + +Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions (256 by default). Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. + +[PQL](../query-language/): Pilosa Query Language. + +[Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. + +[Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. + +[Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. + +[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [bitmap][#bitmap]. [Slice](../data-model/#slice): [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). SliceWidth: This is the number of [columns](#column) in a [slice](#slice). By default, 220 or about one million. -MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#columns). This value is important for all [nodes](#node) to efficiently distribute queries. - -[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. - -Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). - -Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. - -TopN: A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). - [Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). -[Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. - [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration). -[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. +TopN: A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). -Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions (256 by default). Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. - -[Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. +[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. From a562e73fa4df109afdf7eefeea9e643a6a1cba33 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 7 Mar 2018 18:11:22 -0600 Subject: [PATCH 133/311] Minor updates --- docs/administration.md | 2 +- docs/data-model.md | 6 ++++-- docs/query-language.md | 4 ++-- docs/tutorials.md | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 664301286..bf3da9626 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -64,7 +64,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count ```
-

Note that you must first create a frame with Range Encoding enabled and a field. View Create Frame for more details.

+

Note that you must first create a frame with range-encoding enabled and a field. View Create Frame for more details.

#### Exporting diff --git a/docs/data-model.md b/docs/data-model.md index 1fc5f899a..9ab684203 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -63,7 +63,7 @@ The LRU cache maintains the most recently accessed Rows. ### Time Quantum -Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. +Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. ### Attribute @@ -112,7 +112,7 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") #### BSI Range-Encoding Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded -bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate, aggregate, and range queries on these BSI integers. +bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. @@ -129,3 +129,5 @@ SetFieldValue(col=3, frame="A", field1=6) ![BSI frame diagram](/img/docs/frame-bsi.svg) *BSI frame diagram* + +Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa. diff --git a/docs/query-language.md b/docs/query-language.md index 83098fe0a..56d7c2087 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -473,7 +473,7 @@ Returns bits that are true for the comparison operator. **Examples:** In our source data, commitactivity was counted over the last year. -The following greater-than Range query returns all repositories having more than 100 commits. +The following greater-than `Range` query returns all repositories having more than 100 commits. ``` Range(frame="stats", commitactivity > 100) @@ -513,7 +513,7 @@ Sum([BITMAP_CALL], , ) **Description:** -Returns the count and computed sum of all bitmap encoded integer values across the `field` in this `frame`. The optional Bitmap call filters the bits used in this computation. +Returns the count and computed sum of all BSI integer values across the `field` in this `frame`. The optional `Bitmap` call filters the bits used in this computation. **Result Type:** object with the computed sum and count of the bitmap field. diff --git a/docs/tutorials.md b/docs/tutorials.md index 2ffb1b41d..9e6364cf0 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -241,7 +241,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: ``` request From f371e1c2f2a21d59e76e07fa8c276a0f367449fe Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 7 Mar 2018 18:11:39 -0600 Subject: [PATCH 134/311] Add glossary terms --- docs/glossary.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/glossary.md b/docs/glossary.md index 2d56bcbd3..5b2a24ffd 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -12,12 +12,16 @@ nav = [] [Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column). -[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). +[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. + +[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range) and [Sum](#sum) queries. Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. [Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). +[Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range) and [Sum](#sum) queries. + Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). [Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. @@ -30,26 +34,34 @@ nav = [] Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). -Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions (256 by default). Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. +Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data. [PQL](../query-language/): Pilosa Query Language. [Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. +[Range](../query-lanuage/#range):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). + +[Range (BSI)](../query-lanuage/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). + [Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. [Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. -[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [bitmap][#bitmap]. +[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap). [Slice](../data-model/#slice): [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). -SliceWidth: This is the number of [columns](#column) in a [slice](#slice). By default, 220 or about one million. +SliceWidth: This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. -[Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). +[Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field). + +[Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). + +[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for time [Range](#range) queries. [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration). -TopN: A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). +[TopN](../query-language#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). -[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. From 0a6f2d07f6d39b31605edb3c5702fc529a93d87b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 8 Mar 2018 08:38:29 -0600 Subject: [PATCH 135/311] Move statik filesystem implemention to subpackage of statik package. --- Makefile | 4 ++-- handler.go | 2 -- handler_test.go | 4 ++-- server/server.go | 4 ++-- statik/.gitignore | 2 +- statik/doc.go | 1 + statikfs/filesystem.go => statik/filesystem/statik.go | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) rename statikfs/filesystem.go => statik/filesystem/statik.go (98%) diff --git a/Makefile b/Makefile index b1b545042..8fdbcea3d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test +.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -103,7 +103,7 @@ generate-protoc: .protoc-gen-gofast go generate github.com/pilosa/pilosa/internal generate-statik: statik - go generate github.com/pilosa/pilosa + go generate github.com/pilosa/pilosa/statik generate: generate-protoc generate-statik diff --git a/handler.go b/handler.go index 067219c8f..e4a707d8f 100644 --- a/handler.go +++ b/handler.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate statik -src=./webui - package pilosa import ( diff --git a/handler_test.go b/handler_test.go index 4f3f12f77..5d8af43a9 100644 --- a/handler_test.go +++ b/handler_test.go @@ -31,7 +31,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/statikfs" + statik "github.com/pilosa/pilosa/statik/filesystem" "github.com/pilosa/pilosa/test" ) @@ -1854,7 +1854,7 @@ func TestHandler_WebUI(t *testing.T) { h := test.NewHandler() h.Holder = hldr.Holder h.Cluster = test.NewCluster(1) - h.FileSystem = &statikfs.FileSystem{} + h.FileSystem = &statik.FileSystem{} w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) diff --git a/server/server.go b/server/server.go index 52df0e52e..67498d7d4 100644 --- a/server/server.go +++ b/server/server.go @@ -35,7 +35,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gossip" - "github.com/pilosa/pilosa/statikfs" + statik "github.com/pilosa/pilosa/statik/filesystem" "github.com/pilosa/pilosa/statsd" ) @@ -193,7 +193,7 @@ func (m *Command) SetupServer() error { m.Server.Cluster.RemoteClient = c // Statik file system. - m.Server.Handler.FileSystem = &statikfs.FileSystem{} + m.Server.Handler.FileSystem = &statik.FileSystem{} // Default coordintor to port 0 when not specified so that coordinator // can be set to the value of server.URI after server binds to a port. diff --git a/statik/.gitignore b/statik/.gitignore index 514ee40a1..485c0c57d 100644 --- a/statik/.gitignore +++ b/statik/.gitignore @@ -1 +1 @@ -statik.go +/statik.go diff --git a/statik/doc.go b/statik/doc.go index 85edd9e6f..9310eb508 100644 --- a/statik/doc.go +++ b/statik/doc.go @@ -1,3 +1,4 @@ // Package statik contains static assets for the Web UI. `go generate` will // produce statik.go, which is ignored by git. +//go:generate statik -src=../webui -dest=.. package statik diff --git a/statikfs/filesystem.go b/statik/filesystem/statik.go similarity index 98% rename from statikfs/filesystem.go rename to statik/filesystem/statik.go index 306034a1f..3f9582a71 100644 --- a/statikfs/filesystem.go +++ b/statik/filesystem/statik.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package statikfs +package filesystem import ( "net/http" From 1cc45b22a2edf49cab4b2bdf9d9e957c1d6f2e2c Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 5 Mar 2018 12:14:10 -0600 Subject: [PATCH 136/311] Change Config.Coordinator from URI to bool --- cluster.go | 30 ++++-- cluster_internal_test.go | 16 +-- cluster_test.go | 8 +- config.go | 4 +- ctl/server.go | 2 +- handler_test.go | 4 +- internal/private.pb.go | 210 +++++++++++++++++++++++---------------- internal/private.proto | 1 + server.go | 11 +- server/cluster_test.go | 56 +++++------ server/server.go | 27 ++--- test/cluster.go | 4 +- test/pilosa.go | 25 +++-- 13 files changed, 218 insertions(+), 180 deletions(-) diff --git a/cluster.go b/cluster.go index ee0912b09..8cdd1e922 100644 --- a/cluster.go +++ b/cluster.go @@ -66,8 +66,9 @@ const ( // Node represents a node in the cluster. type Node struct { - ID string `json:"id"` - URI URI `json:"uri"` + ID string `json:"id"` + URI URI `json:"uri"` + IsCoordinator bool `json:"isCoordinator"` } func (n Node) String() string { @@ -86,8 +87,9 @@ func EncodeNodes(a []*Node) []*internal.Node { // EncodeNode converts a Node into its internal representation. func EncodeNode(n *Node) *internal.Node { return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), + ID: n.ID, + URI: n.URI.Encode(), + IsCoordinator: n.IsCoordinator, } } @@ -106,8 +108,9 @@ func DecodeNodes(a []*internal.Node) []*Node { // DecodeNode converts a proto message into a Node. func DecodeNode(node *internal.Node) *Node { return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), + ID: node.ID, + URI: decodeURI(node.URI), + IsCoordinator: node.IsCoordinator, } } @@ -238,7 +241,7 @@ type Cluster struct { // Required for cluster Resize. Static bool // Static is primarily used for testing in a non-gossip environment. state string - Coordinator URI + Coordinator string Holder *Holder Broadcaster Broadcaster @@ -291,12 +294,12 @@ func (c *Cluster) logger() *log.Logger { // Coordinator returns the coordinator node. func (c *Cluster) CoordinatorNode() *Node { - return c.nodeByURI(c.Coordinator) + return c.nodeByID(c.Coordinator) } // IsCoordinator is true if this node is the coordinator. func (c *Cluster) IsCoordinator() bool { - return c.Static || c.Coordinator == c.Node.URI + return c.Coordinator == c.Node.ID } // SetCoordinator updates the Coordinator to n. @@ -308,8 +311,8 @@ func (c *Cluster) SetCoordinator(n *Node) bool { return false } - if c.Coordinator != newNode.URI { - c.Coordinator = newNode.URI + if c.Coordinator != newNode.ID { + c.Coordinator = newNode.ID return true } return false @@ -320,6 +323,11 @@ func (c *Cluster) SetCoordinator(n *Node) bool { func (c *Cluster) AddNode(node *Node) error { c.logger().Printf("add node %s to cluster on %s", node, c.Node) + // If the node being added is the coordinator, set it for this node. + if node.IsCoordinator { + c.Coordinator = node.ID + } + // add to cluster if !c.addNodeBasicSorted(node) { return nil diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 6c2c98e40..da9989760 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -180,8 +180,8 @@ func TestFragSources(t *testing.T) { "node0": []*internal.ResizeSource{}, "node1": []*internal.ResizeSource{}, "node2": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}}, "i", "f", "standard", uint64(2)}, + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -192,11 +192,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*internal.ResizeSource{ "node0": []*internal.ResizeSource{ - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}}, "i", "f", "standard", uint64(1)}, + {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, }, "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(2)}, + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -207,11 +207,11 @@ func TestFragSources(t *testing.T) { idx: idx, expected: map[string][]*internal.ResizeSource{ "node0": []*internal.ResizeSource{ - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}}, "i", "f", "standard", uint64(2)}, + {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, }, "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}}, "i", "f", "standard", uint64(3)}, + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, }, "node2": []*internal.ResizeSource{}, }, diff --git a/cluster_test.go b/cluster_test.go index 5a5fa4de4..5c5c5f89c 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -180,10 +180,10 @@ func TestCluster_Coordinator(t *testing.T) { c1 := *pilosa.NewCluster() c1.Node = node1 - c1.Coordinator = node1.URI + c1.Coordinator = node1.ID c2 := *pilosa.NewCluster() c2.Node = node2 - c2.Coordinator = node1.URI + c2.Coordinator = node1.ID t.Run("IsCoordinator", func(t *testing.T) { if !c1.IsCoordinator() { @@ -519,14 +519,14 @@ func TestCluster_SetCoordinator(t *testing.T) { // Set coordinator to the same value. if c.SetCoordinator(oldNode) { t.Errorf("did not expect coordinator to change") - } else if c.Coordinator != oldNode.URI { + } else if c.Coordinator != oldNode.ID { t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) } // Set coordinator to a new value. if !c.SetCoordinator(newNode) { t.Errorf("expected coordinator to change") - } else if c.Coordinator != newNode.URI { + } else if c.Coordinator != newNode.ID { t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) } }) diff --git a/config.go b/config.go index 4fa06fc51..17841c967 100644 --- a/config.go +++ b/config.go @@ -139,7 +139,7 @@ type Config struct { Cluster struct { Disabled bool `toml:"disabled"` - Coordinator string `toml:"coordinator"` + Coordinator bool `toml:"coordinator"` ReplicaN int `toml:"replicas"` Hosts []string `toml:"hosts"` LongQueryTime Duration `toml:"long-query-time"` @@ -183,7 +183,7 @@ func NewConfig() *Config { // Cluster config. c.Cluster.Disabled = DefaultClusterDisabled - // c.Cluster.Coordinator = "" + // c.Cluster.Coordinator = false c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Hosts = []string{} c.Cluster.LongQueryTime = Duration(time.Minute) diff --git a/ctl/server.go b/ctl/server.go index 18f4f6f93..80eb909d2 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -34,7 +34,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Cluster flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") - flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") + flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") diff --git a/handler_test.go b/handler_test.go index 1caf7bf99..0f527514d 100644 --- a/handler_test.go +++ b/handler_test.go @@ -147,7 +147,7 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101}}]}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101},"isCoordinator":false}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -1212,7 +1212,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"}},{"id":"node0","uri":{"scheme":"http","host":"host0"}}]`+"\n" { + } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { t.Fatalf("unexpected body: %q", body) } diff --git a/internal/private.pb.go b/internal/private.pb.go index cdf0dab40..4d2bcdd08 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -742,8 +742,9 @@ func (m *URI) GetPort() uint32 { } type Node struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` } func (m *Node) Reset() { *m = Node{} } @@ -765,6 +766,13 @@ func (m *Node) GetURI() *URI { return nil } +func (m *Node) GetIsCoordinator() bool { + if m != nil { + return m.IsCoordinator + } + return false +} + type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -2136,6 +2144,16 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { } i += n12 } + if m.IsCoordinator { + dAtA[i] = 0x18 + i++ + if m.IsCoordinator { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -3068,6 +3086,9 @@ func (m *Node) Size() (n int) { l = m.URI.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.IsCoordinator { + n += 2 + } return n } @@ -6575,6 +6596,26 @@ func (m *Node) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsCoordinator", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IsCoordinator = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8313,86 +8354,87 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1296 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0xb6, 0x63, 0x3f, 0xc7, 0x89, 0x33, 0x4d, 0x83, 0x13, 0x45, 0xae, 0x19, 0x01, - 0x0d, 0x95, 0x88, 0x8a, 0x2b, 0x01, 0x0d, 0xaa, 0x54, 0x12, 0xbb, 0xea, 0x02, 0x09, 0x65, 0x9c, - 0x06, 0x89, 0x03, 0xd2, 0xc4, 0x1e, 0xd2, 0x55, 0xd6, 0xbb, 0x66, 0x77, 0x9c, 0xc4, 0x3d, 0x70, - 0x44, 0x48, 0x88, 0x3b, 0xe2, 0xca, 0x97, 0xe1, 0xc8, 0x47, 0x40, 0xe1, 0x43, 0x20, 0x71, 0x01, - 0xcd, 0xbf, 0xdd, 0xf5, 0xdf, 0x90, 0xd0, 0xdb, 0xbe, 0xdf, 0xfb, 0x33, 0xbf, 0x79, 0xef, 0xcd, - 0x9b, 0x59, 0x28, 0xf7, 0x43, 0xf7, 0x8c, 0x72, 0xb6, 0xdd, 0x0f, 0x03, 0x1e, 0xa0, 0x82, 0xeb, - 0x73, 0x16, 0xfa, 0xd4, 0xc3, 0x9f, 0x43, 0xd1, 0xf1, 0xbb, 0xec, 0x62, 0x9f, 0x71, 0x8a, 0xea, - 0x50, 0xda, 0x0b, 0xbc, 0x41, 0xcf, 0xff, 0x8c, 0x1e, 0x33, 0xaf, 0x6a, 0xd5, 0xad, 0xad, 0x22, - 0x49, 0x43, 0xc2, 0xe2, 0xd0, 0xed, 0xb1, 0x2f, 0x06, 0xd4, 0xe7, 0x83, 0x5e, 0x35, 0xa3, 0x2c, - 0x52, 0x10, 0xfe, 0xdb, 0x82, 0xe2, 0x93, 0x90, 0xf6, 0x98, 0x8c, 0xb8, 0x01, 0x05, 0x12, 0x9c, - 0xa7, 0xc3, 0xc5, 0x32, 0x7a, 0x1b, 0x96, 0x1c, 0xff, 0x8c, 0x85, 0x11, 0x6b, 0xf9, 0xf4, 0xd8, - 0x63, 0x5d, 0x19, 0xae, 0x40, 0xc6, 0x50, 0xb4, 0x09, 0xc5, 0x3d, 0xda, 0x79, 0xc1, 0x0e, 0x87, - 0x7d, 0x56, 0xb5, 0x65, 0x90, 0x04, 0x88, 0xb5, 0x6d, 0xf7, 0x25, 0xab, 0x66, 0xeb, 0xd6, 0x56, - 0x99, 0x24, 0xc0, 0x38, 0xdf, 0xdc, 0x04, 0x5f, 0x84, 0x61, 0x91, 0x50, 0xff, 0x24, 0xe6, 0x90, - 0x97, 0x1c, 0x46, 0x30, 0x74, 0x17, 0xf2, 0x4f, 0x5c, 0xe6, 0x75, 0xa3, 0xea, 0x42, 0xdd, 0xde, - 0x2a, 0x35, 0x96, 0xb7, 0x4d, 0xfe, 0xb6, 0x25, 0x4e, 0xb4, 0x1a, 0x63, 0x58, 0x72, 0x7a, 0xfd, - 0x20, 0xe4, 0x84, 0x45, 0xfd, 0xc0, 0x8f, 0x18, 0xaa, 0x80, 0xdd, 0x0a, 0x43, 0xbd, 0x77, 0xf1, - 0x89, 0xbf, 0x83, 0xca, 0xae, 0x17, 0x74, 0x4e, 0x9b, 0x94, 0x53, 0xc2, 0xbe, 0x1d, 0xb0, 0x88, - 0xa3, 0x55, 0xc8, 0xc9, 0x2a, 0x68, 0x3b, 0x25, 0x08, 0x54, 0x66, 0x52, 0xa7, 0x59, 0x09, 0x02, - 0x95, 0xfe, 0x32, 0x15, 0x59, 0xa2, 0x04, 0x81, 0xb6, 0x3d, 0xb7, 0xa3, 0x52, 0x90, 0x25, 0x4a, - 0x40, 0x08, 0xb2, 0x47, 0x2e, 0x3b, 0xd7, 0xfb, 0x96, 0xdf, 0xd8, 0x81, 0x95, 0xd4, 0xfa, 0x9a, - 0xe6, 0x1a, 0xe4, 0x49, 0x70, 0xee, 0x34, 0xa3, 0xaa, 0x55, 0xb7, 0xb7, 0xb2, 0x44, 0x4b, 0x32, - 0xbb, 0xb2, 0xfc, 0x42, 0x95, 0x91, 0xaa, 0x04, 0xc0, 0xeb, 0x90, 0x93, 0xa9, 0x16, 0xbb, 0x4c, - 0x7c, 0xc5, 0x27, 0xfe, 0xc7, 0x82, 0xe2, 0x3e, 0xbd, 0x90, 0x34, 0x22, 0xf4, 0x08, 0x0a, 0x6d, - 0x4e, 0xfd, 0x2e, 0x0d, 0xbb, 0xd2, 0xa8, 0xd4, 0x78, 0x23, 0x49, 0x61, 0x6c, 0xb6, 0x6d, 0x6c, - 0x5a, 0x3e, 0x0f, 0x87, 0x24, 0x76, 0x41, 0x3b, 0xb0, 0xa0, 0x7b, 0x42, 0x72, 0x28, 0x35, 0xea, - 0xd3, 0xbc, 0xe3, 0xb6, 0x11, 0xce, 0xc6, 0x61, 0xe3, 0x23, 0x28, 0x8f, 0x84, 0x15, 0x5c, 0x4f, - 0xd9, 0xd0, 0x54, 0xe4, 0x94, 0x0d, 0x45, 0xee, 0xce, 0xa8, 0x37, 0x50, 0x79, 0xce, 0x12, 0x25, - 0xec, 0x64, 0x3e, 0xb4, 0x36, 0x76, 0x60, 0x31, 0x1d, 0xf5, 0x3a, 0xbe, 0xf8, 0x6b, 0x40, 0x7b, - 0x21, 0xa3, 0x9c, 0x49, 0x7a, 0xfb, 0x2c, 0x8a, 0xe8, 0x09, 0x9b, 0x5d, 0x69, 0x55, 0xbd, 0x4c, - 0xba, 0x7a, 0x9b, 0x50, 0x74, 0x22, 0xb3, 0x71, 0x5b, 0xf6, 0x65, 0x02, 0xe0, 0x7b, 0x80, 0x9a, - 0xcc, 0x63, 0x9c, 0xe9, 0xf3, 0x3b, 0x27, 0x3e, 0x6e, 0x1b, 0x2e, 0x57, 0xdb, 0xa2, 0xbb, 0x90, - 0x15, 0x47, 0x57, 0x52, 0x29, 0x35, 0x6e, 0x25, 0x99, 0x8e, 0xe7, 0x04, 0x91, 0x06, 0xd8, 0x35, - 0x41, 0xf5, 0x71, 0xbf, 0x62, 0x83, 0x53, 0x5a, 0xd9, 0x2c, 0x65, 0x8f, 0x2f, 0x15, 0x0f, 0x10, - 0xbd, 0xd4, 0x63, 0xb3, 0xd7, 0x9b, 0x2e, 0x85, 0x4f, 0x62, 0xb2, 0xe2, 0xa4, 0xde, 0x84, 0xec, - 0x5b, 0x90, 0x93, 0xbe, 0x9a, 0xed, 0xc4, 0x0c, 0x50, 0x5a, 0x7c, 0x14, 0x53, 0xbd, 0xe9, 0x42, - 0xab, 0xe9, 0x85, 0x8a, 0x26, 0xee, 0x57, 0xda, 0x56, 0x9c, 0xe9, 0x03, 0xe1, 0xa3, 0x22, 0xc9, - 0xef, 0xd9, 0x35, 0x1b, 0x4b, 0xa4, 0x88, 0x2d, 0x86, 0x40, 0x54, 0xb5, 0xeb, 0xb6, 0x88, 0x2d, - 0x05, 0xfc, 0x00, 0xf2, 0xed, 0xce, 0x0b, 0xd6, 0xa3, 0xe8, 0x1d, 0x71, 0xd2, 0xba, 0xec, 0x82, - 0x45, 0xfa, 0x9c, 0x2e, 0x8f, 0xd5, 0x9f, 0x18, 0x3d, 0xfe, 0xd1, 0xd2, 0x7b, 0x9a, 0xc1, 0x28, - 0x2f, 0xd7, 0x8e, 0xaa, 0xd9, 0x89, 0x91, 0x29, 0x70, 0xa2, 0xd5, 0xa8, 0x05, 0x15, 0xc7, 0xef, - 0x0f, 0x78, 0x93, 0x7d, 0xe3, 0xfa, 0x2e, 0x77, 0x03, 0x3f, 0xaa, 0xe6, 0xa5, 0xcb, 0x7a, 0x7a, - 0xe9, 0x11, 0x0b, 0x32, 0xe1, 0x82, 0xbf, 0xb7, 0x60, 0x79, 0x0c, 0xbc, 0x82, 0x57, 0x66, 0x3e, - 0xaf, 0xf7, 0xe3, 0x99, 0x6f, 0x4b, 0xc3, 0xda, 0x4c, 0x36, 0xa3, 0x57, 0xc0, 0xaf, 0x16, 0xac, - 0x4e, 0x33, 0x98, 0xca, 0xa6, 0x06, 0xf0, 0x2c, 0x74, 0x7b, 0x34, 0x1c, 0x7e, 0xca, 0x86, 0xfa, - 0xfa, 0x4b, 0x21, 0xe8, 0x4b, 0x58, 0x1b, 0x8b, 0xf5, 0x71, 0x47, 0xa5, 0x48, 0x91, 0xba, 0x33, - 0x93, 0x94, 0xb2, 0x23, 0x33, 0xdc, 0xf1, 0x5f, 0x16, 0xdc, 0x9e, 0xaa, 0x4a, 0x7a, 0xd2, 0x4a, - 0xf7, 0xe4, 0x3d, 0xa8, 0x1c, 0x89, 0xc9, 0xd6, 0x64, 0x11, 0x77, 0x7d, 0x2a, 0x2c, 0x75, 0xd3, - 0x4e, 0xe0, 0xc8, 0x81, 0x82, 0xc4, 0xf6, 0x69, 0x5f, 0xd3, 0x7c, 0xf7, 0x0a, 0x9a, 0xdb, 0xc6, - 0x5e, 0x0f, 0x7e, 0x23, 0x0a, 0x32, 0xf2, 0x22, 0x32, 0xb7, 0x9a, 0x14, 0xc4, 0x48, 0x1f, 0x71, - 0xb8, 0xd6, 0x58, 0x0e, 0x60, 0xd3, 0x8c, 0xc2, 0x11, 0x26, 0xf3, 0x4f, 0xea, 0x43, 0x80, 0xc4, - 0x54, 0x4f, 0x80, 0x39, 0xfd, 0x99, 0x32, 0xc6, 0x4f, 0x61, 0xd3, 0xcc, 0xe9, 0x6b, 0x2c, 0x68, - 0xba, 0x25, 0x93, 0x74, 0x0b, 0x6e, 0x81, 0xfd, 0x9c, 0x38, 0xe2, 0xae, 0x96, 0xa7, 0xd5, 0x94, - 0x48, 0x4b, 0xc2, 0xe5, 0x69, 0x10, 0x71, 0xe3, 0x22, 0xbe, 0x05, 0xf6, 0x2c, 0x08, 0xb9, 0x64, - 0x5c, 0x26, 0xf2, 0x1b, 0x7f, 0x00, 0xd9, 0x83, 0xa0, 0xcb, 0xd0, 0x12, 0x64, 0x9c, 0xa6, 0x8e, - 0x91, 0x71, 0x9a, 0xe8, 0x8e, 0x0c, 0xaf, 0x67, 0x48, 0x39, 0xd9, 0xdc, 0x73, 0xe2, 0x10, 0xa1, - 0xc1, 0x8f, 0xa1, 0x22, 0x1c, 0xdb, 0x9c, 0xf2, 0x78, 0x06, 0xaf, 0x41, 0x5e, 0x60, 0x71, 0x20, - 0x2d, 0xc9, 0x1b, 0x4d, 0xd8, 0x99, 0xd1, 0x26, 0x05, 0xfc, 0x93, 0x05, 0x60, 0x42, 0x0c, 0x22, - 0x84, 0x15, 0x13, 0xe9, 0x5a, 0x6a, 0x2c, 0x25, 0x4b, 0x0a, 0x94, 0x28, 0x96, 0xef, 0xa5, 0xde, - 0x11, 0x93, 0xf3, 0x2d, 0x56, 0x91, 0xd4, 0x6b, 0x63, 0xcb, 0x8c, 0x33, 0x5d, 0xa8, 0x4a, 0x62, - 0xaf, 0x70, 0x9d, 0x32, 0x71, 0x85, 0x95, 0xf7, 0xbc, 0x41, 0xc4, 0x59, 0xa8, 0x19, 0x89, 0xf7, - 0x8e, 0x02, 0xe2, 0x1d, 0x25, 0xc0, 0xf4, 0x4d, 0xa1, 0x37, 0x21, 0x27, 0x98, 0x9a, 0x33, 0x39, - 0xbe, 0x0d, 0xa5, 0xc4, 0x6d, 0x3d, 0xd5, 0xa7, 0xce, 0x01, 0x04, 0x59, 0xf9, 0xba, 0xd5, 0xa5, - 0x93, 0x0f, 0xdb, 0x0a, 0xd8, 0xfb, 0xae, 0xea, 0x35, 0x9b, 0x88, 0x4f, 0x89, 0xd0, 0x0b, 0x79, - 0x16, 0x04, 0x42, 0xc5, 0xbd, 0xbe, 0xa2, 0x9a, 0x59, 0xcc, 0xf1, 0x9b, 0xdc, 0x35, 0xe6, 0x81, - 0x68, 0xa7, 0x1e, 0x88, 0x6d, 0x58, 0x51, 0x0d, 0xfb, 0x2a, 0x83, 0xfe, 0x92, 0x81, 0x15, 0xc2, - 0x22, 0xf7, 0x25, 0x73, 0xfc, 0x88, 0x87, 0x83, 0x78, 0xd8, 0x7c, 0x12, 0x1c, 0xeb, 0x54, 0xdb, - 0x44, 0x09, 0x71, 0x5b, 0x64, 0xe6, 0xb4, 0xc5, 0x7d, 0xf1, 0xab, 0x12, 0x84, 0x5d, 0x31, 0x74, - 0x82, 0x50, 0x17, 0x7a, 0xdc, 0x34, 0x6d, 0x82, 0xee, 0xc3, 0x42, 0x3b, 0x18, 0x84, 0x9d, 0xf8, - 0x4a, 0x5a, 0x4b, 0xac, 0x15, 0x33, 0xa5, 0x26, 0xc6, 0x2c, 0xd5, 0x47, 0xb9, 0xf9, 0x7d, 0x84, - 0x1e, 0x8d, 0xf5, 0x91, 0xfc, 0x8b, 0x28, 0x35, 0x5e, 0x4f, 0x1c, 0x46, 0xd4, 0x64, 0xd4, 0x1a, - 0xff, 0x60, 0xc1, 0x62, 0x9a, 0xc2, 0x7f, 0x3a, 0x18, 0x71, 0x45, 0x32, 0x53, 0x2b, 0x62, 0x4f, - 0xab, 0x48, 0x36, 0xa9, 0x48, 0xf2, 0xe6, 0xcc, 0xa5, 0xde, 0x9c, 0xf8, 0x14, 0xd6, 0x27, 0xca, - 0xb4, 0x17, 0xf4, 0xfa, 0xa2, 0x1f, 0xfe, 0x47, 0xb9, 0x56, 0x21, 0xd7, 0x0a, 0x43, 0x5d, 0xa8, - 0x22, 0x51, 0x02, 0x7e, 0x08, 0xb7, 0xdb, 0x8c, 0xa7, 0x8a, 0x64, 0xba, 0xad, 0x0e, 0xf6, 0x01, - 0x3b, 0x9f, 0xb1, 0x7d, 0xa1, 0xc2, 0xbb, 0x50, 0x38, 0x0c, 0xfa, 0x81, 0x17, 0x9c, 0x0c, 0xaf, - 0x38, 0xb4, 0x55, 0x58, 0x50, 0x33, 0x49, 0x5d, 0xf9, 0x45, 0x62, 0x44, 0x7c, 0x4b, 0xb4, 0x64, - 0x87, 0x7a, 0x9d, 0x81, 0x47, 0x39, 0x93, 0x7f, 0x32, 0xd1, 0x6e, 0xe5, 0xb7, 0xcb, 0x9a, 0xf5, - 0xfb, 0x65, 0xcd, 0xfa, 0xe3, 0xb2, 0x66, 0xfd, 0xfc, 0x67, 0xed, 0xb5, 0xe3, 0xbc, 0xfc, 0x67, - 0x7e, 0xf0, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x14, 0x23, 0x92, 0x89, 0x44, 0x0f, 0x00, 0x00, + // 1306 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0x45, + 0x14, 0x66, 0xbd, 0xb6, 0x63, 0x1f, 0xd7, 0xa9, 0x33, 0x4d, 0x83, 0x13, 0x45, 0xae, 0x19, 0x15, + 0x1a, 0x2a, 0x11, 0x15, 0x57, 0x42, 0x34, 0xa8, 0x52, 0x89, 0xed, 0xaa, 0x0b, 0x24, 0x94, 0x71, + 0x1a, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x24, 0xee, 0x05, + 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0x65, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x08, 0x24, 0x6e, + 0x40, 0xf3, 0xb7, 0xbb, 0xfe, 0x0d, 0x09, 0xdc, 0xed, 0xf9, 0xe6, 0x9c, 0x33, 0xdf, 0x9c, 0xbf, + 0x99, 0x85, 0xf2, 0x20, 0x74, 0x4f, 0x29, 0x67, 0xdb, 0x83, 0x30, 0xe0, 0x01, 0x2a, 0xb8, 0x3e, + 0x67, 0xa1, 0x4f, 0x3d, 0xfc, 0x29, 0x14, 0x1d, 0xbf, 0xc7, 0xce, 0xf7, 0x18, 0xa7, 0xa8, 0x0e, + 0xa5, 0x66, 0xe0, 0x0d, 0xfb, 0xfe, 0x27, 0xf4, 0x88, 0x79, 0x55, 0xab, 0x6e, 0x6d, 0x15, 0x49, + 0x1a, 0x12, 0x1a, 0x07, 0x6e, 0x9f, 0x7d, 0x36, 0xa4, 0x3e, 0x1f, 0xf6, 0xab, 0x19, 0xa5, 0x91, + 0x82, 0xf0, 0x5f, 0x16, 0x14, 0x9f, 0x86, 0xb4, 0xcf, 0xa4, 0xc7, 0x0d, 0x28, 0x90, 0xe0, 0x2c, + 0xed, 0x2e, 0x96, 0xd1, 0x5b, 0xb0, 0xec, 0xf8, 0xa7, 0x2c, 0x8c, 0x58, 0xdb, 0xa7, 0x47, 0x1e, + 0xeb, 0x49, 0x77, 0x05, 0x32, 0x81, 0xa2, 0x4d, 0x28, 0x36, 0x69, 0xf7, 0x25, 0x3b, 0x18, 0x0d, + 0x58, 0xd5, 0x96, 0x4e, 0x12, 0x20, 0x5e, 0xed, 0xb8, 0xaf, 0x58, 0x35, 0x5b, 0xb7, 0xb6, 0xca, + 0x24, 0x01, 0x26, 0xf9, 0xe6, 0xa6, 0xf8, 0x22, 0x0c, 0x37, 0x08, 0xf5, 0x8f, 0x63, 0x0e, 0x79, + 0xc9, 0x61, 0x0c, 0x43, 0xf7, 0x20, 0xff, 0xd4, 0x65, 0x5e, 0x2f, 0xaa, 0x2e, 0xd5, 0xed, 0xad, + 0x52, 0xe3, 0xe6, 0xb6, 0x89, 0xdf, 0xb6, 0xc4, 0x89, 0x5e, 0xc6, 0x18, 0x96, 0x9d, 0xfe, 0x20, + 0x08, 0x39, 0x61, 0xd1, 0x20, 0xf0, 0x23, 0x86, 0x2a, 0x60, 0xb7, 0xc3, 0x50, 0x9f, 0x5d, 0x7c, + 0xe2, 0x6f, 0xa1, 0xb2, 0xeb, 0x05, 0xdd, 0x93, 0x16, 0xe5, 0x94, 0xb0, 0x6f, 0x86, 0x2c, 0xe2, + 0x68, 0x15, 0x72, 0x32, 0x0b, 0x5a, 0x4f, 0x09, 0x02, 0x95, 0x91, 0xd4, 0x61, 0x56, 0x82, 0x40, + 0xa5, 0xbd, 0x0c, 0x45, 0x96, 0x28, 0x41, 0xa0, 0x1d, 0xcf, 0xed, 0xaa, 0x10, 0x64, 0x89, 0x12, + 0x10, 0x82, 0xec, 0xa1, 0xcb, 0xce, 0xf4, 0xb9, 0xe5, 0x37, 0x76, 0x60, 0x25, 0xb5, 0xbf, 0xa6, + 0xb9, 0x06, 0x79, 0x12, 0x9c, 0x39, 0xad, 0xa8, 0x6a, 0xd5, 0xed, 0xad, 0x2c, 0xd1, 0x92, 0x8c, + 0xae, 0x4c, 0xbf, 0x58, 0xca, 0xc8, 0xa5, 0x04, 0xc0, 0xeb, 0x90, 0x93, 0xa1, 0x16, 0xa7, 0x4c, + 0x6c, 0xc5, 0x27, 0xfe, 0xdb, 0x82, 0xe2, 0x1e, 0x3d, 0x97, 0x34, 0x22, 0xf4, 0x18, 0x0a, 0x1d, + 0x4e, 0xfd, 0x1e, 0x0d, 0x7b, 0x52, 0xa9, 0xd4, 0x78, 0x23, 0x09, 0x61, 0xac, 0xb6, 0x6d, 0x74, + 0xda, 0x3e, 0x0f, 0x47, 0x24, 0x36, 0x41, 0x3b, 0xb0, 0xa4, 0x6b, 0x42, 0x72, 0x28, 0x35, 0xea, + 0xb3, 0xac, 0xe3, 0xb2, 0x11, 0xc6, 0xc6, 0x60, 0xe3, 0x03, 0x28, 0x8f, 0xb9, 0x15, 0x5c, 0x4f, + 0xd8, 0xc8, 0x64, 0xe4, 0x84, 0x8d, 0x44, 0xec, 0x4e, 0xa9, 0x37, 0x54, 0x71, 0xce, 0x12, 0x25, + 0xec, 0x64, 0xde, 0xb7, 0x36, 0x76, 0xe0, 0x46, 0xda, 0xeb, 0x55, 0x6c, 0xf1, 0x57, 0x80, 0x9a, + 0x21, 0xa3, 0x9c, 0x49, 0x7a, 0x7b, 0x2c, 0x8a, 0xe8, 0x31, 0x9b, 0x9f, 0x69, 0x95, 0xbd, 0x4c, + 0x3a, 0x7b, 0x9b, 0x50, 0x74, 0x22, 0x73, 0x70, 0x5b, 0xd6, 0x65, 0x02, 0xe0, 0xfb, 0x80, 0x5a, + 0xcc, 0x63, 0x9c, 0xe9, 0xfe, 0x5d, 0xe0, 0x1f, 0x77, 0x0c, 0x97, 0xcb, 0x75, 0xd1, 0x3d, 0xc8, + 0x8a, 0xd6, 0x95, 0x54, 0x4a, 0x8d, 0x5b, 0x49, 0xa4, 0xe3, 0x39, 0x41, 0xa4, 0x02, 0x76, 0x8d, + 0x53, 0xdd, 0xee, 0x97, 0x1c, 0x70, 0x46, 0x29, 0x9b, 0xad, 0xec, 0xc9, 0xad, 0xe2, 0x01, 0xa2, + 0xb7, 0x7a, 0x62, 0xce, 0x7a, 0xdd, 0xad, 0xf0, 0x71, 0x4c, 0x56, 0x74, 0xea, 0x75, 0xc8, 0xbe, + 0x09, 0x39, 0x69, 0xab, 0xd9, 0x4e, 0xcd, 0x00, 0xb5, 0x8a, 0x0f, 0x63, 0xaa, 0xd7, 0xdd, 0x68, + 0x35, 0xbd, 0x51, 0xd1, 0xf8, 0xfd, 0x42, 0xeb, 0x8a, 0x9e, 0xde, 0x17, 0x36, 0xca, 0x93, 0xfc, + 0x9e, 0x9f, 0xb3, 0x89, 0x40, 0x0a, 0xdf, 0x62, 0x08, 0x44, 0x55, 0xbb, 0x6e, 0x0b, 0xdf, 0x52, + 0xc0, 0x0f, 0x21, 0xdf, 0xe9, 0xbe, 0x64, 0x7d, 0x8a, 0xde, 0x16, 0x9d, 0xd6, 0x63, 0xe7, 0x2c, + 0xd2, 0x7d, 0x7a, 0x73, 0x22, 0xff, 0xc4, 0xac, 0xe3, 0x1f, 0x2c, 0x7d, 0xa6, 0x39, 0x8c, 0xf2, + 0x72, 0xef, 0xa8, 0x9a, 0x9d, 0x1a, 0x99, 0x02, 0x27, 0x7a, 0x19, 0xb5, 0xa1, 0xe2, 0xf8, 0x83, + 0x21, 0x6f, 0xb1, 0xaf, 0x5d, 0xdf, 0xe5, 0x6e, 0xe0, 0x47, 0xd5, 0xbc, 0x34, 0x59, 0x4f, 0x6f, + 0x3d, 0xa6, 0x41, 0xa6, 0x4c, 0xf0, 0x77, 0x16, 0xdc, 0x9c, 0x00, 0x2f, 0xe1, 0x95, 0x59, 0xcc, + 0xeb, 0xbd, 0x78, 0xe6, 0xdb, 0x52, 0xb1, 0x36, 0x97, 0xcd, 0xf8, 0x15, 0xf0, 0x8b, 0x05, 0xab, + 0xb3, 0x14, 0x66, 0xb2, 0xa9, 0x01, 0x3c, 0x0f, 0xdd, 0x3e, 0x0d, 0x47, 0x1f, 0xb3, 0x91, 0xbe, + 0xfe, 0x52, 0x08, 0xfa, 0x1c, 0xd6, 0x26, 0x7c, 0x7d, 0xd8, 0x55, 0x21, 0x52, 0xa4, 0xee, 0xcc, + 0x25, 0xa5, 0xf4, 0xc8, 0x1c, 0x73, 0xfc, 0xa7, 0x05, 0xb7, 0x67, 0x2e, 0x25, 0x35, 0x69, 0xa5, + 0x6b, 0xf2, 0x3e, 0x54, 0x0e, 0xc5, 0x64, 0x6b, 0xb1, 0x88, 0xbb, 0x3e, 0x15, 0x9a, 0xba, 0x68, + 0xa7, 0x70, 0xe4, 0x40, 0x41, 0x62, 0x7b, 0x74, 0xa0, 0x69, 0xbe, 0x73, 0x09, 0xcd, 0x6d, 0xa3, + 0xaf, 0x07, 0xbf, 0x11, 0x05, 0x19, 0x79, 0x11, 0x99, 0x5b, 0x4d, 0x0a, 0x62, 0xa4, 0x8f, 0x19, + 0x5c, 0x69, 0x2c, 0x07, 0xb0, 0x69, 0x46, 0xe1, 0x18, 0x93, 0xc5, 0x9d, 0xfa, 0x08, 0x20, 0x51, + 0xd5, 0x13, 0x60, 0x41, 0x7d, 0xa6, 0x94, 0xf1, 0x33, 0xd8, 0x34, 0x73, 0xfa, 0x0a, 0x1b, 0x9a, + 0x6a, 0xc9, 0x24, 0xd5, 0x82, 0xdb, 0x60, 0xbf, 0x20, 0x8e, 0xb8, 0xab, 0x65, 0xb7, 0x9a, 0x14, + 0x69, 0x49, 0x98, 0x3c, 0x0b, 0x22, 0x6e, 0x4c, 0xc4, 0xb7, 0xc0, 0x9e, 0x07, 0x21, 0x97, 0x8c, + 0xcb, 0x44, 0x7e, 0xe3, 0x2f, 0x21, 0xbb, 0x1f, 0xf4, 0x18, 0x5a, 0x86, 0x8c, 0xd3, 0xd2, 0x3e, + 0x32, 0x4e, 0x0b, 0xdd, 0x91, 0xee, 0xf5, 0x0c, 0x29, 0x27, 0x87, 0x7b, 0x41, 0x1c, 0x22, 0x37, + 0xbe, 0x0b, 0x65, 0x27, 0x6a, 0x06, 0x41, 0xd8, 0x13, 0xa9, 0x0e, 0x42, 0x7d, 0x27, 0x8d, 0x83, + 0xf8, 0x09, 0x54, 0x84, 0xfb, 0x0e, 0xa7, 0x3c, 0x9e, 0xd4, 0x6b, 0x90, 0x17, 0x58, 0xbc, 0x9d, + 0x96, 0xe4, 0xbd, 0x27, 0xf4, 0xcc, 0x00, 0x94, 0x02, 0xfe, 0xd1, 0x02, 0x30, 0x2e, 0x86, 0x11, + 0xc2, 0x8a, 0xaf, 0x34, 0x2d, 0x35, 0x96, 0x13, 0x62, 0x02, 0x25, 0xea, 0x2c, 0xef, 0xa6, 0x5e, + 0x1b, 0xd3, 0x53, 0x30, 0x5e, 0x22, 0xa9, 0x37, 0xc9, 0x96, 0x19, 0x7a, 0x3a, 0x9d, 0x95, 0x44, + 0x5f, 0xe1, 0x3a, 0xb0, 0xe2, 0xa2, 0x2b, 0x37, 0xbd, 0x61, 0xc4, 0x59, 0xa8, 0x19, 0x89, 0x57, + 0x91, 0x02, 0xe2, 0x13, 0x25, 0xc0, 0xec, 0x43, 0xa1, 0xbb, 0x90, 0x13, 0x4c, 0x4d, 0xe7, 0x4e, + 0x1e, 0x43, 0x2d, 0xe2, 0x8e, 0x9e, 0xfd, 0x33, 0xa7, 0x05, 0x82, 0xac, 0x7c, 0x03, 0xeb, 0x04, + 0xcb, 0xe7, 0x6f, 0x05, 0xec, 0x3d, 0x57, 0x55, 0xa4, 0x4d, 0xc4, 0xa7, 0x44, 0xe8, 0xb9, 0xec, + 0x18, 0x81, 0x50, 0x71, 0xfb, 0xaf, 0xa8, 0x92, 0x17, 0xd3, 0xfe, 0x3a, 0x37, 0x92, 0x79, 0x46, + 0xda, 0xa9, 0x67, 0x64, 0x07, 0x56, 0x54, 0x59, 0xff, 0x9f, 0x4e, 0x7f, 0xce, 0xc0, 0x0a, 0x61, + 0x91, 0xfb, 0x8a, 0x39, 0x7e, 0xc4, 0xc3, 0x61, 0x3c, 0x92, 0x3e, 0x0a, 0x8e, 0x74, 0xa8, 0x6d, + 0xa2, 0x84, 0xb8, 0x2c, 0x32, 0x0b, 0xca, 0xe2, 0x81, 0xf8, 0xa1, 0x19, 0xaf, 0xd7, 0x69, 0xd5, + 0xb4, 0x0a, 0x7a, 0x00, 0x4b, 0x9d, 0x60, 0x18, 0x76, 0xe3, 0x8b, 0x6b, 0x2d, 0xd1, 0x56, 0xcc, + 0xd4, 0x32, 0x31, 0x6a, 0xa9, 0x3a, 0xca, 0x2d, 0xae, 0x23, 0xf4, 0x78, 0xa2, 0x8e, 0xe4, 0xbf, + 0x46, 0xa9, 0xf1, 0x7a, 0x62, 0x30, 0xb6, 0x4c, 0xc6, 0xb5, 0xf1, 0xf7, 0x16, 0xdc, 0x48, 0x53, + 0xf8, 0x57, 0x8d, 0x11, 0x67, 0x24, 0x33, 0x33, 0x23, 0xf6, 0xac, 0x8c, 0x64, 0x93, 0x8c, 0x24, + 0x2f, 0xd3, 0x5c, 0xea, 0x65, 0x8a, 0x4f, 0x60, 0x7d, 0x2a, 0x4d, 0xcd, 0xa0, 0x3f, 0x10, 0xf5, + 0xf0, 0x1f, 0xd2, 0xb5, 0x0a, 0xb9, 0x76, 0x18, 0xea, 0x44, 0x15, 0x89, 0x12, 0xf0, 0x23, 0xb8, + 0xdd, 0x61, 0x3c, 0x95, 0x24, 0x53, 0x6d, 0x75, 0xb0, 0xf7, 0xd9, 0xd9, 0x9c, 0xe3, 0x8b, 0x25, + 0xbc, 0x0b, 0x85, 0x83, 0x60, 0x10, 0x78, 0xc1, 0xf1, 0xe8, 0x92, 0xa6, 0xad, 0xc2, 0x92, 0x9a, + 0x49, 0xea, 0x61, 0x50, 0x24, 0x46, 0xc4, 0xb7, 0x44, 0x49, 0x76, 0xa9, 0xd7, 0x1d, 0x7a, 0x94, + 0x33, 0xf9, 0xbf, 0x13, 0xed, 0x56, 0x7e, 0xbd, 0xa8, 0x59, 0xbf, 0x5d, 0xd4, 0xac, 0xdf, 0x2f, + 0x6a, 0xd6, 0x4f, 0x7f, 0xd4, 0x5e, 0x3b, 0xca, 0xcb, 0x3f, 0xeb, 0x87, 0xff, 0x04, 0x00, 0x00, + 0xff, 0xff, 0xd7, 0xef, 0xfa, 0x68, 0x6a, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index b012936d9..5f4a1c6f6 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -136,6 +136,7 @@ message URI { message Node { string ID = 1; URI URI = 2; + bool IsCoordinator = 3; } message NodeStateMessage { diff --git a/server.go b/server.go index be63a39e3..0dcca4d1e 100644 --- a/server.go +++ b/server.go @@ -132,7 +132,11 @@ func (s *Server) Open() error { s.NodeID = s.LoadNodeID() // Set Cluster Node. - node := &Node{ID: s.NodeID, URI: s.URI} + node := &Node{ + ID: s.NodeID, + URI: s.URI, + IsCoordinator: s.Cluster.Coordinator == s.NodeID, + } s.Cluster.Node = node // Append the NodeID tag to stats. @@ -185,11 +189,6 @@ func (s *Server) Open() error { return fmt.Errorf("starting BroadcastReceiver: %v", err) } - // If a Coordinator is not specified, then default to s.URI. - if s.Cluster.Coordinator.Port() == 0 { - s.Cluster.Coordinator = s.URI - } - // Open Cluster management. if err := s.Cluster.Open(); err != nil { return fmt.Errorf("opening Cluster: %v", err) diff --git a/server/cluster_test.go b/server/cluster_test.go index 299415bc3..81a410e83 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -52,7 +52,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Config.Gossip.Port = "0" m0.Config.Gossip.Seeds = []string{} - m0.Server.Cluster.Coordinator = m0.Server.URI + m0.Server.Cluster.Coordinator = m0.Server.NodeID m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput) gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) @@ -80,7 +80,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = gossipMemberSet0.Seeds() - m1.Server.Cluster.Coordinator = m0.Server.URI + m1.Server.Cluster.Coordinator = m0.Server.NodeID m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput) gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) if err != nil { @@ -220,21 +220,21 @@ func TestClusterResize_EmptyNode(t *testing.T) { // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster() + m0 := test.NewMainWithCluster(true) defer m0.Close() gossipHost := "localhost" gossipPort := 0 - seed, coord, err := m0.RunWithTransport(gossipHost, gossipPort, []string{}, pilosa.URI{}) + seed, err := m0.RunWithTransport(gossipHost, gossipPort, []string{}) if err != nil { t.Fatal(err) } // Configure node1 - m1 := test.NewMainWithCluster() + m1 := test.NewMainWithCluster(false) defer m1.Close() - seed, coord, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed}, coord) + seed, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed}) if err != nil { t.Fatal(err) } @@ -250,21 +250,21 @@ func TestClusterResize_EmptyNodes(t *testing.T) { func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster() + m0 := test.NewMainWithCluster(true) defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) + seed, err := m0.RunWithTransport("localhost", 0, []string{}) if err != nil { t.Fatal(err) } // Configure node1 - m1 := test.NewMainWithCluster() + m1 := test.NewMainWithCluster(false) defer m1.Close() var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) if err != nil { return err } @@ -284,10 +284,10 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster() + m0 := test.NewMainWithCluster(true) defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) + seed, err := m0.RunWithTransport("localhost", 0, []string{}) if err != nil { t.Fatal(err) } @@ -303,12 +303,12 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMainWithCluster() + m1 := test.NewMainWithCluster(false) defer m1.Close() var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) if err != nil { return err } @@ -330,10 +330,10 @@ func TestClusterResize_AddNode(t *testing.T) { t.Run("ContinuousSlices", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster() + m0 := test.NewMainWithCluster(true) defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) + seed, err := m0.RunWithTransport("localhost", 0, []string{}) if err != nil { t.Fatal(err) } @@ -358,12 +358,12 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMainWithCluster() + m1 := test.NewMainWithCluster(false) defer m1.Close() var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) if err != nil { return err } @@ -385,10 +385,10 @@ func TestClusterResize_AddNode(t *testing.T) { t.Run("SkippedSlice", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster() + m0 := test.NewMainWithCluster(true) defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) + seed, err := m0.RunWithTransport("localhost", 0, []string{}) if err != nil { t.Fatal(err) } @@ -413,12 +413,12 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMainWithCluster() + m1 := test.NewMainWithCluster(false) defer m1.Close() var eg errgroup.Group eg.Go(func() error { - _, _, err = m1.RunWithTransport("localhost", 0, []string{seed}, coord) + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) if err != nil { return err } @@ -443,22 +443,22 @@ func TestClusterResize_AddNode(t *testing.T) { func TestCluster_GossipMembership(t *testing.T) { t.Run("Node0Down", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster() + m0 := test.NewMainWithCluster(true) defer m0.Close() - seed, coord, err := m0.RunWithTransport("localhost", 0, []string{}, pilosa.URI{}) + seed, err := m0.RunWithTransport("localhost", 0, []string{}) if err != nil { t.Fatal(err) } // Configure node1 - m1 := test.NewMainWithCluster() + m1 := test.NewMainWithCluster(false) defer m1.Close() var eg errgroup.Group eg.Go(func() error { // Pass invalid seed as first in list - _, _, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}, coord) + _, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) if err != nil { return err } @@ -466,12 +466,12 @@ func TestCluster_GossipMembership(t *testing.T) { }) // Configure node2 - m2 := test.NewMainWithCluster() + m2 := test.NewMainWithCluster(false) defer m2.Close() eg.Go(func() error { // Pass invalid seed as last in list - _, _, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}, coord) + _, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) if err != nil { return err } diff --git a/server/server.go b/server/server.go index cd98ce594..240b8b16b 100644 --- a/server/server.go +++ b/server/server.go @@ -181,31 +181,13 @@ func (m *Command) SetupServer() error { InsecureSkipVerify: m.Config.TLS.SkipVerify, } - // TODO Review this location - TLSConfig = m.Server.TLS - } c := pilosa.GetHTTPClient(TLSConfig) m.Server.RemoteClient = c m.Server.Handler.RemoteClient = c m.Server.Cluster.RemoteClient = c - // Default coordintor to port 0 when not specified so that coordinator - // can be set to the value of server.URI after server binds to a port. - // This would only be useful in a one-node cluster. - coord := m.Config.Cluster.Coordinator - if coord == "" { - coord = ":0" - } - - // Set the coordinator node. - curi, err := pilosa.AddressWithDefaults(coord) - if err != nil { - return err - } - m.Server.Cluster.Coordinator = *curi - // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) @@ -214,8 +196,12 @@ func (m *Command) SetupServer() error { // SetupNetworking sets up internode communication based on the configuration. func (m *Command) SetupNetworking() error { + + m.Server.NodeID = m.Server.LoadNodeID() + if m.Config.Cluster.Disabled { m.Server.Cluster.Static = true + m.Server.Cluster.Coordinator = m.Server.NodeID for _, address := range m.Config.Cluster.Hosts { uri, err := pilosa.NewURIFromAddress(address) if err != nil { @@ -256,7 +242,10 @@ func (m *Command) SetupNetworking() error { } } - m.Server.NodeID = m.Server.LoadNodeID() + // Set Coordinator. + if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { + m.Server.Cluster.Coordinator = m.Server.NodeID + } m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput) gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) diff --git a/test/cluster.go b/test/cluster.go index 536641b13..8362ca964 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -49,7 +49,7 @@ func NewCluster(n int) *pilosa.Cluster { } c.Node = c.Nodes[0] - c.Coordinator = c.Nodes[0].URI + c.Coordinator = c.Nodes[0].ID return c } @@ -262,7 +262,7 @@ func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, err c.Holder = h c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) c.Node = node - c.Coordinator = t.common.Nodes[0].URI // the first node is the coordinator + c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator c.Broadcaster = t // add nodes diff --git a/test/pilosa.go b/test/pilosa.go index 8facad563..2bfd0c1a6 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -65,9 +65,10 @@ func NewMain() *Main { } // NewMainWithCluster returns a new instance of Main with clustering enabled. -func NewMainWithCluster() *Main { +func NewMainWithCluster(isCoordinator bool) *Main { m := NewMain() m.Config.Cluster.Disabled = false + m.Config.Cluster.Coordinator = isCoordinator return m } @@ -94,12 +95,11 @@ func runMainWithCluster(size int) ([]*Main, error) { gossipPort := 0 var err error var gossipSeeds = make([]string, size) - var coordinator pilosa.URI for i := 0; i < size; i++ { - m := NewMainWithCluster() + m := NewMainWithCluster(i == 0) - gossipSeeds[i], coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i], coordinator) + gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) if err != nil { return nil, errors.Wrap(err, "RunWithTransport") } @@ -146,7 +146,7 @@ func (m *Main) Reopen() error { } // RunWithTransport runs Main and returns the dynamically allocated gossip port. -func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string, coordinator pilosa.URI) (seed string, coord pilosa.URI, err error) { +func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (seed string, err error) { defer close(m.Started) /* @@ -166,19 +166,19 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string, c // SetupServer err = m.SetupServer() if err != nil { - return seed, coord, err + return seed, err } // Open server listener. err = m.Server.OpenListener() if err != nil { - return seed, coord, err + return seed, err } // Open gossip transport to use in SetupServer. transport, err := gossip.NewTransport(host, bindPort) if err != nil { - return seed, coord, err + return seed, err } m.GossipTransport = transport @@ -193,23 +193,22 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string, c // SetupNetworking err = m.SetupNetworking() if err != nil { - return seed, coord, err + return seed, err } if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { - return seed, coord, err + return seed, err } - m.Server.Cluster.Coordinator = coordinator m.Server.Cluster.Static = false // Initialize server. err = m.Server.Open() if err != nil { - return seed, coord, err + return seed, err } - return seed, m.Server.Cluster.Coordinator, nil + return seed, nil } // URL returns the base URL string for accessing the running program. From fa4e543e84c5ecaa4ea5222da2454d35085f54f7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 26 Feb 2018 15:41:04 -0600 Subject: [PATCH 137/311] send a NodeJoin event on startup for cases where a quick restart has occurred and memberlist is not aware of it --- broadcast.go | 5 + cluster.go | 31 ++- internal/private.pb.go | 421 +++++++++++++++++++++++++++++------------ internal/private.proto | 5 + server.go | 2 + 5 files changed, 338 insertions(+), 126 deletions(-) diff --git a/broadcast.go b/broadcast.go index 62d69e9ff..6dbb0f771 100644 --- a/broadcast.go +++ b/broadcast.go @@ -136,6 +136,7 @@ const ( MessageTypeSetCoordinator MessageTypeNodeState MessageTypeRecalculateCaches + MessageTypeNodeEvent ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -176,6 +177,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeNodeState case *internal.RecalculateCaches: typ = MessageTypeRecalculateCaches + case *internal.NodeEventMessage: + typ = MessageTypeNodeEvent default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -226,6 +229,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.NodeStateMessage{} case MessageTypeRecalculateCaches: m = &internal.RecalculateCaches{} + case MessageTypeNodeEvent: + m = &internal.NodeEventMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/cluster.go b/cluster.go index 8cdd1e922..b93e6105c 100644 --- a/cluster.go +++ b/cluster.go @@ -114,6 +114,13 @@ func DecodeNode(node *internal.Node) *Node { } } +func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { + return &NodeEvent{ + Event: NodeEventType(ne.Event), + Node: DecodeNode(ne.Node), + } +} + // Nodes represents a list of nodes. type Nodes []*Node @@ -897,6 +904,22 @@ func (c *Cluster) Open() error { // If not coordinator then wait for ClusterStatus from coordinator. if !c.IsCoordinator() { + // In the case where a node has been restarted and memberlist has + // not had enough time to determine the node went down/up, then + // the coorninator needs to be alerted that this node is back up + // (and now in a state of STARTING) so that it can be put to the correct + // cluster state. + // TODO: Because the normal code path already sends a NodeJoin event (via + // memberlist), this it a bit redundant in most cases. Perhaps determine + // that the node has been restarted and don't do this step. + msg := &internal.NodeEventMessage{ + Event: uint32(NodeJoin), + Node: EncodeNode(c.Node), + } + if err := c.Broadcaster.SendAsync(msg); err != nil { + return fmt.Errorf("sending restart NodeJoin: %v", err) + } + c.logger().Printf("wait for joining to complete") <-c.joining c.logger().Printf("joining has completed") @@ -1678,9 +1701,11 @@ func (c *Cluster) nodeJoin(node *Node) error { return nil } - // Don't do anything else if the cluster already contains the node. - if c.nodeByID(node.ID) != nil { - return nil + // If the cluster already contains the node, just send it the cluster status. + // This is useful in the case where a node is restarted or temporarily leaves + // the cluster. + if node := c.nodeByID(node.ID); node != nil { + return c.sendTo(node, c.Status()) } // If the holder does not yet contain data, go ahead and add the node. diff --git a/internal/private.pb.go b/internal/private.pb.go index 4d2bcdd08..37b109014 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -34,6 +34,7 @@ URI Node NodeStateMessage + NodeEventMessage NodeStatus ClusterStatus Field @@ -797,6 +798,30 @@ func (m *NodeStateMessage) GetState() string { return "" } +type NodeEventMessage struct { + Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` +} + +func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } +func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } +func (*NodeEventMessage) ProtoMessage() {} +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } + +func (m *NodeEventMessage) GetEvent() uint32 { + if m != nil { + return m.Event + } + return 0 +} + +func (m *NodeEventMessage) GetNode() *Node { + if m != nil { + return m.Node + } + return nil +} + type NodeStatus struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` @@ -806,7 +831,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *NodeStatus) GetNode() *Node { if m != nil { @@ -838,7 +863,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *ClusterStatus) GetClusterID() string { if m != nil { @@ -871,7 +896,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *Field) GetName() string { if m != nil { @@ -910,7 +935,7 @@ type CreateViewMessage struct { func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -942,7 +967,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -977,7 +1002,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -1032,7 +1057,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } func (m *ResizeSource) GetNode() *Node { if m != nil { @@ -1079,7 +1104,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{32} + return fileDescriptorPrivate, []int{33} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -1110,7 +1135,7 @@ type SetCoordinatorMessage struct { func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{33} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{34} } func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { @@ -1127,7 +1152,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{34} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{35} } func (m *Topology) GetClusterID() string { if m != nil { @@ -1149,7 +1174,7 @@ type RecalculateCaches struct { func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{35} } +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{36} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") @@ -1177,6 +1202,7 @@ func init() { proto.RegisterType((*URI)(nil), "internal.URI") proto.RegisterType((*Node)(nil), "internal.Node") proto.RegisterType((*NodeStateMessage)(nil), "internal.NodeStateMessage") + proto.RegisterType((*NodeEventMessage)(nil), "internal.NodeEventMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") proto.RegisterType((*Field)(nil), "internal.Field") @@ -2187,6 +2213,39 @@ func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *NodeEventMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Event != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Event)) + } + if m.Node != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n13, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n13 + } + return i, nil +} + func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2206,32 +2265,32 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n13, err := m.Node.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n13 - } - if m.MaxSlices != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n14, err := m.MaxSlices.MarshalTo(dAtA[i:]) + n14, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n14 } - if m.Schema != nil { - dAtA[i] = 0x1a + if m.MaxSlices != nil { + dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n15, err := m.Schema.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) + n15, err := m.MaxSlices.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n15 } + if m.Schema != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n16, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 + } return i, nil } @@ -2413,21 +2472,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n16, err := m.Node.MarshalTo(dAtA[i:]) + n17, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n17 } if m.Coordinator != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n17, err := m.Coordinator.MarshalTo(dAtA[i:]) + n18, err := m.Coordinator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n18 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -2445,21 +2504,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n18, err := m.Schema.MarshalTo(dAtA[i:]) + n19, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n19 } if m.ClusterStatus != nil { dAtA[i] = 0x32 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n19, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + n20, err := m.ClusterStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n20 } return i, nil } @@ -2483,11 +2542,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n20, err := m.Node.MarshalTo(dAtA[i:]) + n21, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n21 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2539,11 +2598,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n21, err := m.Node.MarshalTo(dAtA[i:]) + n22, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n22 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2573,11 +2632,11 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n22, err := m.New.MarshalTo(dAtA[i:]) + n23, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n23 } return i, nil } @@ -3106,6 +3165,19 @@ func (m *NodeStateMessage) Size() (n int) { return n } +func (m *NodeEventMessage) Size() (n int) { + var l int + _ = l + if m.Event != 0 { + n += 1 + sovPrivate(uint64(m.Event)) + } + if m.Node != nil { + l = m.Node.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *NodeStatus) Size() (n int) { var l int _ = l @@ -6745,6 +6817,108 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeEventMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeEventMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Event", wireType) + } + m.Event = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Event |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Node == nil { + m.Node = &Node{} + } + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -8354,87 +8528,88 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1306 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0x66, 0xbd, 0xb6, 0x63, 0x1f, 0xd7, 0xa9, 0x33, 0x4d, 0x83, 0x13, 0x45, 0xae, 0x19, 0x15, - 0x1a, 0x2a, 0x11, 0x15, 0x57, 0x42, 0x34, 0xa8, 0x52, 0x89, 0xed, 0xaa, 0x0b, 0x24, 0x94, 0x71, - 0x1a, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x24, 0xee, 0x05, - 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0x65, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x08, 0x24, 0x6e, - 0x40, 0xf3, 0xb7, 0xbb, 0xfe, 0x0d, 0x09, 0xdc, 0xed, 0xf9, 0xe6, 0x9c, 0x33, 0xdf, 0x9c, 0xbf, - 0x99, 0x85, 0xf2, 0x20, 0x74, 0x4f, 0x29, 0x67, 0xdb, 0x83, 0x30, 0xe0, 0x01, 0x2a, 0xb8, 0x3e, - 0x67, 0xa1, 0x4f, 0x3d, 0xfc, 0x29, 0x14, 0x1d, 0xbf, 0xc7, 0xce, 0xf7, 0x18, 0xa7, 0xa8, 0x0e, - 0xa5, 0x66, 0xe0, 0x0d, 0xfb, 0xfe, 0x27, 0xf4, 0x88, 0x79, 0x55, 0xab, 0x6e, 0x6d, 0x15, 0x49, - 0x1a, 0x12, 0x1a, 0x07, 0x6e, 0x9f, 0x7d, 0x36, 0xa4, 0x3e, 0x1f, 0xf6, 0xab, 0x19, 0xa5, 0x91, - 0x82, 0xf0, 0x5f, 0x16, 0x14, 0x9f, 0x86, 0xb4, 0xcf, 0xa4, 0xc7, 0x0d, 0x28, 0x90, 0xe0, 0x2c, - 0xed, 0x2e, 0x96, 0xd1, 0x5b, 0xb0, 0xec, 0xf8, 0xa7, 0x2c, 0x8c, 0x58, 0xdb, 0xa7, 0x47, 0x1e, - 0xeb, 0x49, 0x77, 0x05, 0x32, 0x81, 0xa2, 0x4d, 0x28, 0x36, 0x69, 0xf7, 0x25, 0x3b, 0x18, 0x0d, - 0x58, 0xd5, 0x96, 0x4e, 0x12, 0x20, 0x5e, 0xed, 0xb8, 0xaf, 0x58, 0x35, 0x5b, 0xb7, 0xb6, 0xca, - 0x24, 0x01, 0x26, 0xf9, 0xe6, 0xa6, 0xf8, 0x22, 0x0c, 0x37, 0x08, 0xf5, 0x8f, 0x63, 0x0e, 0x79, - 0xc9, 0x61, 0x0c, 0x43, 0xf7, 0x20, 0xff, 0xd4, 0x65, 0x5e, 0x2f, 0xaa, 0x2e, 0xd5, 0xed, 0xad, - 0x52, 0xe3, 0xe6, 0xb6, 0x89, 0xdf, 0xb6, 0xc4, 0x89, 0x5e, 0xc6, 0x18, 0x96, 0x9d, 0xfe, 0x20, - 0x08, 0x39, 0x61, 0xd1, 0x20, 0xf0, 0x23, 0x86, 0x2a, 0x60, 0xb7, 0xc3, 0x50, 0x9f, 0x5d, 0x7c, - 0xe2, 0x6f, 0xa1, 0xb2, 0xeb, 0x05, 0xdd, 0x93, 0x16, 0xe5, 0x94, 0xb0, 0x6f, 0x86, 0x2c, 0xe2, - 0x68, 0x15, 0x72, 0x32, 0x0b, 0x5a, 0x4f, 0x09, 0x02, 0x95, 0x91, 0xd4, 0x61, 0x56, 0x82, 0x40, - 0xa5, 0xbd, 0x0c, 0x45, 0x96, 0x28, 0x41, 0xa0, 0x1d, 0xcf, 0xed, 0xaa, 0x10, 0x64, 0x89, 0x12, - 0x10, 0x82, 0xec, 0xa1, 0xcb, 0xce, 0xf4, 0xb9, 0xe5, 0x37, 0x76, 0x60, 0x25, 0xb5, 0xbf, 0xa6, - 0xb9, 0x06, 0x79, 0x12, 0x9c, 0x39, 0xad, 0xa8, 0x6a, 0xd5, 0xed, 0xad, 0x2c, 0xd1, 0x92, 0x8c, - 0xae, 0x4c, 0xbf, 0x58, 0xca, 0xc8, 0xa5, 0x04, 0xc0, 0xeb, 0x90, 0x93, 0xa1, 0x16, 0xa7, 0x4c, - 0x6c, 0xc5, 0x27, 0xfe, 0xdb, 0x82, 0xe2, 0x1e, 0x3d, 0x97, 0x34, 0x22, 0xf4, 0x18, 0x0a, 0x1d, - 0x4e, 0xfd, 0x1e, 0x0d, 0x7b, 0x52, 0xa9, 0xd4, 0x78, 0x23, 0x09, 0x61, 0xac, 0xb6, 0x6d, 0x74, - 0xda, 0x3e, 0x0f, 0x47, 0x24, 0x36, 0x41, 0x3b, 0xb0, 0xa4, 0x6b, 0x42, 0x72, 0x28, 0x35, 0xea, - 0xb3, 0xac, 0xe3, 0xb2, 0x11, 0xc6, 0xc6, 0x60, 0xe3, 0x03, 0x28, 0x8f, 0xb9, 0x15, 0x5c, 0x4f, - 0xd8, 0xc8, 0x64, 0xe4, 0x84, 0x8d, 0x44, 0xec, 0x4e, 0xa9, 0x37, 0x54, 0x71, 0xce, 0x12, 0x25, - 0xec, 0x64, 0xde, 0xb7, 0x36, 0x76, 0xe0, 0x46, 0xda, 0xeb, 0x55, 0x6c, 0xf1, 0x57, 0x80, 0x9a, - 0x21, 0xa3, 0x9c, 0x49, 0x7a, 0x7b, 0x2c, 0x8a, 0xe8, 0x31, 0x9b, 0x9f, 0x69, 0x95, 0xbd, 0x4c, - 0x3a, 0x7b, 0x9b, 0x50, 0x74, 0x22, 0x73, 0x70, 0x5b, 0xd6, 0x65, 0x02, 0xe0, 0xfb, 0x80, 0x5a, - 0xcc, 0x63, 0x9c, 0xe9, 0xfe, 0x5d, 0xe0, 0x1f, 0x77, 0x0c, 0x97, 0xcb, 0x75, 0xd1, 0x3d, 0xc8, - 0x8a, 0xd6, 0x95, 0x54, 0x4a, 0x8d, 0x5b, 0x49, 0xa4, 0xe3, 0x39, 0x41, 0xa4, 0x02, 0x76, 0x8d, - 0x53, 0xdd, 0xee, 0x97, 0x1c, 0x70, 0x46, 0x29, 0x9b, 0xad, 0xec, 0xc9, 0xad, 0xe2, 0x01, 0xa2, - 0xb7, 0x7a, 0x62, 0xce, 0x7a, 0xdd, 0xad, 0xf0, 0x71, 0x4c, 0x56, 0x74, 0xea, 0x75, 0xc8, 0xbe, - 0x09, 0x39, 0x69, 0xab, 0xd9, 0x4e, 0xcd, 0x00, 0xb5, 0x8a, 0x0f, 0x63, 0xaa, 0xd7, 0xdd, 0x68, - 0x35, 0xbd, 0x51, 0xd1, 0xf8, 0xfd, 0x42, 0xeb, 0x8a, 0x9e, 0xde, 0x17, 0x36, 0xca, 0x93, 0xfc, - 0x9e, 0x9f, 0xb3, 0x89, 0x40, 0x0a, 0xdf, 0x62, 0x08, 0x44, 0x55, 0xbb, 0x6e, 0x0b, 0xdf, 0x52, - 0xc0, 0x0f, 0x21, 0xdf, 0xe9, 0xbe, 0x64, 0x7d, 0x8a, 0xde, 0x16, 0x9d, 0xd6, 0x63, 0xe7, 0x2c, - 0xd2, 0x7d, 0x7a, 0x73, 0x22, 0xff, 0xc4, 0xac, 0xe3, 0x1f, 0x2c, 0x7d, 0xa6, 0x39, 0x8c, 0xf2, - 0x72, 0xef, 0xa8, 0x9a, 0x9d, 0x1a, 0x99, 0x02, 0x27, 0x7a, 0x19, 0xb5, 0xa1, 0xe2, 0xf8, 0x83, - 0x21, 0x6f, 0xb1, 0xaf, 0x5d, 0xdf, 0xe5, 0x6e, 0xe0, 0x47, 0xd5, 0xbc, 0x34, 0x59, 0x4f, 0x6f, - 0x3d, 0xa6, 0x41, 0xa6, 0x4c, 0xf0, 0x77, 0x16, 0xdc, 0x9c, 0x00, 0x2f, 0xe1, 0x95, 0x59, 0xcc, - 0xeb, 0xbd, 0x78, 0xe6, 0xdb, 0x52, 0xb1, 0x36, 0x97, 0xcd, 0xf8, 0x15, 0xf0, 0x8b, 0x05, 0xab, - 0xb3, 0x14, 0x66, 0xb2, 0xa9, 0x01, 0x3c, 0x0f, 0xdd, 0x3e, 0x0d, 0x47, 0x1f, 0xb3, 0x91, 0xbe, - 0xfe, 0x52, 0x08, 0xfa, 0x1c, 0xd6, 0x26, 0x7c, 0x7d, 0xd8, 0x55, 0x21, 0x52, 0xa4, 0xee, 0xcc, - 0x25, 0xa5, 0xf4, 0xc8, 0x1c, 0x73, 0xfc, 0xa7, 0x05, 0xb7, 0x67, 0x2e, 0x25, 0x35, 0x69, 0xa5, - 0x6b, 0xf2, 0x3e, 0x54, 0x0e, 0xc5, 0x64, 0x6b, 0xb1, 0x88, 0xbb, 0x3e, 0x15, 0x9a, 0xba, 0x68, - 0xa7, 0x70, 0xe4, 0x40, 0x41, 0x62, 0x7b, 0x74, 0xa0, 0x69, 0xbe, 0x73, 0x09, 0xcd, 0x6d, 0xa3, - 0xaf, 0x07, 0xbf, 0x11, 0x05, 0x19, 0x79, 0x11, 0x99, 0x5b, 0x4d, 0x0a, 0x62, 0xa4, 0x8f, 0x19, - 0x5c, 0x69, 0x2c, 0x07, 0xb0, 0x69, 0x46, 0xe1, 0x18, 0x93, 0xc5, 0x9d, 0xfa, 0x08, 0x20, 0x51, - 0xd5, 0x13, 0x60, 0x41, 0x7d, 0xa6, 0x94, 0xf1, 0x33, 0xd8, 0x34, 0x73, 0xfa, 0x0a, 0x1b, 0x9a, - 0x6a, 0xc9, 0x24, 0xd5, 0x82, 0xdb, 0x60, 0xbf, 0x20, 0x8e, 0xb8, 0xab, 0x65, 0xb7, 0x9a, 0x14, - 0x69, 0x49, 0x98, 0x3c, 0x0b, 0x22, 0x6e, 0x4c, 0xc4, 0xb7, 0xc0, 0x9e, 0x07, 0x21, 0x97, 0x8c, - 0xcb, 0x44, 0x7e, 0xe3, 0x2f, 0x21, 0xbb, 0x1f, 0xf4, 0x18, 0x5a, 0x86, 0x8c, 0xd3, 0xd2, 0x3e, - 0x32, 0x4e, 0x0b, 0xdd, 0x91, 0xee, 0xf5, 0x0c, 0x29, 0x27, 0x87, 0x7b, 0x41, 0x1c, 0x22, 0x37, - 0xbe, 0x0b, 0x65, 0x27, 0x6a, 0x06, 0x41, 0xd8, 0x13, 0xa9, 0x0e, 0x42, 0x7d, 0x27, 0x8d, 0x83, - 0xf8, 0x09, 0x54, 0x84, 0xfb, 0x0e, 0xa7, 0x3c, 0x9e, 0xd4, 0x6b, 0x90, 0x17, 0x58, 0xbc, 0x9d, - 0x96, 0xe4, 0xbd, 0x27, 0xf4, 0xcc, 0x00, 0x94, 0x02, 0xfe, 0xd1, 0x02, 0x30, 0x2e, 0x86, 0x11, - 0xc2, 0x8a, 0xaf, 0x34, 0x2d, 0x35, 0x96, 0x13, 0x62, 0x02, 0x25, 0xea, 0x2c, 0xef, 0xa6, 0x5e, - 0x1b, 0xd3, 0x53, 0x30, 0x5e, 0x22, 0xa9, 0x37, 0xc9, 0x96, 0x19, 0x7a, 0x3a, 0x9d, 0x95, 0x44, - 0x5f, 0xe1, 0x3a, 0xb0, 0xe2, 0xa2, 0x2b, 0x37, 0xbd, 0x61, 0xc4, 0x59, 0xa8, 0x19, 0x89, 0x57, - 0x91, 0x02, 0xe2, 0x13, 0x25, 0xc0, 0xec, 0x43, 0xa1, 0xbb, 0x90, 0x13, 0x4c, 0x4d, 0xe7, 0x4e, - 0x1e, 0x43, 0x2d, 0xe2, 0x8e, 0x9e, 0xfd, 0x33, 0xa7, 0x05, 0x82, 0xac, 0x7c, 0x03, 0xeb, 0x04, - 0xcb, 0xe7, 0x6f, 0x05, 0xec, 0x3d, 0x57, 0x55, 0xa4, 0x4d, 0xc4, 0xa7, 0x44, 0xe8, 0xb9, 0xec, - 0x18, 0x81, 0x50, 0x71, 0xfb, 0xaf, 0xa8, 0x92, 0x17, 0xd3, 0xfe, 0x3a, 0x37, 0x92, 0x79, 0x46, - 0xda, 0xa9, 0x67, 0x64, 0x07, 0x56, 0x54, 0x59, 0xff, 0x9f, 0x4e, 0x7f, 0xce, 0xc0, 0x0a, 0x61, - 0x91, 0xfb, 0x8a, 0x39, 0x7e, 0xc4, 0xc3, 0x61, 0x3c, 0x92, 0x3e, 0x0a, 0x8e, 0x74, 0xa8, 0x6d, - 0xa2, 0x84, 0xb8, 0x2c, 0x32, 0x0b, 0xca, 0xe2, 0x81, 0xf8, 0xa1, 0x19, 0xaf, 0xd7, 0x69, 0xd5, - 0xb4, 0x0a, 0x7a, 0x00, 0x4b, 0x9d, 0x60, 0x18, 0x76, 0xe3, 0x8b, 0x6b, 0x2d, 0xd1, 0x56, 0xcc, - 0xd4, 0x32, 0x31, 0x6a, 0xa9, 0x3a, 0xca, 0x2d, 0xae, 0x23, 0xf4, 0x78, 0xa2, 0x8e, 0xe4, 0xbf, - 0x46, 0xa9, 0xf1, 0x7a, 0x62, 0x30, 0xb6, 0x4c, 0xc6, 0xb5, 0xf1, 0xf7, 0x16, 0xdc, 0x48, 0x53, - 0xf8, 0x57, 0x8d, 0x11, 0x67, 0x24, 0x33, 0x33, 0x23, 0xf6, 0xac, 0x8c, 0x64, 0x93, 0x8c, 0x24, - 0x2f, 0xd3, 0x5c, 0xea, 0x65, 0x8a, 0x4f, 0x60, 0x7d, 0x2a, 0x4d, 0xcd, 0xa0, 0x3f, 0x10, 0xf5, - 0xf0, 0x1f, 0xd2, 0xb5, 0x0a, 0xb9, 0x76, 0x18, 0xea, 0x44, 0x15, 0x89, 0x12, 0xf0, 0x23, 0xb8, - 0xdd, 0x61, 0x3c, 0x95, 0x24, 0x53, 0x6d, 0x75, 0xb0, 0xf7, 0xd9, 0xd9, 0x9c, 0xe3, 0x8b, 0x25, - 0xbc, 0x0b, 0x85, 0x83, 0x60, 0x10, 0x78, 0xc1, 0xf1, 0xe8, 0x92, 0xa6, 0xad, 0xc2, 0x92, 0x9a, - 0x49, 0xea, 0x61, 0x50, 0x24, 0x46, 0xc4, 0xb7, 0x44, 0x49, 0x76, 0xa9, 0xd7, 0x1d, 0x7a, 0x94, - 0x33, 0xf9, 0xbf, 0x13, 0xed, 0x56, 0x7e, 0xbd, 0xa8, 0x59, 0xbf, 0x5d, 0xd4, 0xac, 0xdf, 0x2f, - 0x6a, 0xd6, 0x4f, 0x7f, 0xd4, 0x5e, 0x3b, 0xca, 0xcb, 0x3f, 0xeb, 0x87, 0xff, 0x04, 0x00, 0x00, - 0xff, 0xff, 0xd7, 0xef, 0xfa, 0x68, 0x6a, 0x0f, 0x00, 0x00, + // 1325 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0xb6, 0x13, 0xbf, 0xd4, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15, + 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0xb4, 0xa8, 0x52, 0xa9, 0xed, 0xaa, 0x0b, 0x4d, 0x29, 0xe3, + 0xb6, 0x48, 0x48, 0x20, 0x4d, 0xec, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03, + 0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x04, 0x54, 0x3e, 0x04, 0x12, 0x17, + 0xd0, 0xbc, 0x99, 0xd9, 0x5d, 0xff, 0x4b, 0x9a, 0xc0, 0x6d, 0xdf, 0x6f, 0xde, 0x7b, 0xf3, 0x9b, + 0xf7, 0x6f, 0x66, 0xa1, 0x3a, 0x8c, 0xfd, 0x03, 0x2e, 0xc5, 0xd6, 0x30, 0x8e, 0x64, 0x44, 0x96, + 0xfd, 0x50, 0x8a, 0x38, 0xe4, 0x01, 0xfd, 0x0c, 0x2a, 0x5e, 0xd8, 0x17, 0x47, 0x3b, 0x42, 0x72, + 0xd2, 0x84, 0x95, 0x56, 0x14, 0x8c, 0x06, 0xe1, 0x23, 0xbe, 0x2b, 0x82, 0xba, 0xd3, 0x74, 0x36, + 0x2b, 0x2c, 0x0f, 0x29, 0x8d, 0xa7, 0xfe, 0x40, 0x7c, 0x3e, 0xe2, 0xa1, 0x1c, 0x0d, 0xea, 0x05, + 0xad, 0x91, 0x83, 0xe8, 0xdf, 0x0e, 0x54, 0x1e, 0xc4, 0x7c, 0x20, 0xd0, 0xe3, 0x15, 0x58, 0x66, + 0xd1, 0x61, 0xde, 0x5d, 0x2a, 0x93, 0x77, 0x60, 0xd5, 0x0b, 0x0f, 0x44, 0x9c, 0x88, 0x4e, 0xc8, + 0x77, 0x03, 0xd1, 0x47, 0x77, 0xcb, 0x6c, 0x0a, 0x25, 0x1b, 0x50, 0x69, 0xf1, 0xde, 0x0b, 0xf1, + 0x74, 0x3c, 0x14, 0x75, 0x17, 0x9d, 0x64, 0x40, 0xba, 0xda, 0xf5, 0x5f, 0x8a, 0x7a, 0xb1, 0xe9, + 0x6c, 0x56, 0x59, 0x06, 0x4c, 0xf3, 0x2d, 0xcd, 0xf0, 0x25, 0x14, 0xce, 0x31, 0x1e, 0xee, 0xa5, + 0x1c, 0xca, 0xc8, 0x61, 0x02, 0x23, 0xd7, 0xa1, 0xfc, 0xc0, 0x17, 0x41, 0x3f, 0xa9, 0x2f, 0x35, + 0xdd, 0xcd, 0x95, 0xed, 0xf3, 0x5b, 0x36, 0x7e, 0x5b, 0x88, 0x33, 0xb3, 0x4c, 0x29, 0xac, 0x7a, + 0x83, 0x61, 0x14, 0x4b, 0x26, 0x92, 0x61, 0x14, 0x26, 0x82, 0xd4, 0xc0, 0xed, 0xc4, 0xb1, 0x39, + 0xbb, 0xfa, 0xa4, 0xdf, 0x41, 0xed, 0x7e, 0x10, 0xf5, 0xf6, 0xdb, 0x5c, 0x72, 0x26, 0xbe, 0x1d, + 0x89, 0x44, 0x92, 0x8b, 0x50, 0xc2, 0x2c, 0x18, 0x3d, 0x2d, 0x28, 0x14, 0x23, 0x69, 0xc2, 0xac, + 0x05, 0x85, 0xa2, 0x3d, 0x86, 0xa2, 0xc8, 0xb4, 0xa0, 0xd0, 0x6e, 0xe0, 0xf7, 0x74, 0x08, 0x8a, + 0x4c, 0x0b, 0x84, 0x40, 0xf1, 0xb9, 0x2f, 0x0e, 0xcd, 0xb9, 0xf1, 0x9b, 0x7a, 0xb0, 0x96, 0xdb, + 0xdf, 0xd0, 0x5c, 0x87, 0x32, 0x8b, 0x0e, 0xbd, 0x76, 0x52, 0x77, 0x9a, 0xee, 0x66, 0x91, 0x19, + 0x09, 0xa3, 0x8b, 0xe9, 0x57, 0x4b, 0x05, 0x5c, 0xca, 0x00, 0x7a, 0x19, 0x4a, 0x18, 0x6a, 0x75, + 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8, 0xec, 0xf0, 0x23, 0xa4, 0x91, 0x90, 0xbb, 0xb0, + 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a, 0x2b, 0xdb, 0x6f, 0x65, 0x21, 0x4c, 0xd5, 0xb6, + 0xac, 0x4e, 0x27, 0x94, 0xf1, 0x98, 0xa5, 0x26, 0xe4, 0x0e, 0x2c, 0x99, 0x9a, 0x40, 0x0e, 0x2b, + 0xdb, 0xcd, 0x79, 0xd6, 0x69, 0xd9, 0x28, 0x63, 0x6b, 0x70, 0xe5, 0x23, 0xa8, 0x4e, 0xb8, 0x55, + 0x5c, 0xf7, 0xc5, 0xd8, 0x66, 0x64, 0x5f, 0x8c, 0x55, 0xec, 0x0e, 0x78, 0x30, 0xd2, 0x71, 0x2e, + 0x32, 0x2d, 0xdc, 0x29, 0x7c, 0xe8, 0x5c, 0xb9, 0x03, 0xe7, 0xf2, 0x5e, 0x4f, 0x63, 0x4b, 0xbf, + 0x06, 0xd2, 0x8a, 0x05, 0x97, 0x02, 0xe9, 0xed, 0x88, 0x24, 0xe1, 0x7b, 0x62, 0x71, 0xa6, 0x75, + 0xf6, 0x0a, 0xf9, 0xec, 0x6d, 0x40, 0xc5, 0x4b, 0xec, 0xc1, 0x5d, 0xac, 0xcb, 0x0c, 0xa0, 0x37, + 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0xfd, 0x7b, 0x8c, 0x7f, 0xda, 0xb5, 0x5c, 0x4e, 0xd6, 0x25, + 0xd7, 0xa1, 0xa8, 0x5a, 0x17, 0xa9, 0xac, 0x6c, 0x5f, 0xc8, 0x22, 0x9d, 0xce, 0x09, 0x86, 0x0a, + 0xd4, 0xb7, 0x4e, 0x4d, 0xbb, 0x9f, 0x70, 0xc0, 0x39, 0xa5, 0x6c, 0xb7, 0x72, 0xa7, 0xb7, 0x4a, + 0x07, 0x88, 0xd9, 0xea, 0x9e, 0x3d, 0xeb, 0x59, 0xb7, 0xa2, 0x7b, 0x29, 0x59, 0xd5, 0xa9, 0x67, + 0x21, 0xfb, 0x36, 0x94, 0xd0, 0xd6, 0xb0, 0x9d, 0x99, 0x01, 0x7a, 0x95, 0x3e, 0x4f, 0xa9, 0x9e, + 0x75, 0xa3, 0x8b, 0xf9, 0x8d, 0x2a, 0xd6, 0xef, 0x97, 0x46, 0x57, 0xf5, 0xf4, 0x63, 0x65, 0xa3, + 0x3d, 0xe1, 0xf7, 0xe2, 0x9c, 0x4d, 0x05, 0x52, 0xf9, 0x56, 0x43, 0x20, 0xa9, 0xbb, 0x4d, 0x57, + 0xf9, 0x46, 0x81, 0xde, 0x82, 0x72, 0xb7, 0xf7, 0x42, 0x0c, 0x38, 0x79, 0x57, 0x75, 0x5a, 0x5f, + 0x1c, 0x89, 0xc4, 0xf4, 0xe9, 0xf9, 0xa9, 0xfc, 0x33, 0xbb, 0x4e, 0x7f, 0x74, 0xcc, 0x99, 0x16, + 0x30, 0x2a, 0xe3, 0xde, 0x49, 0xbd, 0x38, 0x33, 0x32, 0x15, 0xce, 0xcc, 0x32, 0xe9, 0x40, 0xcd, + 0x0b, 0x87, 0x23, 0xd9, 0x16, 0xdf, 0xf8, 0xa1, 0x2f, 0xfd, 0x28, 0x4c, 0xea, 0x65, 0x34, 0xb9, + 0x9c, 0xdf, 0x7a, 0x42, 0x83, 0xcd, 0x98, 0xd0, 0xef, 0x1d, 0x38, 0x3f, 0x05, 0x9e, 0xc0, 0xab, + 0x70, 0x3c, 0xaf, 0x0f, 0xd2, 0x99, 0xef, 0xa2, 0x62, 0x63, 0x21, 0x9b, 0xc9, 0x2b, 0xe0, 0x57, + 0x07, 0x2e, 0xce, 0x53, 0x98, 0xcb, 0xa6, 0x01, 0xf0, 0x24, 0xf6, 0x07, 0x3c, 0x1e, 0x7f, 0x2a, + 0xc6, 0xe6, 0xfa, 0xcb, 0x21, 0xe4, 0x0b, 0x58, 0x9f, 0xf2, 0xf5, 0x71, 0x4f, 0x87, 0x48, 0x93, + 0xba, 0xba, 0x90, 0x94, 0xd6, 0x63, 0x0b, 0xcc, 0xe9, 0x5f, 0x0e, 0x5c, 0x9a, 0xbb, 0x94, 0xd5, + 0xa4, 0x93, 0xaf, 0xc9, 0x1b, 0x50, 0x7b, 0xae, 0x26, 0x5b, 0x5b, 0x24, 0xd2, 0x0f, 0xb9, 0xd2, + 0x34, 0x45, 0x3b, 0x83, 0x13, 0x0f, 0x96, 0x11, 0xdb, 0xe1, 0x43, 0x43, 0xf3, 0xbd, 0x13, 0x68, + 0x6e, 0x59, 0x7d, 0x33, 0xf8, 0xad, 0xa8, 0xc8, 0xe0, 0x45, 0x64, 0x6f, 0x35, 0x14, 0xd4, 0x48, + 0x9f, 0x30, 0x38, 0xd5, 0x58, 0x8e, 0x60, 0xc3, 0x8e, 0xc2, 0x09, 0x26, 0xc7, 0x77, 0xea, 0x6d, + 0x80, 0x4c, 0xd5, 0x4c, 0x80, 0x63, 0xea, 0x33, 0xa7, 0x4c, 0x1f, 0xc2, 0x86, 0x9d, 0xd3, 0xa7, + 0xd8, 0xd0, 0x56, 0x4b, 0x21, 0xab, 0x16, 0xda, 0x01, 0xf7, 0x19, 0xf3, 0xd4, 0x5d, 0x8d, 0xdd, + 0x6a, 0x53, 0x64, 0x24, 0x65, 0xf2, 0x30, 0x4a, 0xa4, 0x35, 0x51, 0xdf, 0x0a, 0x7b, 0x12, 0xc5, + 0x12, 0x19, 0x57, 0x19, 0x7e, 0xd3, 0xaf, 0xa0, 0xf8, 0x38, 0xea, 0x0b, 0xb2, 0x0a, 0x05, 0xaf, + 0x6d, 0x7c, 0x14, 0xbc, 0x36, 0xb9, 0x8a, 0xee, 0xcd, 0x0c, 0xa9, 0x66, 0x87, 0x7b, 0xc6, 0x3c, + 0x86, 0x1b, 0x5f, 0x83, 0xaa, 0x97, 0xb4, 0xa2, 0x28, 0xee, 0xab, 0x54, 0x47, 0xb1, 0xb9, 0x93, + 0x26, 0x41, 0x7a, 0x0f, 0x6a, 0xca, 0x7d, 0x57, 0x72, 0x99, 0x4e, 0xea, 0x75, 0x28, 0x2b, 0x2c, + 0xdd, 0xce, 0x48, 0x78, 0xef, 0x29, 0x3d, 0x3b, 0x00, 0x51, 0xa0, 0x8f, 0xb4, 0x87, 0xce, 0x81, + 0x08, 0x65, 0x2e, 0x4a, 0x28, 0xa3, 0x83, 0x2a, 0xd3, 0x02, 0xa1, 0xfa, 0x28, 0x86, 0xf3, 0x6a, + 0xc6, 0x59, 0xa1, 0x0c, 0xd7, 0xe8, 0x4f, 0x0e, 0x80, 0x25, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, + 0x09, 0x79, 0x3f, 0xf7, 0x76, 0x99, 0x9d, 0xa9, 0xe9, 0x12, 0xcb, 0xbd, 0x70, 0x36, 0xed, 0x08, + 0x35, 0xc5, 0x51, 0xcb, 0xf4, 0x35, 0x6e, 0xd2, 0xa4, 0xae, 0xcd, 0x6a, 0x2b, 0x18, 0x25, 0x52, + 0xc4, 0x86, 0x91, 0x7a, 0x63, 0x69, 0x20, 0x8d, 0x4f, 0x06, 0xcc, 0x0f, 0x11, 0xb9, 0x06, 0x25, + 0xc5, 0xd4, 0xce, 0x81, 0xe9, 0x63, 0xe8, 0x45, 0xda, 0x35, 0x37, 0xc9, 0xdc, 0xd9, 0x43, 0xa0, + 0x88, 0x2f, 0x6a, 0x53, 0x2e, 0xf8, 0x98, 0xae, 0x81, 0xbb, 0xe3, 0xeb, 0xfa, 0x76, 0x99, 0xfa, + 0x44, 0x84, 0x1f, 0x61, 0xff, 0x29, 0x84, 0xab, 0xb7, 0xc4, 0x9a, 0x6e, 0x20, 0x75, 0x77, 0x9c, + 0xe5, 0x7e, 0xb3, 0x8f, 0x52, 0x37, 0xf7, 0x28, 0xed, 0xc2, 0x9a, 0x6e, 0x92, 0xff, 0xd3, 0xe9, + 0x2f, 0x05, 0x58, 0x63, 0x22, 0xf1, 0x5f, 0x0a, 0x2f, 0x4c, 0x64, 0x3c, 0x4a, 0x07, 0xdc, 0x27, + 0xd1, 0xae, 0x09, 0xb5, 0xcb, 0xb4, 0xf0, 0x3a, 0x95, 0x44, 0x6e, 0xaa, 0xdf, 0xa3, 0xc9, 0xea, + 0x9f, 0x55, 0xcd, 0xab, 0x90, 0x9b, 0xb0, 0xd4, 0x8d, 0x46, 0x71, 0x2f, 0xbd, 0x06, 0xd7, 0x33, + 0x6d, 0xcd, 0x4c, 0x2f, 0x33, 0xab, 0x96, 0xab, 0xa3, 0xd2, 0xf1, 0x75, 0x44, 0xee, 0x4e, 0xd5, + 0x11, 0xfe, 0xb9, 0xac, 0x6c, 0xbf, 0x99, 0x19, 0x4c, 0x2c, 0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, + 0xe7, 0xf2, 0x14, 0x5e, 0xab, 0x31, 0xd2, 0x8c, 0x14, 0xe6, 0x66, 0xc4, 0x9d, 0x97, 0x91, 0x62, + 0x96, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x3e, 0x5c, 0x9e, 0x49, 0x53, 0x2b, 0x1a, + 0x0c, 0x55, 0x3d, 0xfc, 0x87, 0x74, 0xa9, 0x91, 0x11, 0xc7, 0x26, 0x51, 0x15, 0xa6, 0x05, 0x7a, + 0x1b, 0x2e, 0x75, 0x85, 0xcc, 0x25, 0xc9, 0x56, 0x5b, 0x13, 0xdc, 0xc7, 0xe2, 0x70, 0xc1, 0xf1, + 0xd5, 0x12, 0xbd, 0x0f, 0xcb, 0x4f, 0xa3, 0x61, 0x14, 0x44, 0x7b, 0xe3, 0x13, 0x9a, 0xb6, 0x0e, + 0x4b, 0x7a, 0xc2, 0xe9, 0x67, 0x46, 0x85, 0x59, 0x91, 0x5e, 0x50, 0x25, 0xd9, 0xe3, 0x41, 0x6f, + 0x14, 0x70, 0x29, 0xf0, 0xef, 0x29, 0xb9, 0x5f, 0xfb, 0xed, 0x55, 0xc3, 0xf9, 0xfd, 0x55, 0xc3, + 0xf9, 0xe3, 0x55, 0xc3, 0xf9, 0xf9, 0xcf, 0xc6, 0x1b, 0xbb, 0x65, 0xfc, 0x4f, 0xbf, 0xf5, 0x6f, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x30, 0x4b, 0x92, 0xf6, 0xb8, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 5f4a1c6f6..5755ce2f6 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -144,6 +144,11 @@ message NodeStateMessage { string State = 2; } +message NodeEventMessage { + uint32 Event = 1; + Node Node = 2; +} + message NodeStatus { Node Node = 1; MaxSlices MaxSlices = 2; diff --git a/server.go b/server.go index 0dcca4d1e..c52c28cbd 100644 --- a/server.go +++ b/server.go @@ -463,6 +463,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { } case *internal.RecalculateCaches: s.Holder.RecalculateCaches() + case *internal.NodeEventMessage: + s.Cluster.ReceiveEvent(DecodeNodeEvent(obj)) } return nil From 2a462d5e42c0fb413a37e10261a41476d62bdfbe Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 8 Mar 2018 14:18:10 -0600 Subject: [PATCH 138/311] make sure cluster.Nodes[].IsCoordinator values get updated. return old coordinator node in response. --- broadcast.go | 5 + cluster.go | 53 +++++-- cluster_test.go | 12 +- handler.go | 15 +- internal/private.pb.go | 312 +++++++++++++++++++++++++++++------------ internal/private.proto | 4 + server.go | 2 + 7 files changed, 295 insertions(+), 108 deletions(-) diff --git a/broadcast.go b/broadcast.go index 6dbb0f771..a4e6403c1 100644 --- a/broadcast.go +++ b/broadcast.go @@ -134,6 +134,7 @@ const ( MessageTypeResizeInstruction MessageTypeResizeInstructionComplete MessageTypeSetCoordinator + MessageTypeUpdateCoordinator MessageTypeNodeState MessageTypeRecalculateCaches MessageTypeNodeEvent @@ -173,6 +174,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeResizeInstructionComplete case *internal.SetCoordinatorMessage: typ = MessageTypeSetCoordinator + case *internal.UpdateCoordinatorMessage: + typ = MessageTypeUpdateCoordinator case *internal.NodeStateMessage: typ = MessageTypeNodeState case *internal.RecalculateCaches: @@ -225,6 +228,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.ResizeInstructionComplete{} case MessageTypeSetCoordinator: m = &internal.SetCoordinatorMessage{} + case MessageTypeUpdateCoordinator: + m = &internal.UpdateCoordinatorMessage{} case MessageTypeNodeState: m = &internal.NodeStateMessage{} case MessageTypeRecalculateCaches: diff --git a/cluster.go b/cluster.go index b93e6105c..a5b7b654f 100644 --- a/cluster.go +++ b/cluster.go @@ -309,20 +309,51 @@ func (c *Cluster) IsCoordinator() bool { return c.Coordinator == c.Node.ID } -// SetCoordinator updates the Coordinator to n. -// Returns true if the Coordinator changed. -func (c *Cluster) SetCoordinator(n *Node) bool { - // Get new node. - newNode := c.nodeByID(n.ID) - if newNode == nil { - return false +// SetCoordinator tells the current node to become the +// Coordinator. In response to this, the current node +// will consider itself coordinator and update the other +// nodes with its version of Cluster.Status. +func (c *Cluster) SetCoordinator(n *Node) error { + // Verify that the new Coordinator value matches + // this node. + if c.Node.ID != n.ID { + return fmt.Errorf("coordinator node does not match this node") } - if c.Coordinator != newNode.ID { - c.Coordinator = newNode.ID - return true + // Update IsCoordinator on all nodes (locally). + _ = c.UpdateCoordinator(n) + + // Send the update coordinator message to all nodes. + err := c.Broadcaster.SendSync( + &internal.UpdateCoordinatorMessage{ + New: EncodeNode(n), + }) + if err != nil { + return fmt.Errorf("problem sending UpdateCoordinator message: %v", err) } - return false + + // Broadcast cluster status. + return c.Broadcaster.SendSync(c.Status()) +} + +// UpdateCoordinator updates this nodes Coordinator value as well as +// changing the corresponding node's IsCoordinator value +// to true, and sets all other nodes to false. Returns true if the value +// changed. +func (c *Cluster) UpdateCoordinator(n *Node) bool { + var changed bool + if c.Coordinator != n.ID { + c.Coordinator = n.ID + changed = true + } + for _, node := range c.Nodes { + if node.ID == n.ID { + node.IsCoordinator = true + } else { + node.IsCoordinator = false + } + } + return changed } // AddNode adds a node to the Cluster and updates and saves the diff --git a/cluster_test.go b/cluster_test.go index 5c5c5f89c..c5c01d58d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -509,22 +509,22 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensures that coordinator can be changed. -func TestCluster_SetCoordinator(t *testing.T) { - t.Run("SetCoordinator", func(t *testing.T) { +func TestCluster_UpdateCoordinator(t *testing.T) { + t.Run("UpdateCoordinator", func(t *testing.T) { c := test.NewCluster(2) oldNode := c.Nodes[0] newNode := c.Nodes[1] - // Set coordinator to the same value. - if c.SetCoordinator(oldNode) { + // Update coordinator to the same value. + if c.UpdateCoordinator(oldNode) { t.Errorf("did not expect coordinator to change") } else if c.Coordinator != oldNode.ID { t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) } - // Set coordinator to a new value. - if !c.SetCoordinator(newNode) { + // Update coordinator to a new value. + if !c.UpdateCoordinator(newNode) { t.Errorf("expected coordinator to change") } else if c.Coordinator != newNode.ID { t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) diff --git a/handler.go b/handler.go index 38fd24eca..cea42e6ac 100644 --- a/handler.go +++ b/handler.go @@ -2026,6 +2026,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r return } + oldNode := h.Cluster.nodeByID(h.Cluster.Coordinator) newNode := h.Cluster.nodeByID(req.ID) if newNode == nil { http.Error(w, "Node with provided ID does not exist", http.StatusBadRequest) @@ -2033,8 +2034,14 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r } if err := func() error { - // Send the set-coordinator message to all nodes. - err := h.Broadcaster.SendSync( + // If the new coordinator is this node, do the SetCoordinator directly. + if newNode.ID == h.Node.ID { + return h.Cluster.SetCoordinator(newNode) + } + + // Send the set-coordinator message to new node. + err := h.Broadcaster.SendTo( + newNode, &internal.SetCoordinatorMessage{ New: EncodeNode(newNode), }) @@ -2042,9 +2049,6 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r return fmt.Errorf("problem sending SetCoordinator message: %s", err) } - // Set Coordinator on local node. - _ = h.Cluster.SetCoordinator(newNode) - return nil }(); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2053,6 +2057,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r // Encode response. if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: oldNode, New: newNode, }); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/internal/private.pb.go b/internal/private.pb.go index 37b109014..1b7f71ac4 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -44,6 +44,7 @@ ResizeSource ResizeInstructionComplete SetCoordinatorMessage + UpdateCoordinatorMessage Topology RecalculateCaches */ @@ -1144,6 +1145,22 @@ func (m *SetCoordinatorMessage) GetNew() *Node { return nil } +type UpdateCoordinatorMessage struct { + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` +} + +func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } +func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*UpdateCoordinatorMessage) ProtoMessage() {} +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{35} } + +func (m *UpdateCoordinatorMessage) GetNew() *Node { + if m != nil { + return m.New + } + return nil +} + type Topology struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` @@ -1152,7 +1169,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{35} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{36} } func (m *Topology) GetClusterID() string { if m != nil { @@ -1174,7 +1191,7 @@ type RecalculateCaches struct { func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{36} } +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{37} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") @@ -1212,6 +1229,7 @@ func init() { proto.RegisterType((*ResizeSource)(nil), "internal.ResizeSource") proto.RegisterType((*ResizeInstructionComplete)(nil), "internal.ResizeInstructionComplete") proto.RegisterType((*SetCoordinatorMessage)(nil), "internal.SetCoordinatorMessage") + proto.RegisterType((*UpdateCoordinatorMessage)(nil), "internal.UpdateCoordinatorMessage") proto.RegisterType((*Topology)(nil), "internal.Topology") proto.RegisterType((*RecalculateCaches)(nil), "internal.RecalculateCaches") } @@ -2641,6 +2659,34 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *UpdateCoordinatorMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.New != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n24, err := m.New.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n24 + } + return i, nil +} + func (m *Topology) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -3355,6 +3401,16 @@ func (m *SetCoordinatorMessage) Size() (n int) { return n } +func (m *UpdateCoordinatorMessage) Size() (n int) { + var l int + _ = l + if m.New != nil { + l = m.New.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *Topology) Size() (n int) { var l int _ = l @@ -8262,6 +8318,89 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UpdateCoordinatorMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UpdateCoordinatorMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field New", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.New == nil { + m.New = &Node{} + } + if err := m.New.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Topology) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -8528,88 +8667,89 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1325 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0xb6, 0x13, 0xbf, 0xd4, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15, - 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0xb4, 0xa8, 0x52, 0xa9, 0xed, 0xaa, 0x0b, 0x4d, 0x29, 0xe3, - 0xb6, 0x48, 0x48, 0x20, 0x4d, 0xec, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03, - 0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x04, 0x54, 0x3e, 0x04, 0x12, 0x17, - 0xd0, 0xbc, 0x99, 0xd9, 0x5d, 0xff, 0x4b, 0x9a, 0xc0, 0x6d, 0xdf, 0x6f, 0xde, 0x7b, 0xf3, 0x9b, - 0xf7, 0x6f, 0x66, 0xa1, 0x3a, 0x8c, 0xfd, 0x03, 0x2e, 0xc5, 0xd6, 0x30, 0x8e, 0x64, 0x44, 0x96, - 0xfd, 0x50, 0x8a, 0x38, 0xe4, 0x01, 0xfd, 0x0c, 0x2a, 0x5e, 0xd8, 0x17, 0x47, 0x3b, 0x42, 0x72, - 0xd2, 0x84, 0x95, 0x56, 0x14, 0x8c, 0x06, 0xe1, 0x23, 0xbe, 0x2b, 0x82, 0xba, 0xd3, 0x74, 0x36, - 0x2b, 0x2c, 0x0f, 0x29, 0x8d, 0xa7, 0xfe, 0x40, 0x7c, 0x3e, 0xe2, 0xa1, 0x1c, 0x0d, 0xea, 0x05, - 0xad, 0x91, 0x83, 0xe8, 0xdf, 0x0e, 0x54, 0x1e, 0xc4, 0x7c, 0x20, 0xd0, 0xe3, 0x15, 0x58, 0x66, - 0xd1, 0x61, 0xde, 0x5d, 0x2a, 0x93, 0x77, 0x60, 0xd5, 0x0b, 0x0f, 0x44, 0x9c, 0x88, 0x4e, 0xc8, - 0x77, 0x03, 0xd1, 0x47, 0x77, 0xcb, 0x6c, 0x0a, 0x25, 0x1b, 0x50, 0x69, 0xf1, 0xde, 0x0b, 0xf1, - 0x74, 0x3c, 0x14, 0x75, 0x17, 0x9d, 0x64, 0x40, 0xba, 0xda, 0xf5, 0x5f, 0x8a, 0x7a, 0xb1, 0xe9, - 0x6c, 0x56, 0x59, 0x06, 0x4c, 0xf3, 0x2d, 0xcd, 0xf0, 0x25, 0x14, 0xce, 0x31, 0x1e, 0xee, 0xa5, - 0x1c, 0xca, 0xc8, 0x61, 0x02, 0x23, 0xd7, 0xa1, 0xfc, 0xc0, 0x17, 0x41, 0x3f, 0xa9, 0x2f, 0x35, - 0xdd, 0xcd, 0x95, 0xed, 0xf3, 0x5b, 0x36, 0x7e, 0x5b, 0x88, 0x33, 0xb3, 0x4c, 0x29, 0xac, 0x7a, - 0x83, 0x61, 0x14, 0x4b, 0x26, 0x92, 0x61, 0x14, 0x26, 0x82, 0xd4, 0xc0, 0xed, 0xc4, 0xb1, 0x39, - 0xbb, 0xfa, 0xa4, 0xdf, 0x41, 0xed, 0x7e, 0x10, 0xf5, 0xf6, 0xdb, 0x5c, 0x72, 0x26, 0xbe, 0x1d, - 0x89, 0x44, 0x92, 0x8b, 0x50, 0xc2, 0x2c, 0x18, 0x3d, 0x2d, 0x28, 0x14, 0x23, 0x69, 0xc2, 0xac, - 0x05, 0x85, 0xa2, 0x3d, 0x86, 0xa2, 0xc8, 0xb4, 0xa0, 0xd0, 0x6e, 0xe0, 0xf7, 0x74, 0x08, 0x8a, - 0x4c, 0x0b, 0x84, 0x40, 0xf1, 0xb9, 0x2f, 0x0e, 0xcd, 0xb9, 0xf1, 0x9b, 0x7a, 0xb0, 0x96, 0xdb, - 0xdf, 0xd0, 0x5c, 0x87, 0x32, 0x8b, 0x0e, 0xbd, 0x76, 0x52, 0x77, 0x9a, 0xee, 0x66, 0x91, 0x19, - 0x09, 0xa3, 0x8b, 0xe9, 0x57, 0x4b, 0x05, 0x5c, 0xca, 0x00, 0x7a, 0x19, 0x4a, 0x18, 0x6a, 0x75, - 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8, 0xec, 0xf0, 0x23, 0xa4, 0x91, 0x90, 0xbb, 0xb0, - 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a, 0x2b, 0xdb, 0x6f, 0x65, 0x21, 0x4c, 0xd5, 0xb6, - 0xac, 0x4e, 0x27, 0x94, 0xf1, 0x98, 0xa5, 0x26, 0xe4, 0x0e, 0x2c, 0x99, 0x9a, 0x40, 0x0e, 0x2b, - 0xdb, 0xcd, 0x79, 0xd6, 0x69, 0xd9, 0x28, 0x63, 0x6b, 0x70, 0xe5, 0x23, 0xa8, 0x4e, 0xb8, 0x55, - 0x5c, 0xf7, 0xc5, 0xd8, 0x66, 0x64, 0x5f, 0x8c, 0x55, 0xec, 0x0e, 0x78, 0x30, 0xd2, 0x71, 0x2e, - 0x32, 0x2d, 0xdc, 0x29, 0x7c, 0xe8, 0x5c, 0xb9, 0x03, 0xe7, 0xf2, 0x5e, 0x4f, 0x63, 0x4b, 0xbf, - 0x06, 0xd2, 0x8a, 0x05, 0x97, 0x02, 0xe9, 0xed, 0x88, 0x24, 0xe1, 0x7b, 0x62, 0x71, 0xa6, 0x75, - 0xf6, 0x0a, 0xf9, 0xec, 0x6d, 0x40, 0xc5, 0x4b, 0xec, 0xc1, 0x5d, 0xac, 0xcb, 0x0c, 0xa0, 0x37, - 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0xfd, 0x7b, 0x8c, 0x7f, 0xda, 0xb5, 0x5c, 0x4e, 0xd6, 0x25, - 0xd7, 0xa1, 0xa8, 0x5a, 0x17, 0xa9, 0xac, 0x6c, 0x5f, 0xc8, 0x22, 0x9d, 0xce, 0x09, 0x86, 0x0a, - 0xd4, 0xb7, 0x4e, 0x4d, 0xbb, 0x9f, 0x70, 0xc0, 0x39, 0xa5, 0x6c, 0xb7, 0x72, 0xa7, 0xb7, 0x4a, - 0x07, 0x88, 0xd9, 0xea, 0x9e, 0x3d, 0xeb, 0x59, 0xb7, 0xa2, 0x7b, 0x29, 0x59, 0xd5, 0xa9, 0x67, - 0x21, 0xfb, 0x36, 0x94, 0xd0, 0xd6, 0xb0, 0x9d, 0x99, 0x01, 0x7a, 0x95, 0x3e, 0x4f, 0xa9, 0x9e, - 0x75, 0xa3, 0x8b, 0xf9, 0x8d, 0x2a, 0xd6, 0xef, 0x97, 0x46, 0x57, 0xf5, 0xf4, 0x63, 0x65, 0xa3, - 0x3d, 0xe1, 0xf7, 0xe2, 0x9c, 0x4d, 0x05, 0x52, 0xf9, 0x56, 0x43, 0x20, 0xa9, 0xbb, 0x4d, 0x57, - 0xf9, 0x46, 0x81, 0xde, 0x82, 0x72, 0xb7, 0xf7, 0x42, 0x0c, 0x38, 0x79, 0x57, 0x75, 0x5a, 0x5f, - 0x1c, 0x89, 0xc4, 0xf4, 0xe9, 0xf9, 0xa9, 0xfc, 0x33, 0xbb, 0x4e, 0x7f, 0x74, 0xcc, 0x99, 0x16, - 0x30, 0x2a, 0xe3, 0xde, 0x49, 0xbd, 0x38, 0x33, 0x32, 0x15, 0xce, 0xcc, 0x32, 0xe9, 0x40, 0xcd, - 0x0b, 0x87, 0x23, 0xd9, 0x16, 0xdf, 0xf8, 0xa1, 0x2f, 0xfd, 0x28, 0x4c, 0xea, 0x65, 0x34, 0xb9, - 0x9c, 0xdf, 0x7a, 0x42, 0x83, 0xcd, 0x98, 0xd0, 0xef, 0x1d, 0x38, 0x3f, 0x05, 0x9e, 0xc0, 0xab, - 0x70, 0x3c, 0xaf, 0x0f, 0xd2, 0x99, 0xef, 0xa2, 0x62, 0x63, 0x21, 0x9b, 0xc9, 0x2b, 0xe0, 0x57, - 0x07, 0x2e, 0xce, 0x53, 0x98, 0xcb, 0xa6, 0x01, 0xf0, 0x24, 0xf6, 0x07, 0x3c, 0x1e, 0x7f, 0x2a, - 0xc6, 0xe6, 0xfa, 0xcb, 0x21, 0xe4, 0x0b, 0x58, 0x9f, 0xf2, 0xf5, 0x71, 0x4f, 0x87, 0x48, 0x93, - 0xba, 0xba, 0x90, 0x94, 0xd6, 0x63, 0x0b, 0xcc, 0xe9, 0x5f, 0x0e, 0x5c, 0x9a, 0xbb, 0x94, 0xd5, - 0xa4, 0x93, 0xaf, 0xc9, 0x1b, 0x50, 0x7b, 0xae, 0x26, 0x5b, 0x5b, 0x24, 0xd2, 0x0f, 0xb9, 0xd2, - 0x34, 0x45, 0x3b, 0x83, 0x13, 0x0f, 0x96, 0x11, 0xdb, 0xe1, 0x43, 0x43, 0xf3, 0xbd, 0x13, 0x68, - 0x6e, 0x59, 0x7d, 0x33, 0xf8, 0xad, 0xa8, 0xc8, 0xe0, 0x45, 0x64, 0x6f, 0x35, 0x14, 0xd4, 0x48, - 0x9f, 0x30, 0x38, 0xd5, 0x58, 0x8e, 0x60, 0xc3, 0x8e, 0xc2, 0x09, 0x26, 0xc7, 0x77, 0xea, 0x6d, - 0x80, 0x4c, 0xd5, 0x4c, 0x80, 0x63, 0xea, 0x33, 0xa7, 0x4c, 0x1f, 0xc2, 0x86, 0x9d, 0xd3, 0xa7, - 0xd8, 0xd0, 0x56, 0x4b, 0x21, 0xab, 0x16, 0xda, 0x01, 0xf7, 0x19, 0xf3, 0xd4, 0x5d, 0x8d, 0xdd, - 0x6a, 0x53, 0x64, 0x24, 0x65, 0xf2, 0x30, 0x4a, 0xa4, 0x35, 0x51, 0xdf, 0x0a, 0x7b, 0x12, 0xc5, - 0x12, 0x19, 0x57, 0x19, 0x7e, 0xd3, 0xaf, 0xa0, 0xf8, 0x38, 0xea, 0x0b, 0xb2, 0x0a, 0x05, 0xaf, - 0x6d, 0x7c, 0x14, 0xbc, 0x36, 0xb9, 0x8a, 0xee, 0xcd, 0x0c, 0xa9, 0x66, 0x87, 0x7b, 0xc6, 0x3c, - 0x86, 0x1b, 0x5f, 0x83, 0xaa, 0x97, 0xb4, 0xa2, 0x28, 0xee, 0xab, 0x54, 0x47, 0xb1, 0xb9, 0x93, - 0x26, 0x41, 0x7a, 0x0f, 0x6a, 0xca, 0x7d, 0x57, 0x72, 0x99, 0x4e, 0xea, 0x75, 0x28, 0x2b, 0x2c, - 0xdd, 0xce, 0x48, 0x78, 0xef, 0x29, 0x3d, 0x3b, 0x00, 0x51, 0xa0, 0x8f, 0xb4, 0x87, 0xce, 0x81, - 0x08, 0x65, 0x2e, 0x4a, 0x28, 0xa3, 0x83, 0x2a, 0xd3, 0x02, 0xa1, 0xfa, 0x28, 0x86, 0xf3, 0x6a, - 0xc6, 0x59, 0xa1, 0x0c, 0xd7, 0xe8, 0x4f, 0x0e, 0x80, 0x25, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, - 0x09, 0x79, 0x3f, 0xf7, 0x76, 0x99, 0x9d, 0xa9, 0xe9, 0x12, 0xcb, 0xbd, 0x70, 0x36, 0xed, 0x08, - 0x35, 0xc5, 0x51, 0xcb, 0xf4, 0x35, 0x6e, 0xd2, 0xa4, 0xae, 0xcd, 0x6a, 0x2b, 0x18, 0x25, 0x52, - 0xc4, 0x86, 0x91, 0x7a, 0x63, 0x69, 0x20, 0x8d, 0x4f, 0x06, 0xcc, 0x0f, 0x11, 0xb9, 0x06, 0x25, - 0xc5, 0xd4, 0xce, 0x81, 0xe9, 0x63, 0xe8, 0x45, 0xda, 0x35, 0x37, 0xc9, 0xdc, 0xd9, 0x43, 0xa0, - 0x88, 0x2f, 0x6a, 0x53, 0x2e, 0xf8, 0x98, 0xae, 0x81, 0xbb, 0xe3, 0xeb, 0xfa, 0x76, 0x99, 0xfa, - 0x44, 0x84, 0x1f, 0x61, 0xff, 0x29, 0x84, 0xab, 0xb7, 0xc4, 0x9a, 0x6e, 0x20, 0x75, 0x77, 0x9c, - 0xe5, 0x7e, 0xb3, 0x8f, 0x52, 0x37, 0xf7, 0x28, 0xed, 0xc2, 0x9a, 0x6e, 0x92, 0xff, 0xd3, 0xe9, - 0x2f, 0x05, 0x58, 0x63, 0x22, 0xf1, 0x5f, 0x0a, 0x2f, 0x4c, 0x64, 0x3c, 0x4a, 0x07, 0xdc, 0x27, - 0xd1, 0xae, 0x09, 0xb5, 0xcb, 0xb4, 0xf0, 0x3a, 0x95, 0x44, 0x6e, 0xaa, 0xdf, 0xa3, 0xc9, 0xea, - 0x9f, 0x55, 0xcd, 0xab, 0x90, 0x9b, 0xb0, 0xd4, 0x8d, 0x46, 0x71, 0x2f, 0xbd, 0x06, 0xd7, 0x33, - 0x6d, 0xcd, 0x4c, 0x2f, 0x33, 0xab, 0x96, 0xab, 0xa3, 0xd2, 0xf1, 0x75, 0x44, 0xee, 0x4e, 0xd5, - 0x11, 0xfe, 0xb9, 0xac, 0x6c, 0xbf, 0x99, 0x19, 0x4c, 0x2c, 0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, - 0xe7, 0xf2, 0x14, 0x5e, 0xab, 0x31, 0xd2, 0x8c, 0x14, 0xe6, 0x66, 0xc4, 0x9d, 0x97, 0x91, 0x62, - 0x96, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x3e, 0x5c, 0x9e, 0x49, 0x53, 0x2b, 0x1a, - 0x0c, 0x55, 0x3d, 0xfc, 0x87, 0x74, 0xa9, 0x91, 0x11, 0xc7, 0x26, 0x51, 0x15, 0xa6, 0x05, 0x7a, - 0x1b, 0x2e, 0x75, 0x85, 0xcc, 0x25, 0xc9, 0x56, 0x5b, 0x13, 0xdc, 0xc7, 0xe2, 0x70, 0xc1, 0xf1, - 0xd5, 0x12, 0xbd, 0x0f, 0xcb, 0x4f, 0xa3, 0x61, 0x14, 0x44, 0x7b, 0xe3, 0x13, 0x9a, 0xb6, 0x0e, - 0x4b, 0x7a, 0xc2, 0xe9, 0x67, 0x46, 0x85, 0x59, 0x91, 0x5e, 0x50, 0x25, 0xd9, 0xe3, 0x41, 0x6f, - 0x14, 0x70, 0x29, 0xf0, 0xef, 0x29, 0xb9, 0x5f, 0xfb, 0xed, 0x55, 0xc3, 0xf9, 0xfd, 0x55, 0xc3, - 0xf9, 0xe3, 0x55, 0xc3, 0xf9, 0xf9, 0xcf, 0xc6, 0x1b, 0xbb, 0x65, 0xfc, 0x4f, 0xbf, 0xf5, 0x6f, - 0x00, 0x00, 0x00, 0xff, 0xff, 0x30, 0x4b, 0x92, 0xf6, 0xb8, 0x0f, 0x00, 0x00, + // 1334 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, + 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf6, 0x71, 0x9c, 0x38, 0xd3, 0x34, 0xaf, 0x13, 0x45, 0xae, 0x19, + 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0x34, 0x50, 0xa9, 0xc4, 0x76, 0xd5, 0x85, 0x26, 0x94, + 0x71, 0x12, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x24, 0xee, + 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x4f, 0x40, 0xe1, 0x47, 0x20, + 0x71, 0x03, 0x9a, 0xaf, 0xdd, 0xf5, 0x57, 0xd2, 0x04, 0xee, 0xf6, 0x3c, 0x73, 0xce, 0x99, 0x67, + 0xce, 0xd7, 0xcc, 0x42, 0xb9, 0x1f, 0xba, 0xa7, 0x94, 0xb3, 0xcd, 0x7e, 0x18, 0xf0, 0x00, 0x15, + 0x5c, 0x9f, 0xb3, 0xd0, 0xa7, 0x1e, 0xfe, 0x14, 0x8a, 0x8e, 0xdf, 0x65, 0xe7, 0xbb, 0x8c, 0x53, + 0x54, 0x87, 0x52, 0x23, 0xf0, 0x06, 0x3d, 0xff, 0x39, 0x3d, 0x62, 0x5e, 0xd5, 0xaa, 0x5b, 0x1b, + 0x45, 0x92, 0x86, 0x84, 0xc6, 0xbe, 0xdb, 0x63, 0x9f, 0x0d, 0xa8, 0xcf, 0x07, 0xbd, 0x6a, 0x46, + 0x69, 0xa4, 0x20, 0xfc, 0x97, 0x05, 0xc5, 0xa7, 0x21, 0xed, 0x31, 0xe9, 0x71, 0x0d, 0x0a, 0x24, + 0x38, 0x4b, 0xbb, 0x8b, 0x65, 0xf4, 0x16, 0x2c, 0x38, 0xfe, 0x29, 0x0b, 0x23, 0xd6, 0xf2, 0xe9, + 0x91, 0xc7, 0xba, 0xd2, 0x5d, 0x81, 0x8c, 0xa1, 0x68, 0x1d, 0x8a, 0x0d, 0xda, 0x79, 0xc9, 0xf6, + 0x87, 0x7d, 0x56, 0xb5, 0xa5, 0x93, 0x04, 0x88, 0x57, 0xdb, 0xee, 0x2b, 0x56, 0xcd, 0xd6, 0xad, + 0x8d, 0x32, 0x49, 0x80, 0x71, 0xbe, 0xb9, 0x09, 0xbe, 0x08, 0xc3, 0x3c, 0xa1, 0xfe, 0x71, 0xcc, + 0x21, 0x2f, 0x39, 0x8c, 0x60, 0xe8, 0x1e, 0xe4, 0x9f, 0xba, 0xcc, 0xeb, 0x46, 0xd5, 0xb9, 0xba, + 0xbd, 0x51, 0xda, 0x5a, 0xdc, 0x34, 0xf1, 0xdb, 0x94, 0x38, 0xd1, 0xcb, 0x18, 0xc3, 0x82, 0xd3, + 0xeb, 0x07, 0x21, 0x27, 0x2c, 0xea, 0x07, 0x7e, 0xc4, 0x50, 0x05, 0xec, 0x56, 0x18, 0xea, 0xb3, + 0x8b, 0x4f, 0xfc, 0x2d, 0x54, 0x76, 0xbc, 0xa0, 0x73, 0xd2, 0xa4, 0x9c, 0x12, 0xf6, 0xcd, 0x80, + 0x45, 0x1c, 0x2d, 0x43, 0x4e, 0x66, 0x41, 0xeb, 0x29, 0x41, 0xa0, 0x32, 0x92, 0x3a, 0xcc, 0x4a, + 0x10, 0xa8, 0xb4, 0x97, 0xa1, 0xc8, 0x12, 0x25, 0x08, 0xb4, 0xed, 0xb9, 0x1d, 0x15, 0x82, 0x2c, + 0x51, 0x02, 0x42, 0x90, 0x3d, 0x74, 0xd9, 0x99, 0x3e, 0xb7, 0xfc, 0xc6, 0x0e, 0x2c, 0xa5, 0xf6, + 0xd7, 0x34, 0x57, 0x20, 0x4f, 0x82, 0x33, 0xa7, 0x19, 0x55, 0xad, 0xba, 0xbd, 0x91, 0x25, 0x5a, + 0x92, 0xd1, 0x95, 0xe9, 0x17, 0x4b, 0x19, 0xb9, 0x94, 0x00, 0x78, 0x15, 0x72, 0x32, 0xd4, 0xe2, + 0x94, 0x89, 0xad, 0xf8, 0xc4, 0x7f, 0x5b, 0x50, 0xdc, 0xa5, 0xe7, 0x92, 0x46, 0x84, 0x1e, 0x43, + 0xa1, 0xcd, 0xa9, 0xdf, 0xa5, 0x61, 0x57, 0x2a, 0x95, 0xb6, 0xde, 0x48, 0x42, 0x18, 0xab, 0x6d, + 0x1a, 0x9d, 0x96, 0xcf, 0xc3, 0x21, 0x89, 0x4d, 0xd0, 0x36, 0xcc, 0xe9, 0x9a, 0x90, 0x1c, 0x4a, + 0x5b, 0xf5, 0x69, 0xd6, 0x71, 0xd9, 0x08, 0x63, 0x63, 0xb0, 0xf6, 0x01, 0x94, 0x47, 0xdc, 0x0a, + 0xae, 0x27, 0x6c, 0x68, 0x32, 0x72, 0xc2, 0x86, 0x22, 0x76, 0xa7, 0xd4, 0x1b, 0xa8, 0x38, 0x67, + 0x89, 0x12, 0xb6, 0x33, 0xef, 0x5b, 0x6b, 0xdb, 0x30, 0x9f, 0xf6, 0x7a, 0x1d, 0x5b, 0xfc, 0x15, + 0xa0, 0x46, 0xc8, 0x28, 0x67, 0x92, 0xde, 0x2e, 0x8b, 0x22, 0x7a, 0xcc, 0x66, 0x67, 0x5a, 0x65, + 0x2f, 0x93, 0xce, 0xde, 0x3a, 0x14, 0x9d, 0xc8, 0x1c, 0xdc, 0x96, 0x75, 0x99, 0x00, 0xf8, 0x3e, + 0xa0, 0x26, 0xf3, 0x18, 0x67, 0xba, 0x7f, 0x2f, 0xf1, 0x8f, 0xdb, 0x86, 0xcb, 0xd5, 0xba, 0xe8, + 0x1e, 0x64, 0x45, 0xeb, 0x4a, 0x2a, 0xa5, 0xad, 0x5b, 0x49, 0xa4, 0xe3, 0x39, 0x41, 0xa4, 0x02, + 0x76, 0x8d, 0x53, 0xdd, 0xee, 0x57, 0x1c, 0x70, 0x4a, 0x29, 0x9b, 0xad, 0xec, 0xf1, 0xad, 0xe2, + 0x01, 0xa2, 0xb7, 0x7a, 0x62, 0xce, 0x7a, 0xd3, 0xad, 0xf0, 0x71, 0x4c, 0x56, 0x74, 0xea, 0x4d, + 0xc8, 0xbe, 0x09, 0x39, 0x69, 0xab, 0xd9, 0x4e, 0xcc, 0x00, 0xb5, 0x8a, 0x0f, 0x63, 0xaa, 0x37, + 0xdd, 0x68, 0x39, 0xbd, 0x51, 0xd1, 0xf8, 0xfd, 0x42, 0xeb, 0x8a, 0x9e, 0xde, 0x13, 0x36, 0xca, + 0x93, 0xfc, 0x9e, 0x9d, 0xb3, 0xb1, 0x40, 0x0a, 0xdf, 0x62, 0x08, 0x44, 0x55, 0xbb, 0x6e, 0x0b, + 0xdf, 0x52, 0xc0, 0x0f, 0x21, 0xdf, 0xee, 0xbc, 0x64, 0x3d, 0x8a, 0xde, 0x16, 0x9d, 0xd6, 0x65, + 0xe7, 0x2c, 0xd2, 0x7d, 0xba, 0x38, 0x96, 0x7f, 0x62, 0xd6, 0xf1, 0x0f, 0x96, 0x3e, 0xd3, 0x0c, + 0x46, 0x79, 0xb9, 0x77, 0x54, 0xcd, 0x4e, 0x8c, 0x4c, 0x81, 0x13, 0xbd, 0x8c, 0x5a, 0x50, 0x71, + 0xfc, 0xfe, 0x80, 0x37, 0xd9, 0xd7, 0xae, 0xef, 0x72, 0x37, 0xf0, 0xa3, 0x6a, 0x5e, 0x9a, 0xac, + 0xa6, 0xb7, 0x1e, 0xd1, 0x20, 0x13, 0x26, 0xf8, 0x3b, 0x0b, 0x16, 0xc7, 0xc0, 0x2b, 0x78, 0x65, + 0x2e, 0xe7, 0xf5, 0x5e, 0x3c, 0xf3, 0x6d, 0xa9, 0x58, 0x9b, 0xc9, 0x66, 0xf4, 0x0a, 0xf8, 0xc5, + 0x82, 0xe5, 0x69, 0x0a, 0x53, 0xd9, 0xd4, 0x00, 0x5e, 0x84, 0x6e, 0x8f, 0x86, 0xc3, 0x4f, 0xd8, + 0x50, 0x5f, 0x7f, 0x29, 0x04, 0x7d, 0x0e, 0x2b, 0x63, 0xbe, 0x3e, 0xea, 0xa8, 0x10, 0x29, 0x52, + 0x77, 0x66, 0x92, 0x52, 0x7a, 0x64, 0x86, 0x39, 0xfe, 0xd3, 0x82, 0xdb, 0x53, 0x97, 0x92, 0x9a, + 0xb4, 0xd2, 0x35, 0x79, 0x1f, 0x2a, 0x87, 0x62, 0xb2, 0x35, 0x59, 0xc4, 0x5d, 0x9f, 0x0a, 0x4d, + 0x5d, 0xb4, 0x13, 0x38, 0x72, 0xa0, 0x20, 0xb1, 0x5d, 0xda, 0xd7, 0x34, 0xdf, 0xb9, 0x82, 0xe6, + 0xa6, 0xd1, 0xd7, 0x83, 0xdf, 0x88, 0x82, 0x8c, 0xbc, 0x88, 0xcc, 0xad, 0x26, 0x05, 0x31, 0xd2, + 0x47, 0x0c, 0xae, 0x35, 0x96, 0x03, 0x58, 0x37, 0xa3, 0x70, 0x84, 0xc9, 0xe5, 0x9d, 0xfa, 0x08, + 0x20, 0x51, 0xd5, 0x13, 0xe0, 0x92, 0xfa, 0x4c, 0x29, 0xe3, 0x67, 0xb0, 0x6e, 0xe6, 0xf4, 0x35, + 0x36, 0x34, 0xd5, 0x92, 0x49, 0xaa, 0x05, 0xb7, 0xc0, 0x3e, 0x20, 0x8e, 0xb8, 0xab, 0x65, 0xb7, + 0x9a, 0x14, 0x69, 0x49, 0x98, 0x3c, 0x0b, 0x22, 0x6e, 0x4c, 0xc4, 0xb7, 0xc0, 0x5e, 0x04, 0x21, + 0x97, 0x8c, 0xcb, 0x44, 0x7e, 0xe3, 0x2f, 0x21, 0xbb, 0x17, 0x74, 0x19, 0x5a, 0x80, 0x8c, 0xd3, + 0xd4, 0x3e, 0x32, 0x4e, 0x13, 0xdd, 0x91, 0xee, 0xf5, 0x0c, 0x29, 0x27, 0x87, 0x3b, 0x20, 0x0e, + 0x91, 0x1b, 0xdf, 0x85, 0xb2, 0x13, 0x35, 0x82, 0x20, 0xec, 0x8a, 0x54, 0x07, 0xa1, 0xbe, 0x93, + 0x46, 0x41, 0xfc, 0x04, 0x2a, 0xc2, 0x7d, 0x9b, 0x53, 0x1e, 0x4f, 0xea, 0x15, 0xc8, 0x0b, 0x2c, + 0xde, 0x4e, 0x4b, 0xf2, 0xde, 0x13, 0x7a, 0x66, 0x00, 0x4a, 0x01, 0x3f, 0x57, 0x1e, 0x5a, 0xa7, + 0xcc, 0xe7, 0xa9, 0x28, 0x49, 0x59, 0x3a, 0x28, 0x13, 0x25, 0x20, 0xac, 0x8e, 0xa2, 0x39, 0x2f, + 0x24, 0x9c, 0x05, 0x4a, 0xe4, 0x1a, 0xfe, 0xd1, 0x02, 0x30, 0x84, 0x06, 0x51, 0x6c, 0x62, 0xcd, + 0x36, 0x41, 0xef, 0xa6, 0xde, 0x2e, 0x93, 0x33, 0x35, 0x5e, 0x22, 0xa9, 0x17, 0xce, 0x86, 0x19, + 0xa1, 0xba, 0x38, 0x2a, 0x89, 0xbe, 0xc2, 0x75, 0x9a, 0xc4, 0xb5, 0x59, 0x6e, 0x78, 0x83, 0x88, + 0xb3, 0x50, 0x33, 0x12, 0x6f, 0x2c, 0x05, 0xc4, 0xf1, 0x49, 0x80, 0xe9, 0x21, 0x42, 0x77, 0x21, + 0x27, 0x98, 0x9a, 0x39, 0x30, 0x7e, 0x0c, 0xb5, 0x88, 0xdb, 0xfa, 0x26, 0x99, 0x3a, 0x7b, 0x10, + 0x64, 0xe5, 0x8b, 0x5a, 0x97, 0x8b, 0x7c, 0x4c, 0x57, 0xc0, 0xde, 0x75, 0x55, 0x7d, 0xdb, 0x44, + 0x7c, 0x4a, 0x84, 0x9e, 0xcb, 0xfe, 0x13, 0x08, 0x15, 0x6f, 0x89, 0x25, 0xd5, 0x40, 0xe2, 0xee, + 0xb8, 0xc9, 0xfd, 0x66, 0x1e, 0xa5, 0x76, 0xea, 0x51, 0xda, 0x86, 0x25, 0xd5, 0x24, 0xff, 0xa5, + 0xd3, 0x9f, 0x33, 0xb0, 0x44, 0x58, 0xe4, 0xbe, 0x62, 0x8e, 0x1f, 0xf1, 0x70, 0x10, 0x0f, 0xb8, + 0x8f, 0x83, 0x23, 0x1d, 0x6a, 0x9b, 0x28, 0xe1, 0x75, 0x2a, 0x09, 0x3d, 0x10, 0xbf, 0x47, 0xa3, + 0xd5, 0x3f, 0xa9, 0x9a, 0x56, 0x41, 0x0f, 0x60, 0xae, 0x1d, 0x0c, 0xc2, 0x4e, 0x7c, 0x0d, 0xae, + 0x24, 0xda, 0x8a, 0x99, 0x5a, 0x26, 0x46, 0x2d, 0x55, 0x47, 0xb9, 0xcb, 0xeb, 0x08, 0x3d, 0x1e, + 0xab, 0x23, 0xf9, 0xe7, 0x52, 0xda, 0xfa, 0x7f, 0x62, 0x30, 0xb2, 0x4c, 0x46, 0xb5, 0xf1, 0xf7, + 0x16, 0xcc, 0xa7, 0x29, 0xbc, 0x56, 0x63, 0xc4, 0x19, 0xc9, 0x4c, 0xcd, 0x88, 0x3d, 0x2d, 0x23, + 0xd9, 0x24, 0x23, 0xc9, 0x3b, 0x37, 0x97, 0x7a, 0xe7, 0xe2, 0x13, 0x58, 0x9d, 0x48, 0x53, 0x23, + 0xe8, 0xf5, 0x45, 0x3d, 0xfc, 0x8b, 0x74, 0x89, 0x91, 0x11, 0x86, 0x3a, 0x51, 0x45, 0xa2, 0x04, + 0xfc, 0x08, 0x6e, 0xb7, 0x19, 0x4f, 0x25, 0xc9, 0x54, 0x5b, 0x1d, 0xec, 0x3d, 0x76, 0x36, 0xe3, + 0xf8, 0x62, 0x09, 0x7f, 0x08, 0xd5, 0x83, 0x7e, 0x97, 0x72, 0x76, 0x23, 0xeb, 0x1d, 0x28, 0xec, + 0x07, 0xfd, 0xc0, 0x0b, 0x8e, 0x87, 0x57, 0xb4, 0x7c, 0x15, 0xe6, 0xd4, 0x7c, 0x54, 0x8f, 0x94, + 0x22, 0x31, 0x22, 0xbe, 0x25, 0x0a, 0xba, 0x43, 0xbd, 0xce, 0xc0, 0x13, 0x34, 0xc4, 0xbf, 0x57, + 0xb4, 0x53, 0xf9, 0xf5, 0xa2, 0x66, 0xfd, 0x76, 0x51, 0xb3, 0x7e, 0xbf, 0xa8, 0x59, 0x3f, 0xfd, + 0x51, 0xfb, 0xdf, 0x51, 0x5e, 0xfe, 0xe5, 0x3f, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x66, 0x19, + 0x3d, 0xd2, 0xf6, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 5755ce2f6..8642d9a13 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -207,6 +207,10 @@ message SetCoordinatorMessage { Node New = 1; } +message UpdateCoordinatorMessage { + Node New = 1; +} + message Topology { string ClusterID = 1; repeated string NodeIDs = 2; diff --git a/server.go b/server.go index c52c28cbd..531609278 100644 --- a/server.go +++ b/server.go @@ -456,6 +456,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { } case *internal.SetCoordinatorMessage: s.Cluster.SetCoordinator(DecodeNode(obj.New)) + case *internal.UpdateCoordinatorMessage: + s.Cluster.UpdateCoordinator(DecodeNode(obj.New)) case *internal.NodeStateMessage: err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) if err != nil { From cb0ac00eb3a5d74ce4e735997589082e81af954e Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 8 Mar 2018 14:31:03 -0600 Subject: [PATCH 139/311] Switch ref links to relative urls --- docs/api-reference.md | 12 ++++++------ docs/configuration.md | 2 +- docs/examples.md | 2 +- docs/getting-started.md | 2 +- docs/introduction.md | 2 +- docs/query-language.md | 8 ++++---- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 52ea2c369..5a28313b1 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -102,9 +102,9 @@ Creates a frame in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: -* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`. -* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model#time-quantum) for this frame. +* `inverseEnabled` (boolean): Enables [the inverted view](../data-model#inverse) for this frame if `true`. +* `cacheType` (string): [ranked](../data-model#ranked) or [LRU](../data-model#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `rangeEnabled` (boolean): Enables range-encoded fields in this frame. * `fields` (array): List of range-encoded fields. @@ -205,9 +205,9 @@ Creates an input definition in the given index with the given name. The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options: -* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`. -* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model#time-quantum) for this frame. +* `inverseEnabled` (boolean): Enables [the inverted view](../data-model#inverse) for this frame if `true`. +* `cacheType` (string): [ranked](../data-model#ranked) or [LRU](../data-model#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa. diff --git a/docs/configuration.md b/docs/configuration.md index 33a78018e..692a0a17a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,7 +112,7 @@ Any flag that has a value that is a comma separated list on the command line bec #### Gossip Seed -* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. +* Description: When using the gossip [Cluster Type](#cluster-type), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. * Flag: `--gossip.seed="localhost:11101"` * Env: `PILOSA_GOSSIP_SEED="localhost:11101"` * Config: diff --git a/docs/examples.md b/docs/examples.md index 262684615..0cef7b987 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -263,7 +263,7 @@ python import_from_sdf.py -p -file id_fingerprint.csv ``` -First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +First, follow the instruction in the [getting started](../getting-started.md) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 ``` curl localhost:10101/index/mole \ diff --git a/docs/getting-started.md b/docs/getting-started.md index f68805f53..7cfcaac9b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -20,7 +20,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th ### Starting Pilosa -Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa. +Follow the steps in the [Install](../installation.md) document to install Pilosa. Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`): ``` pilosa server diff --git a/docs/introduction.md b/docs/introduction.md index d499c8c0a..a49bd1539 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w "What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface. -Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. +Once you have Pilosa [installed](../installation.md), the [getting started](../getting-started.md) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. diff --git a/docs/query-language.md b/docs/query-language.md index 56d7c2087..105893e66 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -13,7 +13,7 @@ nav = [ ### Overview -This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: +This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: ``` {"results":[...]} @@ -45,7 +45,7 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types -* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `frame` The frame specifies on which Pilosa [frame](../glossary#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. * `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated. * `COL_LABEL` The default column label is `columnID`, changing the default is deprecated. * `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") @@ -118,7 +118,7 @@ SetRowAttrs queries always return `null` upon success. SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true) ``` -Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. +Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. ``` SetRowAttrs(frame="stargazer", rowID=10, username=null) @@ -150,7 +150,7 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` -Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", columnID=10)`. +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`. ``` SetColumnAttrs(columnID=10, url=null) From 85799739af929f270ec12b8c39b7649a29967e89 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 8 Mar 2018 15:55:21 -0600 Subject: [PATCH 140/311] Address review comments --- docs/README.md | 2 +- docs/examples.md | 2 +- docs/getting-started.md | 2 +- docs/glossary.md | 4 ++-- docs/introduction.md | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/README.md b/docs/README.md index 31b60a622..4bad84cd7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,5 @@ Pilosa docs are maintained here, to stay in sync with the codebase. -Please visit [our website](https://www.pilosa.com/docs) to view the docs complete with styles, diagrams, and comprehensive search. +Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search. Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request! diff --git a/docs/examples.md b/docs/examples.md index 0cef7b987..12dd69617 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -263,7 +263,7 @@ python import_from_sdf.py -p -file id_fingerprint.csv ``` -First, follow the instruction in the [getting started](../getting-started.md) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +First, follow the instruction in the [getting started](../getting-started) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 ``` curl localhost:10101/index/mole \ diff --git a/docs/getting-started.md b/docs/getting-started.md index 7cfcaac9b..6a9eb54be 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -20,7 +20,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th ### Starting Pilosa -Follow the steps in the [Install](../installation.md) document to install Pilosa. +Follow the steps in the [Install](../installation) document to install Pilosa. Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`): ``` pilosa server diff --git a/docs/glossary.md b/docs/glossary.md index 5b2a24ffd..f6fa29478 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -40,9 +40,9 @@ nav = [] [Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. -[Range](../query-lanuage/#range):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). +[Range](../query-language/#range):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). -[Range (BSI)](../query-lanuage/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). +[Range (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). [Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. diff --git a/docs/introduction.md b/docs/introduction.md index a49bd1539..7e7049c6c 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w "What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface. -Once you have Pilosa [installed](../installation.md), the [getting started](../getting-started.md) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. +Once you have Pilosa [installed](../installation), the [getting started](../getting-started) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. From 13e3b0444335779da311fa7bc1f948faa1ea9555 Mon Sep 17 00:00:00 2001 From: Ilias Dimos Date: Fri, 9 Mar 2018 14:12:15 +0200 Subject: [PATCH 141/311] Fix misspells in comments --- ctl/import_test.go | 2 +- executor.go | 2 +- fragment_test.go | 2 +- gossip/gossip.go | 2 +- handler.go | 2 +- roaring/roaring.go | 4 ++-- roaring/roaring_test.go | 2 +- statsd/statsd.go | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index 5979bdbee..254c21a05 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -85,7 +85,7 @@ func TestImportCommand_Run(t *testing.T) { } } -// Ensure that the ImportValue path runs (note: we have specifed a value +// Ensure that the ImportValue path runs (note: we have specified a value // for cm.Field. Because the handler doesn't return errors (it sends them // to the logger), we don't get an error returned at `cm.Run()` even though // we haven't setup frame `f` to be RangeEnabled. diff --git a/executor.go b/executor.go index d12aeff1f..9e831c9f0 100644 --- a/executor.go +++ b/executor.go @@ -801,7 +801,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return NewBitmap(), nil } - // LT[E] and GT[E] should return all not-null if selected range fully encompases valid field range. + // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range. if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) || (cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) { return frag.FieldNotNull(field.BitDepth()) diff --git a/fragment_test.go b/fragment_test.go index 8a54d3b07..c08b20a21 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -163,7 +163,7 @@ func TestFragment_SetFieldValue(t *testing.T) { t.Fatal("expected change") } - // Non-existant value. + // Non-existent value. if value, exists, err := f.FieldValue(100, 11); err != nil { t.Fatal(err) } else if value != 0 { diff --git a/gossip/gossip.go b/gossip/gossip.go index 3afdabe31..5a6233a94 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -135,7 +135,7 @@ type gossipConfig struct { // newTransport returns a NetTransport based on the memberlist configuration. // It will dynamically bind to a port if conf.BindPort is 0. -// This is useful for test cases where specifiying a port is not reasonable. +// This is useful for test cases where specifying a port is not reasonable. func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { if conf.LogOutput != nil && conf.Logger != nil { return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") diff --git a/handler.go b/handler.go index ae04625a6..ca2e6a4ef 100644 --- a/handler.go +++ b/handler.go @@ -995,7 +995,7 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ingore this error becuase views do not exist on all nodes due to slice distribution. + // Ingore this error because views do not exist on all nodes due to slice distribution. if err != ErrInvalidView { http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/roaring/roaring.go b/roaring/roaring.go index 3cc537a08..8fb906ef4 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2645,7 +2645,7 @@ func differenceRunRun(a, b *container) *container { for apos < alen && bpos < blen { switch { case alast < bstart: - // current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run + // current A-run entirely precedes current B-run: keep full A-run, advance to next A-run output.runs = append(output.runs, interval16{start: astart, last: alast}) apos++ if apos < alen { @@ -2653,7 +2653,7 @@ func differenceRunRun(a, b *container) *container { alast = a.runs[apos].last } case blast < astart: - // current B-run entirely preceeds current A-run: advance to next B-run + // current B-run entirely precedes current A-run: advance to next B-run bpos++ if bpos < blen { bstart = b.runs[bpos].start diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 6378ee67f..7c5539b34 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -291,7 +291,7 @@ func TestBitmap_Max(t *testing.T) { } } -// Ensure CountRange is correct even if rangekey is prior to inital container. +// Ensure CountRange is correct even if rangekey is prior to initial container. func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { s := uint64(2009 * 1048576) e := uint64(2010 * 1048576) diff --git a/statsd/statsd.go b/statsd/statsd.go index d46dd637f..c7c072ae3 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -24,7 +24,7 @@ import ( "github.com/pilosa/pilosa" ) -// StatsD protocal wrapper using the DataDog library that added Tags to the StatsD protocal +// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol // statsD defailt host is "127.0.0.1:8125" const ( From 1b4445a00d2c25d1192a06ce01e55c3f47074f0d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 9 Mar 2018 15:27:00 -0600 Subject: [PATCH 142/311] Fix formatting --- docs/configuration.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 692a0a17a..bc9bd75b0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -218,7 +218,8 @@ Any flag that has a value that is a comma separated list on the command line bec [profile] cpu-time = "30s" ``` -##### Metric Service + +#### Metric Service * Description: Which stats service to use. Choose from [statsd, expvar]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' @@ -229,7 +230,7 @@ Any flag that has a value that is a comma separated list on the command line bec service = “statsd” ``` -##### Metric Host +#### Metric Host * Description: Address of the StatsD service host. * Flag: `--metric.host=localhost:8125` * Env: `PILOSA_METRIC_HOST=localhost:8125' @@ -240,7 +241,7 @@ Any flag that has a value that is a comma separated list on the command line bec host = "localhost:8125" ``` -##### Metric Poll Interval +#### Metric Poll Interval * Description: Polling interval for runtime metrics. * Flag: `metric.poll-interval=”0m15s”` @@ -252,7 +253,7 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "0m15s" ``` -##### Metric Diagnostics +#### Metric Diagnostics * Description: Enable diagnostic reporting. To disable diagnostics set to false. * Flag: `metric.diagnostics` @@ -265,7 +266,7 @@ Any flag that has a value that is a comma separated list on the command line bec ``` -##### TLS Certificate +#### TLS Certificate * Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions. * Flag: `tls.certificate=/srv/pilosa/certs/server.crt` @@ -277,7 +278,7 @@ Any flag that has a value that is a comma separated list on the command line bec certificate = "/srv/pilosa/certs/server.crt" ``` -##### TLS Certificate Key +#### TLS Certificate Key * Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension. * Flag: `tls.key=/srv/pilosa/certs/server.key` @@ -289,7 +290,7 @@ Any flag that has a value that is a comma separated list on the command line bec key = "/srv/pilosa/certs/server.key" ``` -##### TLS Skip Verify +#### TLS Skip Verify * Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial. * Flag: `tls.skip-verify` From 8cfe7583cec6426b060cf2b645daf5238c3b19cc Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 9 Mar 2018 15:27:21 -0600 Subject: [PATCH 143/311] Fix broken anchor links --- docs/administration.md | 6 +++--- docs/glossary.md | 10 +++++----- docs/tutorials.md | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index bf3da9626..d3548553f 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -150,9 +150,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: - - [Host](../configuration#metrics-host): specify host that receives metric events - - [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics - - [Service](../configuration#metrics-service): declare type StatsD or Expvar + - [Host](../configuration#metric-host): specify host that receives metric events + - [Poll Interval](../configuration#metric-poll-interval): specify polling interval for runtime metrics + - [Service](../configuration#metric-service): declare type StatsD or Expvar #### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: diff --git a/docs/glossary.md b/docs/glossary.md index f6fa29478..6ea0549ef 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -14,13 +14,13 @@ nav = [] [Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. -[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range) and [Sum](#sum) queries. +[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries. Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. [Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). -[Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range) and [Sum](#sum) queries. +[Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries. Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). @@ -30,7 +30,7 @@ nav = [] [Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#columns). This value is important for all [nodes](#node) to efficiently distribute queries. +MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). @@ -40,9 +40,9 @@ nav = [] [Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. -[Range](../query-language/#range):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). +[Range](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). -[Range (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). +[Range (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). [Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. diff --git a/docs/tutorials.md b/docs/tutorials.md index 9e6364cf0..f9b4a1494 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -24,7 +24,7 @@ This tutorial assumes that you are using a UNIX-like system, such as Linux or Ma #### Installing Pilosa and Creating the Directory Structure -If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source). +If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source). After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line: ``` @@ -134,7 +134,7 @@ Here is some explanation of the configuration items: * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. +* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. * `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. From 510c64ef0649df4905cfe4c8353b148d8bcf9c85 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 12 Mar 2018 16:32:35 -0500 Subject: [PATCH 144/311] Move diagnostics into package pilosa --- broadcast.go | 8 +- diagnostics/diagnostics.go => diagnostics.go | 103 +++++++++--------- ...diagnostics_test.go => diagnostics_test.go | 41 +++---- gc.go | 4 +- server.go | 43 +------- 5 files changed, 77 insertions(+), 122 deletions(-) rename diagnostics/diagnostics.go => diagnostics.go (73%) rename diagnostics/diagnostics_test.go => diagnostics_test.go (83%) diff --git a/broadcast.go b/broadcast.go index a4e6403c1..de43f3b85 100644 --- a/broadcast.go +++ b/broadcast.go @@ -63,17 +63,17 @@ var NopBroadcaster Broadcaster type nopBroadcaster struct{} -// SendSync A no-op implemenetation of Broadcaster SendSync method. +// SendSync A no-op implementation of Broadcaster SendSync method. func (n *nopBroadcaster) SendSync(pb proto.Message) error { return nil } -// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +// SendAsync A no-op implementation of Broadcaster SendAsync method. func (n *nopBroadcaster) SendAsync(pb proto.Message) error { return nil } -// SendTo is a no-op implemenetation of Broadcaster SendTo method. +// SendTo is a no-op implementation of Broadcaster SendTo method. func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil } @@ -112,7 +112,7 @@ var NopGossiper Gossiper type nopGossiper struct{} -// SendAsync A no-op implemenetation of Gossiper SendAsync method. +// SendAsync A no-op implementation of Gossiper SendAsync method. func (n *nopGossiper) SendAsync(pb proto.Message) error { return nil } diff --git a/diagnostics/diagnostics.go b/diagnostics.go similarity index 73% rename from diagnostics/diagnostics.go rename to diagnostics.go index 60248d6bb..602e910e0 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package diagnostics +package pilosa import ( "bytes" @@ -36,7 +36,7 @@ import ( // Default version check URL. const ( - DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" + defaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" ) type versionResponse struct { @@ -44,11 +44,9 @@ type versionResponse struct { Message string `json:"message"` } -// Diagnostics represents a client to the Pilosa cluster. -type Diagnostics struct { +// DiagnosticsCollector represents a collector/sender of diagnostics data +type DiagnosticsCollector struct { mu sync.Mutex - wg sync.WaitGroup - closing chan struct{} host string VersionURL string version string @@ -65,13 +63,12 @@ type Diagnostics struct { logOutput io.Writer } -// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". -func New(host string) *Diagnostics { +// New returns a pointer to a new DiagnosticsCollector Client given an addr in the format "hostname:port". +func NewDiagnosticsCollector(host string) *DiagnosticsCollector { - return &Diagnostics{ - closing: make(chan struct{}), + return &DiagnosticsCollector{ host: host, - VersionURL: DefaultVersionCheckURL, + VersionURL: defaultVersionCheckURL, startTime: time.Now().Unix(), start: time.Now(), client: http.DefaultClient, @@ -81,37 +78,21 @@ func New(host string) *Diagnostics { } // SetVersion of locally running Pilosa Cluster to check against master. -func (d *Diagnostics) SetVersion(v string) { +func (d *DiagnosticsCollector) SetVersion(v string) { d.version = v d.Set("Version", v) } // SetInterval of the diagnostic go routine and match with the circuit breaker timeout. -func (d *Diagnostics) SetInterval(i time.Duration) { +func (d *DiagnosticsCollector) SetInterval(i time.Duration) { d.interval = i } -// schedule start the diagnostics service ticker. -func (d *Diagnostics) schedule() { - ticker := time.NewTicker(d.interval) - defer ticker.Stop() - - for { - select { - case <-d.closing: - return - case <-ticker.C: - d.CheckVersion() - d.Flush() - } - } -} - // Flush sends the current metrics. -func (d *Diagnostics) Flush() error { +func (d *DiagnosticsCollector) Flush() error { d.mu.Lock() d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) - buf, _ := d.Encode() + buf, _ := d.encode() d.mu.Unlock() _, err := d.cb.Execute(func() (interface{}, error) { @@ -135,7 +116,7 @@ func (d *Diagnostics) Flush() error { } // Open configures the circuit breaker used by the HTTP client. -func (d *Diagnostics) Open() { +func (d *DiagnosticsCollector) Open() { var st gobreaker.Settings if d.interval > 0 { st.Timeout = d.interval * 2 @@ -145,15 +126,8 @@ func (d *Diagnostics) Open() { d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics") } -// Close notify goroutine to stop. -func (d *Diagnostics) Close() error { - close(d.closing) - d.wg.Wait() - return nil -} - // CheckVersion of the local build against Pilosa master. -func (d *Diagnostics) CheckVersion() error { +func (d *DiagnosticsCollector) CheckVersion() error { var rsp versionResponse req, err := http.NewRequest("GET", d.VersionURL, nil) resp, err := d.client.Do(req) @@ -174,15 +148,15 @@ func (d *Diagnostics) CheckVersion() error { } d.lastVersion = rsp.Version - if err := d.CompareVersion(rsp.Version); err != nil { + if err := d.compareVersion(rsp.Version); err != nil { d.logger().Printf("%s\n", err.Error()) } return nil } -// CompareVersion check version strings. -func (d *Diagnostics) CompareVersion(value string) error { +// compareVersion check version strings. +func (d *DiagnosticsCollector) compareVersion(value string) error { currentVersion := VersionSegments(value) localVersion := VersionSegments(d.version) @@ -198,29 +172,29 @@ func (d *Diagnostics) CompareVersion(value string) error { } // Encode metrics maps into the json message format. -func (d *Diagnostics) Encode() ([]byte, error) { +func (d *DiagnosticsCollector) encode() ([]byte, error) { return json.Marshal(d.metrics) } // Set adds a key value metric. -func (d *Diagnostics) Set(name string, value interface{}) { +func (d *DiagnosticsCollector) Set(name string, value interface{}) { d.mu.Lock() defer d.mu.Unlock() d.metrics[name] = value } // SetLogger Set the logger output type. -func (d *Diagnostics) SetLogger(logger io.Writer) { +func (d *DiagnosticsCollector) SetLogger(logger io.Writer) { d.logOutput = logger } // logger returns a logger that writes to LogOutput. -func (d *Diagnostics) logger() *log.Logger { +func (d *DiagnosticsCollector) logger() *log.Logger { return log.New(d.logOutput, "", log.LstdFlags) } // EnrichWithOSInfo adds OS information to the diagnostics payload. -func (d *Diagnostics) EnrichWithOSInfo() { +func (d *DiagnosticsCollector) EnrichWithOSInfo() { osInfo, err := host.Info() if err != nil { d.logOutput.Write([]byte(err.Error())) @@ -243,7 +217,7 @@ func (d *Diagnostics) EnrichWithOSInfo() { } // EnrichWithMemoryInfo adds memory information to the diagnostics payload. -func (d *Diagnostics) EnrichWithMemoryInfo() { +func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { memory, err := mem.VirtualMemory() if err != nil { d.logOutput.Write([]byte(err.Error())) @@ -254,6 +228,37 @@ func (d *Diagnostics) EnrichWithMemoryInfo() { } +// EnrichWithSchemaProperties adds schema info to the diagnostics payload. +func (d *DiagnosticsCollector) EnrichWithSchemaProperties(holder *Holder) { + var numSlices uint64 + numFrames := 0 + numIndexes := 0 + bsiFieldCount := 0 + timeQuantumEnabled := false + + for _, index := range holder.Indexes() { + numSlices += index.MaxSlice() + 1 + numIndexes += 1 + for _, frame := range index.Frames() { + numFrames += 1 + if frame.rangeEnabled { + if fields, err := frame.GetFields(); err == nil { + bsiFieldCount += len(fields) + } + } + if frame.TimeQuantum() != "" { + timeQuantumEnabled = true + } + } + } + + d.Set("NumIndexes", numIndexes) + d.Set("NumFrames", numFrames) + d.Set("NumSlices", numSlices) + d.Set("BSIFieldCount", bsiFieldCount) + d.Set("TimeQuantumEnabled", timeQuantumEnabled) +} + // VersionSegments returns the numeric segments of the version as a slice of ints. func VersionSegments(segments string) []int { segments = strings.Trim(segments, "v") diff --git a/diagnostics/diagnostics_test.go b/diagnostics_test.go similarity index 83% rename from diagnostics/diagnostics_test.go rename to diagnostics_test.go index 8f85a57db..9d8e35d76 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package diagnostics_test +package pilosa import ( "encoding/json" @@ -23,25 +23,21 @@ import ( "runtime" "strings" "testing" - - "github.com/pilosa/pilosa/diagnostics" ) func TestDiagnosticsClient(t *testing.T) { // Mock server. server := httptest.NewServer(nil) - defer server.Close() // Create a new client. - d := diagnostics.New(server.URL) + d := NewDiagnosticsCollector(server.URL) d.SetLogger(ioutil.Discard) d.Open() - defer d.Close() d.Set("gg", 10) d.Set("ss", "ss") - data, err := d.Encode() + data, err := d.encode() if err != nil { t.Fatal(err) } @@ -58,7 +54,7 @@ func TestDiagnosticsClient(t *testing.T) { // Test the metrics after a flush. d.Flush() - data, err = d.Encode() + data, err = d.encode() if err != nil { t.Fatal(err) } @@ -74,7 +70,7 @@ func TestDiagnosticsClient(t *testing.T) { func TestDiagnosticsVersion_Parse(t *testing.T) { version := "0.1.1" - vs := diagnostics.VersionSegments(version) + vs := VersionSegments(version) output := []int{0, 1, 1} if !reflect.DeepEqual(vs, output) { @@ -83,35 +79,34 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { } func TestDiagnosticsVersion_Compare(t *testing.T) { - d := diagnostics.New("localhost:10101") + d := NewDiagnosticsCollector("localhost:10101") d.Open() - defer d.Close() version := "v0.1.1" d.SetVersion(version) - err := d.CompareVersion("v1.7.0") + err := d.compareVersion("v1.7.0") if !strings.Contains(err.Error(), "A newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } - err = d.CompareVersion("1.7.0") + err = d.compareVersion("1.7.0") if !strings.Contains(err.Error(), "A newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } - err = d.CompareVersion("0.7.0") + err = d.compareVersion("0.7.0") if !strings.Contains(err.Error(), "The latest Minor release is") { t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) } - err = d.CompareVersion("0.1.2") + err = d.compareVersion("0.1.2") if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") { t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) } - err = d.CompareVersion("0.1.1") + err = d.compareVersion("0.1.1") if err != nil { t.Fatalf("Versions should match") } d.SetVersion("v1.7.0") - err = d.CompareVersion("0.7.2") + err = d.compareVersion("0.7.2") if err != nil { t.Fatalf("Local version is greater") } @@ -125,11 +120,9 @@ func TestDiagnosticsVersion_Check(t *testing.T) { Version: "1.1.1", }) })) - defer server.Close() // Create a new client. - d := diagnostics.New("localhost:10101") - defer d.Close() + d := NewDiagnosticsCollector("localhost:10101") version := "0.1.1" d.SetVersion(version) @@ -138,10 +131,6 @@ func TestDiagnosticsVersion_Check(t *testing.T) { d.CheckVersion() } -type versionResponse struct { - Version string `json:"version"` -} - func compareJSON(a, b []byte) (bool, error) { var j1, j2 interface{} if err := json.Unmarshal(a, &j1); err != nil { @@ -156,12 +145,10 @@ func compareJSON(a, b []byte) (bool, error) { func BenchmarkDiagnostics(b *testing.B) { // Mock server. server := httptest.NewServer(nil) - defer server.Close() // Create a new client. - d := diagnostics.New(server.URL) + d := NewDiagnosticsCollector(server.URL) d.SetLogger(ioutil.Discard) - defer d.Close() prev := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(prev) diff --git a/gc.go b/gc.go index a260cc0b5..3f036bdcd 100644 --- a/gc.go +++ b/gc.go @@ -29,10 +29,10 @@ var NopGCNotifier GCNotifier type nopGCNotifier struct{} -// Close is a no-op implemenetation of GCNotifier Close method. +// Close is a no-op implementation of GCNotifier Close method. func (n *nopGCNotifier) Close() {} -// AfterGC is a no-op implemenetation of GCNotifier AfterGC method. +// AfterGC is a no-op implementation of GCNotifier AfterGC method. func (c *nopGCNotifier) AfterGC() <-chan struct{} { return nil } diff --git a/server.go b/server.go index 531609278..82ca2db2d 100644 --- a/server.go +++ b/server.go @@ -32,7 +32,6 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" "golang.org/x/sync/errgroup" @@ -70,7 +69,7 @@ type Server struct { NodeID string URI URI Cluster *Cluster - diagnostics *diagnostics.Diagnostics + diagnostics *DiagnosticsCollector GCNotifier GCNotifier @@ -100,7 +99,7 @@ func NewServer() *Server { Handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, - diagnostics: diagnostics.New(DefaultDiagnosticServer), + diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), Network: "tcp", @@ -622,13 +621,13 @@ func (s *Server) monitorDiagnostics() { // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - enrichDiagnosticsWithSchemaProperties(s.diagnostics, s.Holder) openFiles, err := CountOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.EnrichWithMemoryInfo() + s.diagnostics.EnrichWithSchemaProperties(s.Holder) s.diagnostics.CheckVersion() s.diagnostics.Flush() } @@ -725,39 +724,3 @@ type StatusHandler interface { ClusterStatus() (proto.Message, error) HandleRemoteStatus(proto.Message) error } - -type diagnosticsFrameProperties struct { - BSIFieldCount int - TimeQuantumEnabled bool -} - -func enrichDiagnosticsWithSchemaProperties(d *diagnostics.Diagnostics, holder *Holder) { - // NOTE: this function is not in the diagnostics package, since circular imports are not allowed. - var numSlices uint64 - numFrames := 0 - numIndexes := 0 - bsiFieldCount := 0 - timeQuantumEnabled := false - - for _, index := range holder.Indexes() { - numSlices += index.MaxSlice() + 1 - numIndexes += 1 - for _, frame := range index.Frames() { - numFrames += 1 - if frame.rangeEnabled { - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields) - } - } - if frame.TimeQuantum() != "" { - timeQuantumEnabled = true - } - } - } - - d.Set("NumIndexes", numIndexes) - d.Set("NumFrames", numFrames) - d.Set("NumSlices", numSlices) - d.Set("BSIFieldCount", bsiFieldCount) - d.Set("TimeQuantumEnabled", timeQuantumEnabled) -} From f4c1e0c492f772b979279f591461f825c7980151 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 13 Mar 2018 11:41:45 -0500 Subject: [PATCH 145/311] Refactor gopsutil into SystemInfo interface/subpackage for dependency injection. --- diagnostics.go | 131 ++++++++++++++++++++++++++++++++--------- gc.go | 3 + gopsutil/systeminfo.go | 116 ++++++++++++++++++++++++++++++++++++ server.go | 5 +- server/server.go | 2 + 5 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 gopsutil/systeminfo.go diff --git a/diagnostics.go b/diagnostics.go index 602e910e0..a4caf84cd 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -27,13 +27,9 @@ import ( "sync" "time" - "github.com/shirou/gopsutil/host" - "github.com/shirou/gopsutil/mem" "github.com/sony/gobreaker" ) -// TODO: unique Cluster ID - // Default version check URL. const ( defaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" @@ -61,6 +57,8 @@ type DiagnosticsCollector struct { cb *gobreaker.CircuitBreaker logOutput io.Writer + + server *Server } // New returns a pointer to a new DiagnosticsCollector Client given an addr in the format "hostname:port". @@ -193,50 +191,64 @@ func (d *DiagnosticsCollector) logger() *log.Logger { return log.New(d.logOutput, "", log.LstdFlags) } +// logErr logs the error and returns true if an error exists +func (d *DiagnosticsCollector) logErr(err error) bool { + if err != nil { + d.logOutput.Write([]byte(err.Error())) + return true + } + return false +} + // EnrichWithOSInfo adds OS information to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithOSInfo() { - osInfo, err := host.Info() - if err != nil { - d.logOutput.Write([]byte(err.Error())) + uptime, err := d.server.SystemInfo.Uptime() + if !d.logErr(err) { + d.Set("HostUptime", uptime) } - d.Set("HostUptime", osInfo.Uptime) - - platform, family, version, err := host.PlatformInformation() - if err != nil { - d.logOutput.Write([]byte(err.Error())) + platform, err := d.server.SystemInfo.Platform() + if !d.logErr(err) { + d.Set("OSPlatform", platform) } - d.Set("OSPlatform", platform) - d.Set("OSFamily", family) - d.Set("OSVersion", version) - - kernelVersion, err := host.KernelVersion() - if err != nil { - d.logOutput.Write([]byte(err.Error())) + family, err := d.server.SystemInfo.Family() + if !d.logErr(err) { + d.Set("OSFamily", family) + } + version, err := d.server.SystemInfo.OSVersion() + if !d.logErr(err) { + d.Set("OSVersion", version) + } + kernelVersion, err := d.server.SystemInfo.KernelVersion() + if !d.logErr(err) { + d.Set("OSKernelVersion", kernelVersion) } - d.Set("OSKernelVersion", kernelVersion) } // EnrichWithMemoryInfo adds memory information to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { - memory, err := mem.VirtualMemory() - if err != nil { - d.logOutput.Write([]byte(err.Error())) + memFree, err := d.server.SystemInfo.MemFree() + if !d.logErr(err) { + d.Set("MemFree", memFree) + } + memTotal, err := d.server.SystemInfo.MemTotal() + if !d.logErr(err) { + d.Set("MemTotal", memTotal) + } + memUsed, err := d.server.SystemInfo.MemUsed() + if !d.logErr(err) { + d.Set("MemUsed", memUsed) } - d.Set("MemFree", memory.Free) - d.Set("MemTotal", memory.Total) - d.Set("MemUsed", memory.Used) - } // EnrichWithSchemaProperties adds schema info to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithSchemaProperties(holder *Holder) { +func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { var numSlices uint64 numFrames := 0 numIndexes := 0 bsiFieldCount := 0 timeQuantumEnabled := false - for _, index := range holder.Indexes() { + for _, index := range d.server.Holder.Indexes() { numSlices += index.MaxSlice() + 1 numIndexes += 1 for _, frame := range index.Frames() { @@ -270,3 +282,64 @@ func VersionSegments(segments string) []int { } return segmentSlice } + +// SystemInfo collects information about the host OS +type SystemInfo interface { + Uptime() (uint64, error) + Platform() (string, error) + Family() (string, error) + OSVersion() (string, error) + KernelVersion() (string, error) + MemFree() (uint64, error) + MemTotal() (uint64, error) + MemUsed() (uint64, error) +} + +// NewNopSystemInfo creates a no-op implementation of SystemInfo +func NewNopSystemInfo() *NopSystemInfo { + return &NopSystemInfo{} +} + +// NopSystemInfo is a no-op implementation of SystemInfo +type NopSystemInfo struct { +} + +// Uptime is a no-op implementation of SystemInfo.Uptime +func (n *NopSystemInfo) Uptime() (uint64, error) { + return 0, nil +} + +// Platform is a no-op implementation of SystemInfo.Platform +func (n *NopSystemInfo) Platform() (string, error) { + return "", nil +} + +// Family is a no-op implementation of SystemInfo.Family +func (n *NopSystemInfo) Family() (string, error) { + return "", nil +} + +// OSVersion is a no-op implementation of SystemInfo.OSVersion +func (n *NopSystemInfo) OSVersion() (string, error) { + return "", nil +} + +// KernelVersion is a no-op implementation of SystemInfo.KernelVersion +func (n *NopSystemInfo) KernelVersion() (string, error) { + return "", nil +} + +// MemFree is a no-op implementation of SystemInfo.MemFree +func (n *NopSystemInfo) MemFree() (uint64, error) { + return 0, nil +} + +// MemTotal is a no-op implementation of SystemInfo.MemTotal +func (n *NopSystemInfo) MemTotal() (uint64, error) { + return 0, nil +} + +// MemUsed is a no-op implementation of SystemInfo.MemUsed +func (n *NopSystemInfo) MemUsed() (uint64, error) { + return 0, nil +} diff --git a/gc.go b/gc.go index 3f036bdcd..23dd0f0d0 100644 --- a/gc.go +++ b/gc.go @@ -14,6 +14,9 @@ package pilosa +// Ensure nopGCNotifier implements interface. +var _ GCNotifier = &nopGCNotifier{} + // GCNotifier represents an interface for garbage collection notificationss. type GCNotifier interface { Close() diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go new file mode 100644 index 000000000..12c300ba5 --- /dev/null +++ b/gopsutil/systeminfo.go @@ -0,0 +1,116 @@ +package gopsutil + +import ( + "github.com/pilosa/pilosa" + "github.com/shirou/gopsutil/host" + "github.com/shirou/gopsutil/mem" +) + +var _ pilosa.SystemInfo = NewSystemInfo() + +// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS +type SystemInfo struct { + hostInfo *host.InfoStat + memInfo *mem.VirtualMemoryStat + platform string + family string + osVersion string +} + +// Uptime returns the system uptime in seconds +func (s *SystemInfo) Uptime() (uptime uint64, err error) { + if s.hostInfo == nil { + s.hostInfo, err = host.Info() + if err != nil { + return 0, err + } + } + return s.hostInfo.Uptime, nil +} + +// Uptime returns the system platform +func (s *SystemInfo) Platform() (string, error) { + err := s.collectPlatformInfo() + if err != nil { + return "", err + } + return s.platform, nil +} + +// Family returns the system family +func (s *SystemInfo) Family() (string, error) { + err := s.collectPlatformInfo() + if err != nil { + return "", err + } + return s.family, err +} + +// OSVersion returns the OS Version +func (s *SystemInfo) OSVersion() (string, error) { + err := s.collectPlatformInfo() + if err != nil { + return "", err + } + return s.osVersion, err +} + +// collectPlatformInfo fetches and caches system platform information +func (s *SystemInfo) collectPlatformInfo() error { + var err error + if s.platform == "" { + s.platform, s.family, s.osVersion, err = host.PlatformInformation() + if err != nil { + return err + } + } + return nil +} + +// collectMemoryInfo fetches and caches memory stats +func (s *SystemInfo) collectMemoryInfo() (err error) { + if s.memInfo == nil { + s.memInfo, err = mem.VirtualMemory() + if err != nil { + return err + } + } + return nil +} + +// MemFree returns the amount of free memory in bytes +func (s *SystemInfo) MemFree() (uint64, error) { + err := s.collectMemoryInfo() + if err != nil { + return 0, err + } + return s.memInfo.Free, err +} + +// MemFree returns the amount of total memory in bytes +func (s *SystemInfo) MemTotal() (uint64, error) { + err := s.collectMemoryInfo() + if err != nil { + return 0, err + } + return s.memInfo.Total, err +} + +// MemFree returns the amount of used memory in bytes +func (s *SystemInfo) MemUsed() (uint64, error) { + err := s.collectMemoryInfo() + if err != nil { + return 0, err + } + return s.memInfo.Used, err +} + +// KernelVersion returns the kernel version as a string +func (s *SystemInfo) KernelVersion() (string, error) { + return host.KernelVersion() +} + +// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo +func NewSystemInfo() *SystemInfo { + return &SystemInfo{} +} diff --git a/server.go b/server.go index 82ca2db2d..bd0611041 100644 --- a/server.go +++ b/server.go @@ -70,6 +70,7 @@ type Server struct { URI URI Cluster *Cluster diagnostics *DiagnosticsCollector + SystemInfo SystemInfo GCNotifier GCNotifier @@ -100,6 +101,7 @@ func NewServer() *Server { Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), + SystemInfo: NewNopSystemInfo(), Network: "tcp", @@ -114,6 +116,7 @@ func NewServer() *Server { s.logger = log.New(s.LogOutput, "", log.LstdFlags) s.Handler.Holder = s.Holder + s.diagnostics.server = s return s } @@ -627,7 +630,7 @@ func (s *Server) monitorDiagnostics() { } s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.EnrichWithMemoryInfo() - s.diagnostics.EnrichWithSchemaProperties(s.Holder) + s.diagnostics.EnrichWithSchemaProperties() s.diagnostics.CheckVersion() s.diagnostics.Flush() } diff --git a/server/server.go b/server/server.go index 240b8b16b..eac98121d 100644 --- a/server/server.go +++ b/server/server.go @@ -34,6 +34,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gcnotify" + "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -152,6 +153,7 @@ func (m *Command) SetupServer() error { if m.Config.Metric.Diagnostics { m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval) } + m.Server.SystemInfo = gopsutil.NewSystemInfo() m.Server.GCNotifier = gcnotify.NewActiveGCNotifier() m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) if err != nil { From c2444870c6835c6e5d4cf785f001f23610391617 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 13 Mar 2018 12:27:49 -0500 Subject: [PATCH 146/311] Remove gobreaker dep and add HTTP timeout --- Gopkg.lock | 6 ----- diagnostics.go | 60 ++++++++++++++------------------------------- diagnostics_test.go | 2 -- server.go | 12 ++++++--- 4 files changed, 27 insertions(+), 53 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c77dc951a..af6174c08 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -205,12 +205,6 @@ packages = ["."] revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" -[[projects]] - name = "github.com/sony/gobreaker" - packages = ["."] - revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36" - version = "0.3.0" - [[projects]] branch = "master" name = "github.com/spf13/afero" diff --git a/diagnostics.go b/diagnostics.go index a4caf84cd..1abcdd144 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -26,8 +26,6 @@ import ( "strings" "sync" "time" - - "github.com/sony/gobreaker" ) // Default version check URL. @@ -52,10 +50,8 @@ type DiagnosticsCollector struct { metrics map[string]interface{} - client *http.Client - interval time.Duration + client *http.Client - cb *gobreaker.CircuitBreaker logOutput io.Writer server *Server @@ -69,7 +65,7 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector { VersionURL: defaultVersionCheckURL, startTime: time.Now().Unix(), start: time.Now(), - client: http.DefaultClient, + client: &http.Client{Timeout: 10 * time.Second}, metrics: make(map[string]interface{}), logOutput: ioutil.Discard, } @@ -81,47 +77,29 @@ func (d *DiagnosticsCollector) SetVersion(v string) { d.Set("Version", v) } -// SetInterval of the diagnostic go routine and match with the circuit breaker timeout. -func (d *DiagnosticsCollector) SetInterval(i time.Duration) { - d.interval = i -} - // Flush sends the current metrics. func (d *DiagnosticsCollector) Flush() error { d.mu.Lock() + defer d.mu.Unlock() d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) - buf, _ := d.encode() - d.mu.Unlock() - - _, err := d.cb.Execute(func() (interface{}, error) { - req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - resp, err := d.client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // TODO verify response - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return body, nil - }) - - return err -} - -// Open configures the circuit breaker used by the HTTP client. -func (d *DiagnosticsCollector) Open() { - var st gobreaker.Settings - if d.interval > 0 { - st.Timeout = d.interval * 2 + buf, err := d.encode() + if err != nil { + return err } - d.cb = gobreaker.NewCircuitBreaker(st) + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() - d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics") + // TODO verify response + _, err = ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return nil } // CheckVersion of the local build against Pilosa master. diff --git a/diagnostics_test.go b/diagnostics_test.go index 9d8e35d76..7b66d8e18 100644 --- a/diagnostics_test.go +++ b/diagnostics_test.go @@ -32,7 +32,6 @@ func TestDiagnosticsClient(t *testing.T) { // Create a new client. d := NewDiagnosticsCollector(server.URL) d.SetLogger(ioutil.Discard) - d.Open() d.Set("gg", 10) d.Set("ss", "ss") @@ -80,7 +79,6 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { func TestDiagnosticsVersion_Compare(t *testing.T) { d := NewDiagnosticsCollector("localhost:10101") - d.Open() version := "v0.1.1" d.SetVersion(version) diff --git a/server.go b/server.go index bd0611041..dd8811463 100644 --- a/server.go +++ b/server.go @@ -605,15 +605,16 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // monitorDiagnostics periodically polls the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { - if s.DiagnosticInterval <= 0 { + // Do not send more than once a minute + if s.DiagnosticInterval < time.Minute { s.Logger().Printf("diagnostics disabled") return + } else { + s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics") } s.diagnostics.SetLogger(s.LogOutput) s.diagnostics.SetVersion(Version) - s.diagnostics.SetInterval(s.DiagnosticInterval) - s.diagnostics.Open() s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) @@ -632,7 +633,10 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.EnrichWithMemoryInfo() s.diagnostics.EnrichWithSchemaProperties() s.diagnostics.CheckVersion() - s.diagnostics.Flush() + err = s.diagnostics.Flush() + if err != nil { + s.Logger().Printf("Diagnostics error: %s", err) + } } ticker := time.NewTicker(s.DiagnosticInterval) From 00bbdc1107c52f33f62eccfcf5e3fd4364d0a8c5 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 13 Mar 2018 14:50:09 -0500 Subject: [PATCH 147/311] Move SetFieldValue to write section --- docs/query-language.md | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 105893e66..f14be7ca6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -187,6 +187,30 @@ ClearBit(frame="stargazer", columnID=10, rowID=1) Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame. +#### SetFieldValue + +**Spec:** + +``` +SetFieldValue(, , ) +``` + +**Description:** + +`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`. + +**Result Type:** null + +SetFieldValue returns `null` upon success. + +**Examples:** + +Set the number of pull requests of repository 10. +``` +SetFieldValue(col=10, frame="stats", pullrequests=2) +``` + + ### Read Operations #### Bitmap @@ -527,27 +551,3 @@ Sum(frame="stats", field="diskusage") Return `{"sum":10,"count":3}` * Result is the size of all repositories in kilobytes, plus the number of repositories. - - -#### SetFieldValue - -**Spec:** - -``` -SetFieldValue(, , ) -``` - -**Description:** - -`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`. - -**Result Type:** null - -SetFieldValue returns `null` upon success. - -**Examples:** - -Set the number of pull requests of repository 10. -``` -SetFieldValue(col=10, frame="stats", pullrequests=2) -``` From 88471e94f197e5c6064042bc03eb8f91b9bd2ed7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 14 Mar 2018 15:34:33 -0500 Subject: [PATCH 148/311] Remove caching (the lib code is fast) and add tests --- gopsutil/systeminfo.go | 55 ++++++++++++------------------- gopsutil/systeminfo_test.go | 64 +++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 35 deletions(-) create mode 100644 gopsutil/systeminfo_test.go diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index 12c300ba5..433aacc30 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -10,8 +10,6 @@ var _ pilosa.SystemInfo = NewSystemInfo() // SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS type SystemInfo struct { - hostInfo *host.InfoStat - memInfo *mem.VirtualMemoryStat platform string family string osVersion string @@ -19,13 +17,23 @@ type SystemInfo struct { // Uptime returns the system uptime in seconds func (s *SystemInfo) Uptime() (uptime uint64, err error) { - if s.hostInfo == nil { - s.hostInfo, err = host.Info() + hostInfo, err := host.Info() + if err != nil { + return 0, err + } + return hostInfo.Uptime, nil +} + +// collectPlatformInfo fetches and caches system platform information +func (s *SystemInfo) collectPlatformInfo() error { + var err error + if s.platform == "" { + s.platform, s.family, s.osVersion, err = host.PlatformInformation() if err != nil { - return 0, err + return err } } - return s.hostInfo.Uptime, nil + return nil } // Uptime returns the system platform @@ -55,54 +63,31 @@ func (s *SystemInfo) OSVersion() (string, error) { return s.osVersion, err } -// collectPlatformInfo fetches and caches system platform information -func (s *SystemInfo) collectPlatformInfo() error { - var err error - if s.platform == "" { - s.platform, s.family, s.osVersion, err = host.PlatformInformation() - if err != nil { - return err - } - } - return nil -} - -// collectMemoryInfo fetches and caches memory stats -func (s *SystemInfo) collectMemoryInfo() (err error) { - if s.memInfo == nil { - s.memInfo, err = mem.VirtualMemory() - if err != nil { - return err - } - } - return nil -} - // MemFree returns the amount of free memory in bytes func (s *SystemInfo) MemFree() (uint64, error) { - err := s.collectMemoryInfo() + memInfo, err := mem.VirtualMemory() if err != nil { return 0, err } - return s.memInfo.Free, err + return memInfo.Free, err } // MemFree returns the amount of total memory in bytes func (s *SystemInfo) MemTotal() (uint64, error) { - err := s.collectMemoryInfo() + memInfo, err := mem.VirtualMemory() if err != nil { return 0, err } - return s.memInfo.Total, err + return memInfo.Total, err } // MemFree returns the amount of used memory in bytes func (s *SystemInfo) MemUsed() (uint64, error) { - err := s.collectMemoryInfo() + memInfo, err := mem.VirtualMemory() if err != nil { return 0, err } - return s.memInfo.Used, err + return memInfo.Used, err } // KernelVersion returns the kernel version as a string diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go new file mode 100644 index 000000000..41b58ec1c --- /dev/null +++ b/gopsutil/systeminfo_test.go @@ -0,0 +1,64 @@ +package gopsutil_test + +import ( + "log" + "runtime" + "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gopsutil" +) + +func TestSystemInfo(t *testing.T) { + var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo() + + // Uptime()(uint64, error) + // Platform()(string, error) + // Family()(string, error) + // OSVersion()(string, error) + // KernelVersion()(string, error) + // MemFree()(uint64, error) + // MemTotal()(uint64, error) + // MemUsed()(uint64, error) + // + uptime, err := systemInfo.Uptime() + if err != nil || uptime == 0 { + t.Fatalf("Error collecting uptime (error: %v)", err) + } + + platform, err := systemInfo.Platform() + if err != nil || platform != runtime.GOOS { + t.Fatalf("Platform must be %s. (error: %v)", runtime.GOOS, err) + } + + family, err := systemInfo.Family() + if err != nil { + t.Fatalf("Error getting OS family. (family: %v, error: %v)", family, err) + } + + osversion, err := systemInfo.OSVersion() + if err != nil { + t.Fatalf("Error getting OS version. (osversion: %v, error: %v)", osversion, err) + } + + kernelversion, err := systemInfo.KernelVersion() + if err != nil { + t.Fatalf("Error getting kernel version. (kernelversion: %v, error: %v)", kernelversion, err) + } + + memfree, err := systemInfo.MemFree() + if err != nil { + t.Fatalf("Error getting memfree. (memfree: %v, error: %v)", memfree, err) + } + + memused, err := systemInfo.MemUsed() + if err != nil { + t.Fatalf("Error getting memused. (memused: %v, error: %v)", memused, err) + } + + memtotal, err := systemInfo.MemTotal() + log.Println(memtotal) + if err != nil { + t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err) + } +} From 20dc1212f8d62f8e7db7eca49abc046ce85811f6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 14 Mar 2018 16:51:40 -0500 Subject: [PATCH 149/311] Address code review (mostly comments) --- diagnostics.go | 37 +++++++++---------- ...cs_test.go => diagnostics_internal_test.go | 2 +- gopsutil/systeminfo.go | 22 +++++------ 3 files changed, 30 insertions(+), 31 deletions(-) rename diagnostics_test.go => diagnostics_internal_test.go (99%) diff --git a/diagnostics.go b/diagnostics.go index 1abcdd144..710f66c2d 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -38,7 +38,7 @@ type versionResponse struct { Message string `json:"message"` } -// DiagnosticsCollector represents a collector/sender of diagnostics data +// DiagnosticsCollector represents a collector/sender of diagnostics data. type DiagnosticsCollector struct { mu sync.Mutex host string @@ -57,9 +57,8 @@ type DiagnosticsCollector struct { server *Server } -// New returns a pointer to a new DiagnosticsCollector Client given an addr in the format "hostname:port". +// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". func NewDiagnosticsCollector(host string) *DiagnosticsCollector { - return &DiagnosticsCollector{ host: host, VersionURL: defaultVersionCheckURL, @@ -118,7 +117,7 @@ func (d *DiagnosticsCollector) CheckVersion() error { return fmt.Errorf("json decode: %s", err) } - // Same a version as last test + // If version has not changed since the last check, return if rsp.Version == d.lastVersion { return nil } @@ -133,8 +132,8 @@ func (d *DiagnosticsCollector) CheckVersion() error { // compareVersion check version strings. func (d *DiagnosticsCollector) compareVersion(value string) error { - currentVersion := VersionSegments(value) - localVersion := VersionSegments(d.version) + currentVersion := versionSegments(value) + localVersion := versionSegments(d.version) if localVersion[0] < currentVersion[0] { //Major return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) @@ -249,8 +248,8 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { d.Set("TimeQuantumEnabled", timeQuantumEnabled) } -// VersionSegments returns the numeric segments of the version as a slice of ints. -func VersionSegments(segments string) []int { +// versionSegments returns the numeric segments of the version as a slice of ints. +func versionSegments(segments string) []int { segments = strings.Trim(segments, "v") segments = strings.Split(segments, "-")[0] s := strings.Split(segments, ".") @@ -261,7 +260,7 @@ func VersionSegments(segments string) []int { return segmentSlice } -// SystemInfo collects information about the host OS +// SystemInfo collects information about the host OS. type SystemInfo interface { Uptime() (uint64, error) Platform() (string, error) @@ -273,51 +272,51 @@ type SystemInfo interface { MemUsed() (uint64, error) } -// NewNopSystemInfo creates a no-op implementation of SystemInfo +// NewNopSystemInfo creates a no-op implementation of SystemInfo. func NewNopSystemInfo() *NopSystemInfo { return &NopSystemInfo{} } -// NopSystemInfo is a no-op implementation of SystemInfo +// NopSystemInfo is a no-op implementation of SystemInfo. type NopSystemInfo struct { } -// Uptime is a no-op implementation of SystemInfo.Uptime +// Uptime is a no-op implementation of SystemInfo.Uptime. func (n *NopSystemInfo) Uptime() (uint64, error) { return 0, nil } -// Platform is a no-op implementation of SystemInfo.Platform +// Platform is a no-op implementation of SystemInfo.Platform. func (n *NopSystemInfo) Platform() (string, error) { return "", nil } -// Family is a no-op implementation of SystemInfo.Family +// Family is a no-op implementation of SystemInfo.Family. func (n *NopSystemInfo) Family() (string, error) { return "", nil } -// OSVersion is a no-op implementation of SystemInfo.OSVersion +// OSVersion is a no-op implementation of SystemInfo.OSVersion. func (n *NopSystemInfo) OSVersion() (string, error) { return "", nil } -// KernelVersion is a no-op implementation of SystemInfo.KernelVersion +// KernelVersion is a no-op implementation of SystemInfo.KernelVersion. func (n *NopSystemInfo) KernelVersion() (string, error) { return "", nil } -// MemFree is a no-op implementation of SystemInfo.MemFree +// MemFree is a no-op implementation of SystemInfo.MemFree. func (n *NopSystemInfo) MemFree() (uint64, error) { return 0, nil } -// MemTotal is a no-op implementation of SystemInfo.MemTotal +// MemTotal is a no-op implementation of SystemInfo.MemTotal. func (n *NopSystemInfo) MemTotal() (uint64, error) { return 0, nil } -// MemUsed is a no-op implementation of SystemInfo.MemUsed +// MemUsed is a no-op implementation of SystemInfo.MemUsed. func (n *NopSystemInfo) MemUsed() (uint64, error) { return 0, nil } diff --git a/diagnostics_test.go b/diagnostics_internal_test.go similarity index 99% rename from diagnostics_test.go rename to diagnostics_internal_test.go index 7b66d8e18..eb2498297 100644 --- a/diagnostics_test.go +++ b/diagnostics_internal_test.go @@ -69,7 +69,7 @@ func TestDiagnosticsClient(t *testing.T) { func TestDiagnosticsVersion_Parse(t *testing.T) { version := "0.1.1" - vs := VersionSegments(version) + vs := versionSegments(version) output := []int{0, 1, 1} if !reflect.DeepEqual(vs, output) { diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index 433aacc30..e6285ade8 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -8,14 +8,14 @@ import ( var _ pilosa.SystemInfo = NewSystemInfo() -// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS +// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS. type SystemInfo struct { platform string family string osVersion string } -// Uptime returns the system uptime in seconds +// Uptime returns the system uptime in seconds. func (s *SystemInfo) Uptime() (uptime uint64, err error) { hostInfo, err := host.Info() if err != nil { @@ -24,7 +24,7 @@ func (s *SystemInfo) Uptime() (uptime uint64, err error) { return hostInfo.Uptime, nil } -// collectPlatformInfo fetches and caches system platform information +// collectPlatformInfo fetches and caches system platform information. func (s *SystemInfo) collectPlatformInfo() error { var err error if s.platform == "" { @@ -36,7 +36,7 @@ func (s *SystemInfo) collectPlatformInfo() error { return nil } -// Uptime returns the system platform +// Platform returns the system platform. func (s *SystemInfo) Platform() (string, error) { err := s.collectPlatformInfo() if err != nil { @@ -45,7 +45,7 @@ func (s *SystemInfo) Platform() (string, error) { return s.platform, nil } -// Family returns the system family +// Family returns the system family. func (s *SystemInfo) Family() (string, error) { err := s.collectPlatformInfo() if err != nil { @@ -54,7 +54,7 @@ func (s *SystemInfo) Family() (string, error) { return s.family, err } -// OSVersion returns the OS Version +// OSVersion returns the OS Version. func (s *SystemInfo) OSVersion() (string, error) { err := s.collectPlatformInfo() if err != nil { @@ -63,7 +63,7 @@ func (s *SystemInfo) OSVersion() (string, error) { return s.osVersion, err } -// MemFree returns the amount of free memory in bytes +// MemFree returns the amount of free memory in bytes. func (s *SystemInfo) MemFree() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { @@ -72,7 +72,7 @@ func (s *SystemInfo) MemFree() (uint64, error) { return memInfo.Free, err } -// MemFree returns the amount of total memory in bytes +// MemTotal returns the amount of total memory in bytes. func (s *SystemInfo) MemTotal() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { @@ -81,7 +81,7 @@ func (s *SystemInfo) MemTotal() (uint64, error) { return memInfo.Total, err } -// MemFree returns the amount of used memory in bytes +// MemUsed returns the amount of used memory in bytes. func (s *SystemInfo) MemUsed() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { @@ -90,12 +90,12 @@ func (s *SystemInfo) MemUsed() (uint64, error) { return memInfo.Used, err } -// KernelVersion returns the kernel version as a string +// KernelVersion returns the kernel version as a string. func (s *SystemInfo) KernelVersion() (string, error) { return host.KernelVersion() } -// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo +// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. func NewSystemInfo() *SystemInfo { return &SystemInfo{} } From 2d425e32e91aa9385631bbba7a5f6b6f8ca3286e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 14 Mar 2018 17:33:54 -0500 Subject: [PATCH 150/311] Log platform --- gopsutil/systeminfo_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go index 41b58ec1c..4131942a2 100644 --- a/gopsutil/systeminfo_test.go +++ b/gopsutil/systeminfo_test.go @@ -28,7 +28,7 @@ func TestSystemInfo(t *testing.T) { platform, err := systemInfo.Platform() if err != nil || platform != runtime.GOOS { - t.Fatalf("Platform must be %s. (error: %v)", runtime.GOOS, err) + t.Fatalf("Platform must be %s. (platform: %v, error: %v)", platform, runtime.GOOS, err) } family, err := systemInfo.Family() From 680acf4e3dc175a6199f4cefdcb4afdd06440c93 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 14 Mar 2018 17:41:44 -0500 Subject: [PATCH 151/311] Fix error on linux --- gopsutil/systeminfo_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go index 4131942a2..5d96a499c 100644 --- a/gopsutil/systeminfo_test.go +++ b/gopsutil/systeminfo_test.go @@ -2,7 +2,6 @@ package gopsutil_test import ( "log" - "runtime" "testing" "github.com/pilosa/pilosa" @@ -27,8 +26,8 @@ func TestSystemInfo(t *testing.T) { } platform, err := systemInfo.Platform() - if err != nil || platform != runtime.GOOS { - t.Fatalf("Platform must be %s. (platform: %v, error: %v)", platform, runtime.GOOS, err) + if err != nil { + t.Fatalf("Error getting platform. (platform: %v, error: %v)", platform, err) } family, err := systemInfo.Family() From fae429f971bc79e212c5a7cf6e9f0cdbbd1170a6 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 15 Mar 2018 10:57:19 -0500 Subject: [PATCH 152/311] Remove outdated column labels --- docs/data-model.md | 12 ++++++------ docs/query-language.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 9ab684203..6a57a96a1 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -119,12 +119,12 @@ Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'row For example, the following `SetFieldValue()` queries will result in the data described in the illustration below: ``` -SetFieldValue(col=1, frame="A", field0=1) -SetFieldValue(col=2, frame="A", field0=2) -SetFieldValue(col=3, frame="A", field0=3) -SetFieldValue(col=4, frame="A", field0=7) -SetFieldValue(col=2, frame="A", field1=1) -SetFieldValue(col=3, frame="A", field1=6) +SetFieldValue(columnID=1, frame="A", field0=1) +SetFieldValue(columnID=2, frame="A", field0=2) +SetFieldValue(columnID=3, frame="A", field0=3) +SetFieldValue(columnID=4, frame="A", field0=7) +SetFieldValue(columnID=2, frame="A", field1=1) +SetFieldValue(columnID=3, frame="A", field1=6) ``` ![BSI frame diagram](/img/docs/frame-bsi.svg) diff --git a/docs/query-language.md b/docs/query-language.md index f14be7ca6..d612328c6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -207,7 +207,7 @@ SetFieldValue returns `null` upon success. Set the number of pull requests of repository 10. ``` -SetFieldValue(col=10, frame="stats", pullrequests=2) +SetFieldValue(columnID=10, frame="stats", pullrequests=2) ``` From 1452bf24d640b2b95f1596e359506ea888b381c7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 15 Mar 2018 11:07:07 -0500 Subject: [PATCH 153/311] Clarify Sum description --- docs/query-language.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/query-language.md b/docs/query-language.md index d612328c6..83cb946e3 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -537,7 +537,7 @@ Sum([BITMAP_CALL], , ) **Description:** -Returns the count and computed sum of all BSI integer values across the `field` in this `frame`. The optional `Bitmap` call filters the bits used in this computation. +Returns the count and computed sum of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. **Result Type:** object with the computed sum and count of the bitmap field. From 754de2e057e53026d6f09faf94cd3d8da733faf7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Mar 2018 15:26:36 -0500 Subject: [PATCH 154/311] Add correct diagnostics interval to startup message. --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index dd8811463..54bacda1d 100644 --- a/server.go +++ b/server.go @@ -610,7 +610,7 @@ func (s *Server) monitorDiagnostics() { s.Logger().Printf("diagnostics disabled") return } else { - s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics") + s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) } s.diagnostics.SetLogger(s.LogOutput) From 85b33f1bfb11835b830bcef8fd291cbc9185c614 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Mar 2018 15:27:01 -0500 Subject: [PATCH 155/311] Remove unused code and TODO and clarify with comment. --- diagnostics.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 710f66c2d..08d77a03c 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -91,13 +91,8 @@ func (d *DiagnosticsCollector) Flush() error { if err != nil { return err } + // Intentionally ignoring response body, as user does not need to be notified of error. defer resp.Body.Close() - - // TODO verify response - _, err = ioutil.ReadAll(resp.Body) - if err != nil { - return err - } return nil } From be70bbfed2f5b3032d166bac549e7dec4d713135 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Mar 2018 15:27:48 -0500 Subject: [PATCH 156/311] Fix bug: backend won't store empty strings. --- diagnostics.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/diagnostics.go b/diagnostics.go index 08d77a03c..3ad07ed42 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -148,6 +148,13 @@ func (d *DiagnosticsCollector) encode() ([]byte, error) { // Set adds a key value metric. func (d *DiagnosticsCollector) Set(name string, value interface{}) { + switch v := value.(type) { + case string: + if v == "" { + // Do not set empty string + return + } + } d.mu.Lock() defer d.mu.Unlock() d.metrics[name] = value From f31966553f259dd5602ceb9dadf0f15bd5778ec1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 6 Feb 2018 15:49:46 -0600 Subject: [PATCH 157/311] update docs to include cluster-resize config and instructions --- docs/administration.md | 66 +++++++++++++++++++++++++++++++++++ docs/configuration.md | 77 ++++++++++++++--------------------------- docs/getting-started.md | 8 +++-- 3 files changed, 97 insertions(+), 54 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 97ee688f5..cd1c2fa90 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -5,6 +5,7 @@ nav = [ "Installing in production", "Importing and Exporting Data", "Versioning", + "Resizing the Cluster", "Backup/restore", ] +++ @@ -93,6 +94,71 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes. +### Resizing the Cluster + +If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process. + +#### Adding a Node + +You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify a valid [gossip seed](../configuration/#gossip-seed). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries. + +If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately. + +#### Removing a Node + +In order to remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself. + +To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing an `/id` request to the node: +``` request +curl localhost:10102/id +``` +``` response +40a891fa-243b-4d71-ae24-4f5c78a0f4b1 +``` + +If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node: +``` request +curl localhost:10101/status +``` +``` response +{ + "state":"NORMAL", + "nodes":[ + {"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}} + {"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}} + {"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}} + ] +} +``` + +Once you have the ID of the node that you want to remove from the cluster, issue the following request: +``` +curl localhost:10101/cluster/resize/remove-node \ + -X POST \ + -d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}' +``` +At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries. + +Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator). + +#### Aborting a Resize Job + +If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node. +For example, if your coordinator node is `localhost:10101`, then you can run: +``` +curl localhost:10101/cluster/resize/abort -X POST +``` +This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began. + +#### Changing the Coordinator + +In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator. +``` +curl localhost:10101/cluster/resize/set-coordinator \ + -X POST \ + -d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}' +``` + ### Backup/restore Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. diff --git a/docs/configuration.md b/docs/configuration.md index dfaa01d64..8125bae18 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,23 +27,18 @@ Every command line flag has a corresponding environment variable. The environmen ### Config file -The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flag `--cluster.replicas=1` looks like this in the config file: +The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator="localhost:10101"` and `--cluster.replicas=1` look like this in the config file: ```toml [cluster] + coordinator = "localhost:10101" replicas = 1 ``` -Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes: -```toml -[cluster] - hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"] -``` - ### All Options #### Anti Entropy Interval -* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync. +* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync. * Flag: `--anti-entropy.interval="10m0s"` * Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"` * Config: @@ -83,7 +78,7 @@ Any flag that has a value that is a comma separated list on the command line bec * Config: ```toml - log_path = "/path/to/logfile" + log-path = "/path/to/logfile" ``` #### Max Writes Per Request @@ -132,28 +127,16 @@ Any flag that has a value that is a comma separated list on the command line bec key = "/var/secret/gossip.key32" ``` -#### Cluster Hosts +#### Cluster Coordinator -* Description: List of hosts in the cluster. Multiple hosts should be comma-separated in the flag and env forms. -* Flag: `--cluster.hosts="localhost:10101"` -* Env: `PILOSA_CLUSTER_HOSTS="localhost:10101"` +* Description: Address of the node responsible for coordinating cluster membership and the cluster resize process. This value should be the same on all nodes in the cluster. +* Flag: `cluster.coordinator="localhost:10101"` +* Env: `PILOSA_CLUSTER_COORDINATOR="localhost:10101"` * Config: ```toml [cluster] - hosts = ["localhost:10101"] - ``` - -#### Cluster Poll Interval - -* Description: Polling interval for cluster. -* Flag: `cluster.poll-interval="1m0s"` -* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"` -* Config: - - ```toml - [cluster] - poll-interval = "1m0s" + coordinator = "localhost:10101" ``` #### Cluster Long Query Time @@ -217,7 +200,8 @@ Any flag that has a value that is a comma separated list on the command line bec [profile] cpu-time = "30s" ``` -##### Metric Service + +#### Metric Service * Description: Which stats service to use. Choose from [statsd, expvar]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' @@ -228,7 +212,7 @@ Any flag that has a value that is a comma separated list on the command line bec service = “statsd” ``` -##### Metric Host +#### Metric Host * Description: Address of the StatsD service host. * Flag: `--metric.host=localhost:8125` * Env: `PILOSA_METRIC_HOST=localhost:8125' @@ -239,7 +223,7 @@ Any flag that has a value that is a comma separated list on the command line bec host = "localhost:8125" ``` -##### Metric Poll Interval +#### Metric Poll Interval * Description: Polling interval for runtime metrics. * Flag: `metric.poll-interval=”0m15s”` @@ -251,7 +235,7 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "0m15s" ``` -##### Metric Diagnostics +#### Metric Diagnostics * Description: Enable diagnostic reporting. To disable diagnostics set to false. * Flag: `metric.diagnostics` @@ -264,7 +248,7 @@ Any flag that has a value that is a comma separated list on the command line bec ``` -##### TLS Certificate +#### TLS Certificate * Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions. * Flag: `tls.certificate=/srv/pilosa/certs/server.crt` @@ -276,7 +260,7 @@ Any flag that has a value that is a comma separated list on the command line bec certificate = "/srv/pilosa/certs/server.crt" ``` -##### TLS Certificate Key +#### TLS Certificate Key * Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension. * Flag: `tls.key=/srv/pilosa/certs/server.key` @@ -288,7 +272,7 @@ Any flag that has a value that is a comma separated list on the command line bec key = "/srv/pilosa/certs/server.key" ``` -##### TLS Skip Verify +#### TLS Skip Verify * Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial. * Flag: `tls.skip-verify` @@ -315,8 +299,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - type = "gossip" - hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + coordinator = "node0.pilosa.com:10101" #### Node 1 @@ -329,8 +312,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - type = "gossip" - hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + coordinator = "node0.pilosa.com:10101" #### Node 2 @@ -343,8 +325,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - type = "gossip" - hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + coordinator = "node0.pilosa.com:10101" ### Example Cluster Configuration (HTTPS) @@ -363,8 +344,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - type = "gossip" - hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"] + coordinator = "https://node0.pilosa.com:10101" [tls] certificate = "/home/pilosa/private/server.crt" @@ -382,8 +362,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - type = "gossip" - hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"] + coordinator = "https://node0.pilosa.com:10101" [tls] certificate = "/home/pilosa/private/server.crt" @@ -401,8 +380,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - type = "gossip" - hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"] + coordinator = "https://node0.pilosa.com:10101" [tls] certificate = "/home/pilosa/private/server.crt" @@ -424,8 +402,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - type = "gossip" - hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"] + coordinator = "https://node0.pilosa.com:10100" [tls] certificate = "/home/pilosa/private/server.crt" @@ -443,8 +420,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - type = "gossip" - hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"] + coordinator = "https://node0.pilosa.com:10100" [tls] certificate = "/home/pilosa/private/server.crt" @@ -462,8 +438,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - type = "gossip" - hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"] + coordinator = "https://node0.pilosa.com:10100" [tls] certificate = "/home/pilosa/private/server.crt" diff --git a/docs/getting-started.md b/docs/getting-started.md index f68805f53..e768183b3 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -35,7 +35,7 @@ Let's make sure Pilosa is running: curl localhost:10101/status ``` ``` response -{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}} +{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]} ``` ### Sample Project @@ -46,7 +46,9 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term #### Create the Schema -The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: +Note: +The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: + ``` request curl localhost:10101/schema ``` @@ -107,7 +109,7 @@ docker cp language.csv pilosa:/language.csv docker exec -it pilosa /pilosa import -i repository -f language /language.csv ``` -Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages. +Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages. ### Input Definition Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. From 3690264108bfe735d241319557235e7fc655bd7f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 15 Mar 2018 15:37:48 -0500 Subject: [PATCH 158/311] Change wording to reflect that you can specify more than one gossip seed --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index cd1c2fa90..91cbc6a95 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -100,7 +100,7 @@ If you need to increase (or decrease) the capacity of a Pilosa server, you can a #### Adding a Node -You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify a valid [gossip seed](../configuration/#gossip-seed). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries. +You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries. If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately. From b424811da900493d9daa895a07f262db885765a9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 16 Mar 2018 15:28:03 -0500 Subject: [PATCH 159/311] Add license text --- gopsutil/systeminfo.go | 14 ++++++++++++++ gopsutil/systeminfo_test.go | 14 ++++++++++++++ security_manager.go | 14 ++++++++++++++ statik/doc.go | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index e6285ade8..3310aeae1 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gopsutil import ( diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go index 5d96a499c..0f76b62da 100644 --- a/gopsutil/systeminfo_test.go +++ b/gopsutil/systeminfo_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gopsutil_test import ( diff --git a/security_manager.go b/security_manager.go index fc174866a..80b696c38 100644 --- a/security_manager.go +++ b/security_manager.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa // SecurityManager provides the ability to limit access to restricted endpoints diff --git a/statik/doc.go b/statik/doc.go index 85edd9e6f..e9ca40ffd 100644 --- a/statik/doc.go +++ b/statik/doc.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package statik contains static assets for the Web UI. `go generate` will // produce statik.go, which is ignored by git. package statik From b36e8c7a1486c7944e66d1cd5ddd0f909cd7b47c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 19 Mar 2018 08:39:17 -0500 Subject: [PATCH 160/311] Combine two subpackages and consolidate doc.go into filestystem.go for simplicity. --- handler.go | 7 ++----- handler_test.go | 2 +- server/server.go | 2 +- statik/doc.go | 4 ---- statik/{filesystem/statik.go => filesystem.go} | 7 ++++++- 5 files changed, 10 insertions(+), 12 deletions(-) delete mode 100644 statik/doc.go rename statik/{filesystem/statik.go => filesystem.go} (81%) diff --git a/handler.go b/handler.go index e4a707d8f..6e93ae380 100644 --- a/handler.go +++ b/handler.go @@ -42,9 +42,6 @@ import ( "github.com/pilosa/pilosa/pql" "unicode" - - // Allow building Pilosa without the web UI. - _ "github.com/pilosa/pilosa/statik" ) // Handler represents an HTTP handler. @@ -289,13 +286,13 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) return } - statikFS, err := h.FileSystem.New() + filesystem, err := h.FileSystem.New() if err != nil { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") return } - http.FileServer(statikFS).ServeHTTP(w, r) + http.FileServer(filesystem).ServeHTTP(w, r) } // handleGetSchema handles GET /schema requests. diff --git a/handler_test.go b/handler_test.go index 5d8af43a9..aa8ad2970 100644 --- a/handler_test.go +++ b/handler_test.go @@ -31,7 +31,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - statik "github.com/pilosa/pilosa/statik/filesystem" + "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/test" ) diff --git a/server/server.go b/server/server.go index 67498d7d4..b9140f6a3 100644 --- a/server/server.go +++ b/server/server.go @@ -35,7 +35,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gossip" - statik "github.com/pilosa/pilosa/statik/filesystem" + "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/statsd" ) diff --git a/statik/doc.go b/statik/doc.go deleted file mode 100644 index 9310eb508..000000000 --- a/statik/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package statik contains static assets for the Web UI. `go generate` will -// produce statik.go, which is ignored by git. -//go:generate statik -src=../webui -dest=.. -package statik diff --git a/statik/filesystem/statik.go b/statik/filesystem.go similarity index 81% rename from statik/filesystem/statik.go rename to statik/filesystem.go index 3f9582a71..6d7fd864f 100644 --- a/statik/filesystem/statik.go +++ b/statik/filesystem.go @@ -11,8 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +//go:generate statik -src=../webui -dest=.. +// +// Package statik contains static assets for the Web UI. `go generate` or +// `make generate-statik` will produce statik.go, which is ignored by git. -package filesystem +package statik import ( "net/http" From b66bedd1ef009de63f38314ecd1a6410d9d1c9ac Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 19 Mar 2018 11:30:41 -0500 Subject: [PATCH 161/311] put BoltDB behind AttrStore interface --- attr.go | 522 +++++++++----------------------------------- boltdb/attrstore.go | 478 ++++++++++++++++++++++++++++++++++++++++ fragment.go | 2 +- frame.go | 9 +- holder.go | 7 + index.go | 14 +- server.go | 4 + server/server.go | 4 + test/attr.go | 7 +- test/holder.go | 3 + test/pilosa.go | 5 + view.go | 2 +- 12 files changed, 623 insertions(+), 434 deletions(-) create mode 100644 boltdb/attrstore.go diff --git a/attr.go b/attr.go index 437cab433..7e025ea5b 100644 --- a/attr.go +++ b/attr.go @@ -16,23 +16,12 @@ package pilosa import ( "bytes" - - "encoding/binary" - "fmt" "sort" - "sync" - "time" - "github.com/cespare/xxhash" - - "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) -// AttrBlockSize is the size of attribute blocks for anti-entropy. -const AttrBlockSize = 100 - // Attribute data type enum. const ( AttrTypeString = 1 @@ -41,313 +30,128 @@ const ( AttrTypeFloat = 4 ) -// AttrCache represents a cache for attributes. -type AttrCache struct { - mu sync.RWMutex - attrs map[uint64]map[string]interface{} +// AttrStoreGenerator represents an interface for generating an AttrStore. +type AttrStoreGenerator interface { + New(string) AttrStore } -// Get returns the cached attributes for a given id. -func (c *AttrCache) Get(id uint64) map[string]interface{} { - c.mu.RLock() - defer c.mu.RUnlock() - attrs := c.attrs[id] - if attrs == nil { - return nil - } - - // Make a copy for safety - ret := make(map[string]interface{}) - for k, v := range attrs { - ret[k] = v - } - return ret +// AttrStore represents an interface for handling row/column attributes. +type AttrStore interface { + Path() string + Open() error + Close() error + Attrs(id uint64) (m map[string]interface{}, err error) + SetAttrs(id uint64, m map[string]interface{}) error + SetBulkAttrs(m map[uint64]map[string]interface{}) error + Blocks() ([]AttrBlock, error) + BlockData(i uint64) (map[uint64]map[string]interface{}, error) } -// Set updates the cached attributes for a given id. -func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { - c.mu.Lock() - defer c.mu.Unlock() - c.attrs[id] = attrs +func init() { + NopAttrStoreGenerator = &nopAttrStoreGenerator{ + Name: "NooP", + } + NopAttrStore = &nopAttrStore{} } -// AttrStore represents a storage layer for attributes. -type AttrStore struct { - mu sync.RWMutex - path string - db *bolt.DB - attrCache *AttrCache +// NopAttrStoreGenerator represents an AttrStoreGenerator that return a no-op AttrStore. +var NopAttrStoreGenerator AttrStoreGenerator + +// NopAttrStore represents an AttrStore that doesn't do anything. +var NopAttrStore AttrStore + +// nopAttrStoreGenerator represents a no-op implementation of the AttrStoreGenerator interface. +type nopAttrStoreGenerator struct { + Name string } -// NewAttrCache returns a new instance of AttrCache. -func NewAttrCache() *AttrCache { - return &AttrCache{ - attrs: make(map[uint64]map[string]interface{}), - } +// New is a no-op implementation of AttrStoreGenerator New method. +func (g *nopAttrStoreGenerator) New(string) AttrStore { + return &nopAttrStore{} } -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(path string) *AttrStore { - return &AttrStore{ - path: path, - attrCache: NewAttrCache(), - } -} +// nopAttrStore represents a no-op implementation of the AttrStore interface. +type nopAttrStore struct{} -// Path returns path to the store's data file. -func (s *AttrStore) Path() string { return s.path } - -// Open opens and initializes the store. -func (s *AttrStore) Open() error { - // Open storage. - db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return err - } - s.db = db - - // Initialize database. - if err := s.db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil { - return err - } - return nil - }); err != nil { - return err - } +// Path is a no-op implementation of AttrStore Path method. +func (s *nopAttrStore) Path() string { return "" } +// Open is a no-op implementation of AttrStore Open method. +func (s *nopAttrStore) Open() error { return nil } -// Close closes the store. -func (s *AttrStore) Close() error { - if s.db != nil { - s.db.Close() - } +// Close is a no-op implementation of AttrStore Close method. +func (s *nopAttrStore) Close() error { return nil } -// Attrs returns a set of attributes by ID. -func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { - s.mu.RLock() - defer s.mu.RUnlock() - - // Check cache for map. - if m = s.attrCache.Get(id); m != nil { - return m, nil - } - - // Find attributes from storage. - if err = s.db.View(func(tx *bolt.Tx) error { - m, err = txAttrs(tx, id) - if err != nil { - return err - } - return nil - }); err != nil { - return nil, err - } - - // Add to cache. - s.attrCache.Set(id, m) - - return +// Attrs is a no-op implementation of AttrStore Attrs method. +func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return nil, nil } -// SetAttrs sets attribute values for a given ID. -func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { - // Ignore empty maps. - if len(m) == 0 { - return nil - } - - // Check if the attributes already exist under a read-only lock. - if attr, err := s.Attrs(id); err != nil { - return err - } else if attr != nil && mapContains(attr, m) { - return nil - } - - // Obtain write lock. - s.mu.Lock() - defer s.mu.Unlock() - - var attr map[string]interface{} - if err := s.db.Update(func(tx *bolt.Tx) error { - tmp, err := txUpdateAttrs(tx, id, m) - if err != nil { - return err - } - attr = tmp - - return nil - }); err != nil { - return err - } - - // Swap attributes map in cache. - s.attrCache.Set(id, attr) - +// SetAttrs is a no-op implementation of AttrStore SetAttrs method. +func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil } -// SetBulkAttrs sets attribute values for a set of ids. -func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { - s.mu.Lock() - defer s.mu.Unlock() +// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method. +func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + return nil +} - attrs := make(map[uint64]map[string]interface{}) - if err := s.db.Update(func(tx *bolt.Tx) error { - // Collect and sort keys. - ids := make([]uint64, 0, len(m)) - for id := range m { - ids = append(ids, id) +// Blocks is a no-op implementation of AttrStore Blocks method. +func (s *nopAttrStore) Blocks() ([]AttrBlock, error) { + return nil, nil +} + +// BlockData is a no-op implementation of AttrStore BlockData method. +func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} + +// AttrBlock represents a checksummed block of the attribute store. +type AttrBlock struct { + ID uint64 `json:"id"` + Checksum []byte `json:"checksum"` +} + +// AttrBlocks represents a list of blocks. +type AttrBlocks []AttrBlock + +// Diff returns a list of block ids that are different or are new in other. +// Block lists must be in sorted order. +func (a AttrBlocks) Diff(other []AttrBlock) []uint64 { + var ids []uint64 + for { + // Read next block from each list. + var blk0, blk1 *AttrBlock + if len(a) > 0 { + blk0 = &a[0] + } + if len(other) > 0 { + blk1 = &other[0] } - sort.Sort(uint64Slice(ids)) - // Update attributes for each id. - for _, id := range ids { - attr, err := txUpdateAttrs(tx, id, m[id]) - if err != nil { - return err + // Exit if "a" contains no more blocks. + if blk0 == nil { + return ids + } + + // Add block ID if it's different or if it's only in "a". + if blk1 == nil || blk0.ID < blk1.ID { + ids = append(ids, blk0.ID) + a = a[1:] + } else if blk1.ID < blk0.ID { + other = other[1:] + } else { + if !bytes.Equal(blk0.Checksum, blk1.Checksum) { + ids = append(ids, blk0.ID) } - attrs[id] = attr - } - - return nil - }); err != nil { - return err - } - - // Swap attributes map in cache. - for id, attr := range attrs { - s.attrCache.Set(id, attr) - } - - return nil -} - -// Blocks returns a list of all blocks in the store. -func (s *AttrStore) Blocks() ([]AttrBlock, error) { - tx, err := s.db.Begin(false) - if err != nil { - return nil, err - } - defer tx.Rollback() - - // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize) - - // Iterate over each block. - var blocks []AttrBlock - for cur.nextBlock() { - block := AttrBlock{ID: cur.blockID()} - - // Compute checksum of every key/value in block. - h := xxhash.New() - for k, v := cur.next(); k != nil; k, v = cur.next() { - h.Write(k) - h.Write(v) - } - block.Checksum = h.Sum(nil) - - // Append block. - blocks = append(blocks, block) - } - - return blocks, nil -} - -// BlockData returns all data for a single block. -func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { - m := make(map[uint64]map[string]interface{}) - - // Start read-only transaction. - tx, err := s.db.Begin(false) - if err != nil { - return nil, err - } - defer tx.Rollback() - - // Move to the start of the block. - min := u64tob(uint64(i) * AttrBlockSize) - max := u64tob(uint64(i+1) * AttrBlockSize) - cur := tx.Bucket([]byte("attrs")).Cursor() - for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { - // Exit if we're past the end of the block. - if bytes.Compare(k, max) != -1 { - break - } - - // Decode attribute map and associate with id. - var pb internal.AttrMap - if err := proto.Unmarshal(v, &pb); err != nil { - return nil, err - } - m[btou64(k)] = decodeAttrs(pb.GetAttrs()) - } - - return m, nil -} - -// txAttrs returns a map of attributes for an id. -func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { - v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) - if v == nil { - return emptyMap, nil - } - - var pb internal.AttrMap - if err := proto.Unmarshal(v, &pb); err != nil { - return nil, err - } - return decodeAttrs(pb.GetAttrs()), nil -} - -// txUpdateAttrs updates the attributes for an id. -// Returns the new combined set of attributes for the id. -func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) { - attr, err := txAttrs(tx, id) - if err != nil { - return nil, err - } - - // Create a new map if it is empty so we don't update emptyMap. - if len(attr) == 0 { - attr = make(map[string]interface{}, len(m)) - } - - // Merge attributes with original values. - // Nil values should delete keys. - for k, v := range m { - if v == nil { - delete(attr, k) - continue - } - - switch v := v.(type) { - case int: - attr[k] = int64(v) - case uint: - attr[k] = int64(v) - case uint64: - attr[k] = int64(v) - case string, int64, bool, float64: - attr[k] = v - default: - return nil, fmt.Errorf("invalid attr type: %T", v) + a, other = a[1:], other[1:] } } - - // Marshal and save new values. - buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)}) - if err != nil { - return nil, err - } - if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil { - return nil, err - } - return attr, nil } func encodeAttrs(m map[string]interface{}) []*internal.Attr { @@ -421,136 +225,16 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} { return other } -// u64tob encodes v to big endian encoding. -func u64tob(v uint64) []byte { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, v) - return b +// EncodeAttrs encodes an attribute map into a byte slice. +func EncodeAttrs(attr map[string]interface{}) ([]byte, error) { + return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)}) } -// btou64 decodes b from big endian encoding. -func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } - -// emptyMap is a reusable map that contains no keys. -var emptyMap = make(map[string]interface{}) - -// AttrBlock represents a checksummed block of the attribute store. -type AttrBlock struct { - ID uint64 `json:"id"` - Checksum []byte `json:"checksum"` -} - -// AttrBlocks represents a list of blocks. -type AttrBlocks []AttrBlock - -// Diff returns a list of block ids that are different or are new in other. -// Block lists must be in sorted order. -func (a AttrBlocks) Diff(other []AttrBlock) []uint64 { - var ids []uint64 - for { - // Read next block from each list. - var blk0, blk1 *AttrBlock - if len(a) > 0 { - blk0 = &a[0] - } - if len(other) > 0 { - blk1 = &other[0] - } - - // Exit if "a" contains no more blocks. - if blk0 == nil { - return ids - } - - // Add block ID if it's different or if it's only in "a". - if blk1 == nil || blk0.ID < blk1.ID { - ids = append(ids, blk0.ID) - a = a[1:] - } else if blk1.ID < blk0.ID { - other = other[1:] - } else { - if !bytes.Equal(blk0.Checksum, blk1.Checksum) { - ids = append(ids, blk0.ID) - } - a, other = a[1:], other[1:] - } - } -} - -// blockCursor represents a cursor for iterating over blocks of a bolt bucket. -type blockCursor struct { - cur *bolt.Cursor - base uint64 - n uint64 - - buf struct { - key []byte - value []byte - filled bool - } -} - -// newBlockCursor returns a new block cursor that wraps cur using n sized blocks. -func newBlockCursor(c *bolt.Cursor, n int) blockCursor { - cur := blockCursor{ - cur: c, - n: uint64(n), - } - cur.buf.key, cur.buf.value = c.First() - cur.buf.filled = true - return cur -} - -// blockID returns the current block ID. Only valid after call to nextBlock(). -func (cur *blockCursor) blockID() uint64 { return cur.base } - -// nextBlock moves the cursor to the next block. -// Returns true if another block exists, otherwise returns false. -func (cur *blockCursor) nextBlock() bool { - if cur.buf.key == nil { - return false - } - - cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n - return true -} - -// next returns the next key/value within the block. -// Returns nils at the end of the block. -func (cur *blockCursor) next() (key, value []byte) { - // Use buffered value, if set. - if cur.buf.filled { - key, value = cur.buf.key, cur.buf.value - cur.buf.filled = false - return key, value - } - - // Read next key. - key, value = cur.cur.Next() - - // Fill buffer for EOF. - if key == nil { - cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false - return nil, nil - } - - // Parse key and buffer if outside of block. - id := binary.BigEndian.Uint64(key) - if id/cur.n > cur.base { - cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true - return nil, nil - } - - return key, value -} - -// mapContains returns true if all keys & values of subset are in m. -func mapContains(m, subset map[string]interface{}) bool { - for k, v := range subset { - value, ok := m[k] - if !ok || value != v { - return false - } - } - return true +// DecodeAttrs decodes a byte slice into an attribute map. +func DecodeAttrs(v []byte) (map[string]interface{}, error) { + var pb internal.AttrMap + if err := proto.Unmarshal(v, &pb); err != nil { + return nil, err + } + return decodeAttrs(pb.GetAttrs()), nil } diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go new file mode 100644 index 000000000..0ea35e663 --- /dev/null +++ b/boltdb/attrstore.go @@ -0,0 +1,478 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "bytes" + + "encoding/binary" + "fmt" + "sort" + "sync" + "time" + + "github.com/cespare/xxhash" + + "github.com/boltdb/bolt" + "github.com/pilosa/pilosa" +) + +// AttrBlockSize is the size of attribute blocks for anti-entropy. +const AttrBlockSize = 100 + +// AttrCache represents a cache for attributes. +type AttrCache struct { + mu sync.RWMutex + attrs map[uint64]map[string]interface{} +} + +// Get returns the cached attributes for a given id. +func (c *AttrCache) Get(id uint64) map[string]interface{} { + c.mu.RLock() + defer c.mu.RUnlock() + attrs := c.attrs[id] + if attrs == nil { + return nil + } + + // Make a copy for safety + ret := make(map[string]interface{}) + for k, v := range attrs { + ret[k] = v + } + return ret +} + +// Set updates the cached attributes for a given id. +func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { + c.mu.Lock() + defer c.mu.Unlock() + c.attrs[id] = attrs +} + +// AttrStoreGenerator represents a bolt implementation of the AttrStoreGenerator interface. +type AttrStoreGenerator struct{} + +// New is a bolt implementation of AttrStoreGenerator New method. +func (g *AttrStoreGenerator) New(path string) pilosa.AttrStore { + return NewAttrStore(path) +} + +// AttrStore represents a storage layer for attributes. +type AttrStore struct { + mu sync.RWMutex + path string + db *bolt.DB + attrCache *AttrCache +} + +// NewAttrStoreGenerator returns a new instance of AttrStoreGenerator. +func NewAttrStoreGenerator() *AttrStoreGenerator { + return &AttrStoreGenerator{} +} + +// NewAttrCache returns a new instance of AttrCache. +func NewAttrCache() *AttrCache { + return &AttrCache{ + attrs: make(map[uint64]map[string]interface{}), + } +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore(path string) *AttrStore { + return &AttrStore{ + path: path, + attrCache: NewAttrCache(), + } +} + +// Path returns path to the store's data file. +func (s *AttrStore) Path() string { return s.path } + +// Open opens and initializes the store. +func (s *AttrStore) Open() error { + // Open storage. + db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return err + } + s.db = db + + // Initialize database. + if err := s.db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + return nil +} + +// Close closes the store. +func (s *AttrStore) Close() error { + if s.db != nil { + s.db.Close() + } + return nil +} + +// Attrs returns a set of attributes by ID. +func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + s.mu.RLock() + defer s.mu.RUnlock() + + // Check cache for map. + if m = s.attrCache.Get(id); m != nil { + return m, nil + } + + // Find attributes from storage. + if err = s.db.View(func(tx *bolt.Tx) error { + m, err = txAttrs(tx, id) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + + // Add to cache. + s.attrCache.Set(id, m) + + return +} + +// SetAttrs sets attribute values for a given ID. +func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + // Ignore empty maps. + if len(m) == 0 { + return nil + } + + // Check if the attributes already exist under a read-only lock. + if attr, err := s.Attrs(id); err != nil { + return err + } else if attr != nil && mapContains(attr, m) { + return nil + } + + // Obtain write lock. + s.mu.Lock() + defer s.mu.Unlock() + + var attr map[string]interface{} + if err := s.db.Update(func(tx *bolt.Tx) error { + tmp, err := txUpdateAttrs(tx, id, m) + if err != nil { + return err + } + attr = tmp + + return nil + }); err != nil { + return err + } + + // Swap attributes map in cache. + s.attrCache.Set(id, attr) + + return nil +} + +// SetBulkAttrs sets attribute values for a set of ids. +func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + s.mu.Lock() + defer s.mu.Unlock() + + attrs := make(map[uint64]map[string]interface{}) + if err := s.db.Update(func(tx *bolt.Tx) error { + // Collect and sort keys. + ids := make([]uint64, 0, len(m)) + for id := range m { + ids = append(ids, id) + } + sort.Sort(uint64Slice(ids)) + + // Update attributes for each id. + for _, id := range ids { + attr, err := txUpdateAttrs(tx, id, m[id]) + if err != nil { + return err + } + attrs[id] = attr + } + + return nil + }); err != nil { + return err + } + + // Swap attributes map in cache. + for id, attr := range attrs { + s.attrCache.Set(id, attr) + } + + return nil +} + +// Blocks returns a list of all blocks in the store. +func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { + tx, err := s.db.Begin(false) + if err != nil { + return nil, err + } + defer tx.Rollback() + + // Wrap cursor to segment by block. + cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize) + + // Iterate over each block. + var blocks []pilosa.AttrBlock + for cur.nextBlock() { + block := pilosa.AttrBlock{ID: cur.blockID()} + + // Compute checksum of every key/value in block. + h := xxhash.New() + for k, v := cur.next(); k != nil; k, v = cur.next() { + h.Write(k) + h.Write(v) + } + block.Checksum = h.Sum(nil) + + // Append block. + blocks = append(blocks, block) + } + + return blocks, nil +} + +// BlockData returns all data for a single block. +func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + m := make(map[uint64]map[string]interface{}) + + // Start read-only transaction. + tx, err := s.db.Begin(false) + if err != nil { + return nil, err + } + defer tx.Rollback() + + // Move to the start of the block. + min := u64tob(uint64(i) * AttrBlockSize) + max := u64tob(uint64(i+1) * AttrBlockSize) + cur := tx.Bucket([]byte("attrs")).Cursor() + for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { + // Exit if we're past the end of the block. + if bytes.Compare(k, max) != -1 { + break + } + + // Decode attribute map and associate with id. + attrs, err := pilosa.DecodeAttrs(v) + if err != nil { + return nil, err + } + m[btou64(k)] = attrs + + } + + return m, nil +} + +// txAttrs returns a map of attributes for an id. +func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { + v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) + if v == nil { + return emptyMap, nil + } + return pilosa.DecodeAttrs(v) +} + +// txUpdateAttrs updates the attributes for an id. +// Returns the new combined set of attributes for the id. +func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) { + attr, err := txAttrs(tx, id) + if err != nil { + return nil, err + } + + // Create a new map if it is empty so we don't update emptyMap. + if len(attr) == 0 { + attr = make(map[string]interface{}, len(m)) + } + + // Merge attributes with original values. + // Nil values should delete keys. + for k, v := range m { + if v == nil { + delete(attr, k) + continue + } + + switch v := v.(type) { + case int: + attr[k] = int64(v) + case uint: + attr[k] = int64(v) + case uint64: + attr[k] = int64(v) + case string, int64, bool, float64: + attr[k] = v + default: + return nil, fmt.Errorf("invalid attr type: %T", v) + } + } + + // Marshal and save new values. + buf, err := pilosa.EncodeAttrs(attr) + if err != nil { + return nil, err + } + if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil { + return nil, err + } + return attr, nil +} + +// u64tob encodes v to big endian encoding. +func u64tob(v uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, v) + return b +} + +// btou64 decodes b from big endian encoding. +func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } + +// emptyMap is a reusable map that contains no keys. +var emptyMap = make(map[string]interface{}) + +// mapContains returns true if all keys & values of subset are in m. +func mapContains(m, subset map[string]interface{}) bool { + for k, v := range subset { + value, ok := m[k] + if !ok || value != v { + return false + } + } + return true +} + +// uint64Slice represents a sortable slice of uint64 numbers. +type uint64Slice []uint64 + +func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p uint64Slice) Len() int { return len(p) } +func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } + +// merge combines p and other to a unique sorted set of values. +// p and other must both have unique sets and be sorted. +func (p uint64Slice) merge(other []uint64) []uint64 { + ret := make([]uint64, 0, len(p)) + + i, j := 0, 0 + for i < len(p) && j < len(other) { + a, b := p[i], other[j] + if a == b { + ret = append(ret, a) + i, j = i+1, j+1 + } else if a < b { + ret = append(ret, a) + i++ + } else { + ret = append(ret, b) + j++ + } + } + + if i < len(p) { + ret = append(ret, p[i:]...) + } else if j < len(other) { + ret = append(ret, other[j:]...) + } + + return ret +} + +// blockCursor represents a cursor for iterating over blocks of a bolt bucket. +type blockCursor struct { + cur *bolt.Cursor + base uint64 + n uint64 + + buf struct { + key []byte + value []byte + filled bool + } +} + +// newBlockCursor returns a new block cursor that wraps cur using n sized blocks. +func newBlockCursor(c *bolt.Cursor, n int) blockCursor { + cur := blockCursor{ + cur: c, + n: uint64(n), + } + cur.buf.key, cur.buf.value = c.First() + cur.buf.filled = true + return cur +} + +// blockID returns the current block ID. Only valid after call to nextBlock(). +func (cur *blockCursor) blockID() uint64 { return cur.base } + +// nextBlock moves the cursor to the next block. +// Returns true if another block exists, otherwise returns false. +func (cur *blockCursor) nextBlock() bool { + if cur.buf.key == nil { + return false + } + + cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n + return true +} + +// next returns the next key/value within the block. +// Returns nils at the end of the block. +func (cur *blockCursor) next() (key, value []byte) { + // Use buffered value, if set. + if cur.buf.filled { + key, value = cur.buf.key, cur.buf.value + cur.buf.filled = false + return key, value + } + + // Read next key. + key, value = cur.cur.Next() + + // Fill buffer for EOF. + if key == nil { + cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false + return nil, nil + } + + // Parse key and buffer if outside of block. + id := binary.BigEndian.Uint64(key) + if id/cur.n > cur.base { + cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true + return nil, nil + } + + return key, value +} diff --git a/fragment.go b/fragment.go index 40a42dbe5..ab397af47 100644 --- a/fragment.go +++ b/fragment.go @@ -108,7 +108,7 @@ type Fragment struct { // Row attribute storage. // This is set by the parent frame unless overridden for testing. - RowAttrStore *AttrStore + RowAttrStore AttrStore stats StatsClient } diff --git a/frame.go b/frame.go index 069e0bc89..466d7ae96 100644 --- a/frame.go +++ b/frame.go @@ -51,7 +51,7 @@ type Frame struct { views map[string]*View // Row attribute storage and cache - rowAttrStore *AttrStore + rowAttrStore AttrStore broadcaster Broadcaster Stats StatsClient @@ -80,8 +80,9 @@ func NewFrame(path, index, name string) (*Frame, error) { index: index, name: name, - views: make(map[string]*View), - rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), + views: make(map[string]*View), + + rowAttrStore: NopAttrStore, broadcaster: NopBroadcaster, Stats: NopStatsClient, @@ -108,7 +109,7 @@ func (f *Frame) Index() string { return f.index } func (f *Frame) Path() string { return f.path } // RowAttrStore returns the attribute storage. -func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore } +func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore } // MaxSlice returns the max slice in the frame. func (f *Frame) MaxSlice() uint64 { diff --git a/holder.go b/holder.go index 4c6c10423..2ebc1a639 100644 --- a/holder.go +++ b/holder.go @@ -55,6 +55,9 @@ type Holder struct { opened chan struct{} Broadcaster Broadcaster + + AttrStoreGenerator AttrStoreGenerator + // Close management wg sync.WaitGroup closing chan struct{} @@ -82,6 +85,8 @@ func NewHolder() *Holder { Broadcaster: NopBroadcaster, Stats: NopStatsClient, + AttrStoreGenerator: NopAttrStoreGenerator, + CacheFlushInterval: DefaultCacheFlushInterval, LogOutput: os.Stderr, @@ -372,6 +377,8 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { index.LogOutput = h.LogOutput index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.Broadcaster + index.AttrStoreGenerator = h.AttrStoreGenerator + index.columnAttrStore = h.AttrStoreGenerator.New(filepath.Join(index.path, ".data")) return index, nil } diff --git a/index.go b/index.go index fb445bb57..ba50f47e1 100644 --- a/index.go +++ b/index.go @@ -55,8 +55,10 @@ type Index struct { remoteMaxSlice uint64 remoteMaxInverseSlice uint64 + AttrStoreGenerator AttrStoreGenerator + // Column attribute storage and cache. - columnAttrStore *AttrStore + columnAttrStore AttrStore // InputDefinitions by name. inputDefinitions map[string]*InputDefinition @@ -83,7 +85,8 @@ func NewIndex(path, name string) (*Index, error) { remoteMaxSlice: 0, remoteMaxInverseSlice: 0, - columnAttrStore: NewAttrStore(filepath.Join(path, ".data")), + AttrStoreGenerator: NopAttrStoreGenerator, + columnAttrStore: NopAttrStore, columnLabel: DefaultColumnLabel, @@ -100,7 +103,7 @@ func (i *Index) Name() string { return i.name } func (i *Index) Path() string { return i.path } // ColumnAttrStore returns the storage for column attributes. -func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore } +func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } // SetColumnLabel sets the column label. Persists to meta file on update. func (i *Index) SetColumnLabel(v string) error { @@ -256,9 +259,7 @@ func (i *Index) Close() error { defer i.mu.Unlock() // Close the attribute store. - if i.columnAttrStore != nil { - i.columnAttrStore.Close() - } + i.columnAttrStore.Close() // Close all frames. for _, f := range i.frames { @@ -530,6 +531,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { f.LogOutput = i.LogOutput f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) f.broadcaster = i.broadcaster + f.rowAttrStore = i.AttrStoreGenerator.New(filepath.Join(f.path, ".data")) return f, nil } diff --git a/server.go b/server.go index 54bacda1d..c939b5ba1 100644 --- a/server.go +++ b/server.go @@ -74,6 +74,8 @@ type Server struct { GCNotifier GCNotifier + AttrStoreGenerator AttrStoreGenerator + // Background monitoring intervals. AntiEntropyInterval time.Duration MetricInterval time.Duration @@ -107,6 +109,8 @@ func NewServer() *Server { GCNotifier: NopGCNotifier, + AttrStoreGenerator: NopAttrStoreGenerator, + AntiEntropyInterval: DefaultAntiEntropyInterval, MetricInterval: 0, DiagnosticInterval: 0, diff --git a/server/server.go b/server/server.go index eac98121d..976e277c4 100644 --- a/server/server.go +++ b/server/server.go @@ -33,6 +33,7 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" @@ -146,6 +147,9 @@ func (m *Command) SetupServer() error { // Configure data directory (for Cluster .topology) m.Server.Cluster.Path = m.Config.DataDir + m.Server.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() + m.Server.Holder.AttrStoreGenerator = m.Server.AttrStoreGenerator + // Configure holder. m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir diff --git a/test/attr.go b/test/attr.go index 9363011b1..de87144af 100644 --- a/test/attr.go +++ b/test/attr.go @@ -21,12 +21,13 @@ import ( "sync" "testing" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // AttrStore represents a test wrapper for pilosa.AttrStore. type AttrStore struct { - *pilosa.AttrStore + //*pilosa.AttrStore + *boltdb.AttrStore } // NewAttrStore returns a new instance of AttrStore. @@ -38,7 +39,7 @@ func NewAttrStore() *AttrStore { f.Close() os.Remove(f.Name()) - return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())} + return &AttrStore{AttrStore: boltdb.NewAttrStore(f.Name())} } func BenchmarkAttrStore_Duplicate(b *testing.B) { diff --git a/test/holder.go b/test/holder.go index c9d21d2f2..2778cf8c2 100644 --- a/test/holder.go +++ b/test/holder.go @@ -20,6 +20,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // Holder is a test wrapper for pilosa.Holder. @@ -38,6 +39,7 @@ func NewHolder() *Holder { h := &Holder{Holder: pilosa.NewHolder()} h.Path = path h.Holder.LogOutput = &h.LogOutput + h.Holder.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() return h } @@ -64,6 +66,7 @@ func (h *Holder) Reopen() error { h.Holder = pilosa.NewHolder() h.Holder.Path = path h.Holder.LogOutput = logOutput + h.Holder.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() if err := h.Holder.Open(); err != nil { return err } diff --git a/test/pilosa.go b/test/pilosa.go index 2bfd0c1a6..f798e4a85 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" @@ -49,6 +50,8 @@ func NewMain() *Main { m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} m.Server.Network = *Network + m.Server.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() + m.Server.Holder.AttrStoreGenerator = m.Server.AttrStoreGenerator m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true @@ -136,6 +139,8 @@ func (m *Main) Reopen() error { config := m.Config m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) m.Server.Network = *Network + m.Server.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() + m.Server.Holder.AttrStoreGenerator = m.Server.AttrStoreGenerator m.Config = config // Run new program. diff --git a/view.go b/view.go index db2eb97f1..dfedebbe3 100644 --- a/view.go +++ b/view.go @@ -63,7 +63,7 @@ type View struct { broadcaster Broadcaster stats StatsClient - RowAttrStore *AttrStore + RowAttrStore AttrStore LogOutput io.Writer } From 68581d50c0a53c9fe82484ae55ffb477b457cee6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 19 Mar 2018 14:04:21 -0500 Subject: [PATCH 162/311] Remove empty line between godoc and package declaration --- statik/filesystem.go | 1 - 1 file changed, 1 deletion(-) diff --git a/statik/filesystem.go b/statik/filesystem.go index 6d7fd864f..e3bf95cb1 100644 --- a/statik/filesystem.go +++ b/statik/filesystem.go @@ -16,7 +16,6 @@ // // Package statik contains static assets for the Web UI. `go generate` or // `make generate-statik` will produce statik.go, which is ignored by git. - package statik import ( From 37015e32a5eddb80a812f9ac9a7f51c315ad084a Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 19 Mar 2018 14:25:36 -0500 Subject: [PATCH 163/311] Ensure internal links end with slash --- docs/administration.md | 6 +++--- docs/api-reference.md | 14 +++++++------- docs/examples.md | 4 ++-- docs/getting-started.md | 4 ++-- docs/glossary.md | 2 +- docs/introduction.md | 2 +- docs/query-language.md | 8 ++++---- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index d3548553f..44d3fcbb6 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -150,9 +150,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: - - [Host](../configuration#metric-host): specify host that receives metric events - - [Poll Interval](../configuration#metric-poll-interval): specify polling interval for runtime metrics - - [Service](../configuration#metric-service): declare type StatsD or Expvar + - [Host](../configuration/#metric-host): specify host that receives metric events + - [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics + - [Service](../configuration/#metric-service): declare type StatsD or Expvar #### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: diff --git a/docs/api-reference.md b/docs/api-reference.md index 5a28313b1..1dbc59865 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -78,7 +78,7 @@ In order to send protobuf binaries in the request and response, set `Content-Typ The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`. -The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. +The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. ``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ @@ -102,9 +102,9 @@ Creates a frame in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: -* `timeQuantum` (string): [Time Quantum](../data-model#time-quantum) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view](../data-model#inverse) for this frame if `true`. -* `cacheType` (string): [ranked](../data-model#ranked) or [LRU](../data-model#lru) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. +* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. +* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `rangeEnabled` (boolean): Enables range-encoded fields in this frame. * `fields` (array): List of range-encoded fields. @@ -205,9 +205,9 @@ Creates an input definition in the given index with the given name. The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options: -* `timeQuantum` (string): [Time Quantum](../data-model#time-quantum) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view](../data-model#inverse) for this frame if `true`. -* `cacheType` (string): [ranked](../data-model#ranked) or [LRU](../data-model#lru) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. +* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. +* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa. diff --git a/docs/examples.md b/docs/examples.md index 12dd69617..589539081 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -17,7 +17,7 @@ New York City released an extremely detailed data set of over 1 billion taxi rid Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). -We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. +We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. @@ -263,7 +263,7 @@ python import_from_sdf.py -p -file id_fingerprint.csv ``` -First, follow the instruction in the [getting started](../getting-started) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +First, follow the instruction in the [getting started](../getting-started/) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 ``` curl localhost:10101/index/mole \ diff --git a/docs/getting-started.md b/docs/getting-started.md index 6a9eb54be..0b773ea2c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -20,7 +20,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th ### Starting Pilosa -Follow the steps in the [Install](../installation) document to install Pilosa. +Follow the steps in the [Install](../installation/) document to install Pilosa. Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`): ``` pilosa server @@ -42,7 +42,7 @@ curl localhost:10101/status In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project. -Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation. +Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation. #### Create the Schema diff --git a/docs/glossary.md b/docs/glossary.md index 6ea0549ef..91664eb5c 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -62,6 +62,6 @@ nav = [] [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration). -[TopN](../query-language#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). [View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. diff --git a/docs/introduction.md b/docs/introduction.md index 7e7049c6c..d2ebb4662 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w "What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface. -Once you have Pilosa [installed](../installation), the [getting started](../getting-started) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. +Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. diff --git a/docs/query-language.md b/docs/query-language.md index 83cb946e3..bea99804c 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -13,7 +13,7 @@ nav = [ ### Overview -This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: +This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary/#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: ``` {"results":[...]} @@ -45,7 +45,7 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types -* `frame` The frame specifies on which Pilosa [frame](../glossary#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. * `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated. * `COL_LABEL` The default column label is `columnID`, changing the default is deprecated. * `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") @@ -118,7 +118,7 @@ SetRowAttrs queries always return `null` upon success. SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true) ``` -Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. +Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. ``` SetRowAttrs(frame="stargazer", rowID=10, username=null) @@ -150,7 +150,7 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` -Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`. +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`. ``` SetColumnAttrs(columnID=10, url=null) From 836c8f4c2fef700b0d995456ed3ff4b51e8eaed8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 19 Mar 2018 14:29:53 -0500 Subject: [PATCH 164/311] remove the Gossip stutter from memberlist-related config options --- config.go | 44 ++++++++++++++++++++++---------------------- ctl/server.go | 6 +++--- gossip/gossip.go | 6 +++--- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/config.go b/config.go index 17841c967..f2dd41793 100644 --- a/config.go +++ b/config.go @@ -90,25 +90,25 @@ const ( DefaultGossipProbeInterval = 1 * time.Second DefaultGossipProbeTimeout = 500 * time.Millisecond - // GossipInterval and GossipNodes are used to configure the gossip + // Interval and Nodes are used to configure the gossip // behavior of memberlist. // - // GossipInterval is the interval between sending messages that need + // Interval is the interval between sending messages that need // to be gossiped that haven't been able to piggyback on probing messages. // If this is set to zero, non-piggyback gossip is disabled. By lowering // this value (more frequent) gossip messages are propagated across // the cluster more quickly at the expense of increased bandwidth. // - // GossipNodes is the number of random nodes to send gossip messages to - // per GossipInterval. Increasing this number causes the gossip messages + // Nodes is the number of random nodes to send gossip messages to + // per Interval. Increasing this number causes the gossip messages // to propagate across the cluster more quickly at the expense of // increased bandwidth. // - // GossipToTheDeadTime is the interval after which a node has died that + // ToTheDeadTime is the interval after which a node has died that // we will still try to gossip to it. This gives it a chance to refute. - DefaultGossipGossipInterval = 200 * time.Millisecond - DefaultGossipGossipNodes = 3 - DefaultGossipGossipToTheDeadTime = 30 * time.Second + DefaultGossipInterval = 200 * time.Millisecond + DefaultGossipNodes = 3 + DefaultGossipToTheDeadTime = 30 * time.Second DefaultMetricPollInterval = 0 * time.Minute ) @@ -146,17 +146,17 @@ type Config struct { } `toml:"cluster"` Gossip struct { - Port string `toml:"port"` - Seeds []string `toml:"seeds"` - Key string `toml:"key"` - StreamTimeout Duration `toml:"stream-timeout"` - SuspicionMult int `toml:"suspicion-mult"` - PushPullInterval Duration `toml:"push-pull-interval"` - ProbeTimeout Duration `toml:"probe-timeout"` - ProbeInterval Duration `toml:"probe-interval"` - GossipNodes int `toml:"gossip-nodes"` - GossipInterval Duration `toml:"gossip-interval"` - GossipToTheDeadTime Duration `toml:"gossip-to-the-dead-time"` + Port string `toml:"port"` + Seeds []string `toml:"seeds"` + Key string `toml:"key"` + StreamTimeout Duration `toml:"stream-timeout"` + SuspicionMult int `toml:"suspicion-mult"` + PushPullInterval Duration `toml:"push-pull-interval"` + ProbeTimeout Duration `toml:"probe-timeout"` + ProbeInterval Duration `toml:"probe-interval"` + Nodes int `toml:"nodes"` + Interval Duration `toml:"interval"` + ToTheDeadTime Duration `toml:"to-the-dead-time"` } `toml:"gossip"` AntiEntropy struct { @@ -197,9 +197,9 @@ func NewConfig() *Config { c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout) c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval) - c.Gossip.GossipNodes = DefaultGossipGossipNodes - c.Gossip.GossipInterval = Duration(DefaultGossipGossipInterval) - c.Gossip.GossipToTheDeadTime = Duration(DefaultGossipGossipToTheDeadTime) + c.Gossip.Nodes = DefaultGossipNodes + c.Gossip.Interval = Duration(DefaultGossipInterval) + c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime) // AntiEntropy config. c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) diff --git a/ctl/server.go b/ctl/server.go index e18f1a296..87d6a8e15 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -48,9 +48,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.") flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.") - flags.IntVarP(&srv.Config.Gossip.GossipNodes, "gossip.gossip-nodes", "", srv.Config.Gossip.GossipNodes, "Number of random nodes to send gossip messages to per GossipInterval.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipInterval), "gossip.gossip-interval", "", (time.Duration)(srv.Config.Gossip.GossipInterval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipToTheDeadTime), "gossip.gossip-to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.GossipToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.") + flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.") // AntiEntropy flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.") diff --git a/gossip/gossip.go b/gossip/gossip.go index 0949feb16..866d2e41e 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -187,9 +187,9 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval) conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout) conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval) - conf.GossipNodes = cfg.Gossip.GossipNodes - conf.GossipInterval = time.Duration(cfg.Gossip.GossipInterval) - conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.GossipToTheDeadTime) + conf.GossipNodes = cfg.Gossip.Nodes + conf.GossipInterval = time.Duration(cfg.Gossip.Interval) + conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime) // conf.Delegate = g conf.SecretKey = gossipKey From bd76bfc25bfd5c1ad4915d4fdf721d8670b4db39 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 19 Mar 2018 14:49:41 -0500 Subject: [PATCH 165/311] Fix a few more internal links --- docs/client-libraries.md | 6 +++--- docs/examples.md | 2 +- docs/query-language.md | 2 +- docs/webui.md | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/client-libraries.md b/docs/client-libraries.md index a328318ee..1c360741c 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -16,7 +16,7 @@ This section contains example code for client libraries in several languages. Pl You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. @@ -98,7 +98,7 @@ func main() { You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. @@ -171,7 +171,7 @@ client.query(stargazer.setbit(99999, 77777)) You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. diff --git a/docs/examples.md b/docs/examples.md index 589539081..cf9885a3f 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -163,7 +163,7 @@ durm := pdk.CustomMapper{ #### Import process -After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. +After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk/) for more details on this process. #### Queries diff --git a/docs/query-language.md b/docs/query-language.md index bea99804c..0ccca3a80 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -31,7 +31,7 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch ##### Examples -Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data. +Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data. The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: ``` request diff --git a/docs/webui.md b/docs/webui.md index 86874807b..97fa2ce8f 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -14,7 +14,7 @@ This can be used for constructing queries and viewing the cluster status. ### Console -The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. +The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. Each query's result will be displayed in the Output section along with the query time. @@ -31,7 +31,7 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference). +Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/). - `:create index timeQuantum=YM` - `:create frame inverseEnabled=true cacheSize=10000` From f3b140cfa4766dd9d273060b2eafef6719ecf963 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 19 Mar 2018 14:49:04 -0500 Subject: [PATCH 166/311] update docs to consider coordinator as boolean --- docs/configuration.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8125bae18..633e83f3c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,10 +27,10 @@ Every command line flag has a corresponding environment variable. The environmen ### Config file -The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator="localhost:10101"` and `--cluster.replicas=1` look like this in the config file: +The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file: ```toml [cluster] - coordinator = "localhost:10101" + coordinator = true replicas = 1 ``` @@ -129,14 +129,14 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Cluster Coordinator -* Description: Address of the node responsible for coordinating cluster membership and the cluster resize process. This value should be the same on all nodes in the cluster. -* Flag: `cluster.coordinator="localhost:10101"` -* Env: `PILOSA_CLUSTER_COORDINATOR="localhost:10101"` +* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator. +* Flag: `cluster.coordinator` +* Env: `PILOSA_CLUSTER_COORDINATOR` * Config: ```toml [cluster] - coordinator = "localhost:10101" + coordinator = true ``` #### Cluster Long Query Time @@ -299,7 +299,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - coordinator = "node0.pilosa.com:10101" + coordinator = true #### Node 1 @@ -312,7 +312,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - coordinator = "node0.pilosa.com:10101" + coordinator = false #### Node 2 @@ -325,7 +325,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - coordinator = "node0.pilosa.com:10101" + coordinator = false ### Example Cluster Configuration (HTTPS) @@ -344,7 +344,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - coordinator = "https://node0.pilosa.com:10101" + coordinator = true [tls] certificate = "/home/pilosa/private/server.crt" @@ -362,7 +362,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - coordinator = "https://node0.pilosa.com:10101" + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" @@ -380,7 +380,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - coordinator = "https://node0.pilosa.com:10101" + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" @@ -402,7 +402,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - coordinator = "https://node0.pilosa.com:10100" + coordinator = true [tls] certificate = "/home/pilosa/private/server.crt" @@ -420,7 +420,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - coordinator = "https://node0.pilosa.com:10100" + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" @@ -438,7 +438,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - coordinator = "https://node0.pilosa.com:10100" + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" From fa66de0461cd809ffec2f72d815cd56750321521 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 19 Mar 2018 16:13:54 -0500 Subject: [PATCH 167/311] Fix one more link --- docs/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/glossary.md b/docs/glossary.md index 91664eb5c..612c65dd4 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -60,7 +60,7 @@ nav = [] [Time quantum](../data-model/#time-quantum): Defines the granularity to be used for time [Range](#range) queries. -[TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration). +[TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). [TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). From 02a822bcd389a3b28e3206523f4a5a686ff2fa47 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 12:44:39 -0500 Subject: [PATCH 168/311] Merge README-dev.md into CONTRIBUTING.md --- CONTRIBUTING.md | 73 +++++++++++++++++++++++++++++++++++++++++-------- README-dev.md | 69 ---------------------------------------------- 2 files changed, 61 insertions(+), 81 deletions(-) delete mode 100644 README-dev.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d15c42ed7..10b19eae1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,30 +2,79 @@ ## Reporting a bug -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1] +If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. ## Submitting a feature request -Feature requests are managed in Github issues. New features typically go through a [Proposal Process][4] +Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] which starts by [opening a new issue][1] that describes the new feature proposal. -## Submitting code changes +## Making code contributions Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing before you start working, otherwise you run the risk of duplicating effort. This also +you're doing, otherwise you run the risk of duplicating effort. This also gives others an opportunity to provide input for your feature. If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. -- Fork the [Pilosa repository][2] and then clone your fork: - ```shell - git clone git@github.com:/pilosa.git +### Development Environment + +- Ensure you have a recent version of [Go](https://golang.org/dl/) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information. + +- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`. + +- Fork the [Pilosa repository][2] to your own account. + +- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo: + + ```sh + mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ + git clone git@github.com:${USER}/pilosa.git + ``` + +- `cd` to your pilosa directory: + + ```sh + cd ${GOPATH}/src/github.com/pilosa/pilosa + ``` + +- Install `dep` to manage dependencies: + + ```sh + go get -u github.com/golang/dep/cmd/dep + ``` + +- Install Pilosa command line tools: + + ```sh + make install + # or: + # dep ensure && go install github.com/pilosa/pilosa/cmd/... + ``` + + Running `pilosa` should now run a Pilosa instance. + +- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo: + + ```sh + cd ${GOPATH}/src/github.com/pilosa/pilosa + git remote add upstream git@github.com:pilosa/pilosa.git + ``` + +### Submitting code changes + +- Before starting to work on a task, sync your branch with the upstream: + + ```sh + git fetch upstream + git checkout master + git merge upstream/master ``` - Create a local feature branch: - ```shell + ```sh git checkout -b something-amazing ``` @@ -33,13 +82,13 @@ If you want to help but you aren't sure where to start, check out our [github la - Make sure that you've written tests for your new feature, and then run the tests: - ```shell + ```sh make test ``` - Verify that your pull request is applied to the latest version of code on github: - ```shell + ```sh git remote add upstream git@github.com:pilosa/pilosa.git git fetch upstream git rebase -i upstream/master @@ -47,7 +96,7 @@ If you want to help but you aren't sure where to start, check out our [github la - Push to your fork: - ```shell + ```sh git push -u something-amazing ``` @@ -59,4 +108,4 @@ If you want to help but you aren't sure where to start, check out our [github la [3]: https://github.com/pilosa/pilosa/compare/ [4]: https://github.com/pilosa/general/blob/master/proposal.md [5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer \ No newline at end of file +[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/README-dev.md b/README-dev.md deleted file mode 100644 index 809befbc3..000000000 --- a/README-dev.md +++ /dev/null @@ -1,69 +0,0 @@ - -Development Environment -======================= - -Install Go versions 1.6.2+ or 1.7 for your platform. - -Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private. - -Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`. - -Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo: - -```sh -mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ -git clone git@github.com:${USER}/pilosa.git -``` - -`cd` to your pilosa directory: - -```sh -cd ${GOPATH}/src/github.com/pilosa/pilosa -``` - -Install `dep` to manage dependencies: - -```sh -go get -u github.com/golang/dep/cmd/dep -``` - -Install Pilosa command line tools: - -```sh -make install -# or: -# dep ensure && go install github.com/pilosa/pilosa/cmd/... -``` - -Running `pilosa` should now run a Pilosa instance. - -In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo: - -```sh -cd ${GOPATH}/src/github.com/pilosa/pilosa -git remote add upstream git@github.com:pilosa/pilosa.git -``` - -Before starting to work on a task, sync your branch with the upstream: - -```sh -git fetch upstream -git checkout master -git merge upstream/master -``` - -Create a branch for the task: - -```sh -git checkout -b a-branch-for-the-task -``` - -Update the code in the branch, and commit it. - -Push it to your own repo: - -```sh -git push --set-upstream origin a-branch-for-the-task -``` - -All left to do is creating a pull request on github.com. From 457b1f394c2de0bddb1852c3d5580a6f3ab37de2 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 12:46:11 -0500 Subject: [PATCH 169/311] Add format details to docs readme --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 4bad84cd7..6a9f50b45 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -Pilosa docs are maintained here, to stay in sync with the codebase. +Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/). Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search. From ef4c7098288fd93d4840d9ff0c723d4ad254c232 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 12:47:22 -0500 Subject: [PATCH 170/311] Deprecate input definitionn in docs --- docs/api-reference.md | 16 ++++++++++ docs/getting-started.md | 9 ++++-- docs/input-definition.md | 63 ++++------------------------------------ 3 files changed, 28 insertions(+), 60 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 1dbc59865..2c5309da0 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -199,6 +199,10 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \ ### Create input definition +
+Input definition is deprecated as of v0.9. +
+ `POST /index//input-definition/` Creates an input definition in the given index with the given name. @@ -263,6 +267,10 @@ curl localhost:10101/index/user/input-definition/stargazer-input \ ### Get input definition +
+Input definition is deprecated as of v0.9. +
+ `GET /index//input-definition/` Returns the given input definition as JSON. @@ -276,6 +284,10 @@ curl -XGET localhost:10101/index/user/input-definition/stargazer-input ### Remove input definition +
+Input definition is deprecated as of v0.9. +
+ `DELETE /index//input-definition/` Removes the given input definition. @@ -289,6 +301,10 @@ curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input ### Process input data +
+Input definition is deprecated as of v0.9. +
+ `POST /index//input/` Processes the JSON payload using the given input definition. diff --git a/docs/getting-started.md b/docs/getting-started.md index 0b773ea2c..bd227d154 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -85,7 +85,7 @@ curl localhost:10101/index/repository/frame/language \ #### Import Data From CSV Files -If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo. +Download the `stargazer.csv` and `language.csv` files here: ``` curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv @@ -110,7 +110,12 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages. ### Input Definition -Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. + +
+Input definition is deprecated as of v0.9. +
+ +Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. #### Make Some Queries diff --git a/docs/input-definition.md b/docs/input-definition.md index 7d50c6f9d..9d0e5fe6b 100644 --- a/docs/input-definition.md +++ b/docs/input-definition.md @@ -1,64 +1,11 @@ +++ title = "Input Definition" weight = 8 -nav = [ - "Create the Schema", - "Import Data", -] +++ ## Input Definition -This document builds on the data import concepts introduced in [Getting Started](../getting-started/). -Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data. - -### Create the Schema - -Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations. - -Before creating a schema, let's create the repository index first: - -``` -curl localhost:10101/index/repository -X POST -``` -The sample input definition schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started) in the `input_definition.json` file. Download it using: -``` -curl -OL https://github.com/pilosa/getting-started/raw/master/input_definition.json -``` - -Run the following to create the input definition: -``` -curl localhost:10101/index/repository/input-definition/stargazer -d @input_definition.json -``` - -Instead of creating a `stargazer` frame and a `language` frame individually like in [Getting Started](../getting-started/), we can create multiple frames in one input definition. -We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination: - - - value-to-row: The value for this field is used as the `rowID`. - - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame - -### Import Data - -The sample data for the input definition we created above is in the `json_input.json` file at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download it using: -``` -curl -OL https://github.com/pilosa/getting-started/raw/master/json_input.json -``` - -Then run the following to import it: -``` -curl localhost:10101/index/repository/input/stargazer -d @json_input.json -``` - -As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID. -The data input above is equivalent to the following `SetBit()` operations: - -``` -curl localhost:10101/index/repository/query \ - -X POST \ - -d 'SetBit(frame="stargazer", columnID=91720568, rowID=513114) - SetBit(frame="stargazer", columnID=91720568, rowID=513114, timestamp="2017-05-18T20:40") - SetBit(frame="language", columnID=91720568, rowID=5) - SetBit(frame="language", columnID=95122322, rowID=17) - ' -``` +
+Input definition is deprecated as of Pilosa v0.9.
+
+The previous version of this page is still available here. +
From 0800c0348a7163d1a75be1670673a017ce7fe4b7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 12:48:46 -0500 Subject: [PATCH 171/311] Add readthedocs link --- docs/client-libraries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client-libraries.md b/docs/client-libraries.md index 1c360741c..d5074a059 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -96,7 +96,7 @@ func main() { ### Python -You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions. +You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions. We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. From 54cb6ce47ed3ef01b8c39578f0ffab514ab49ff3 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 14:21:16 -0500 Subject: [PATCH 172/311] Add description for external tutorials --- docs/tutorials.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials.md b/docs/tutorials.md index f9b4a1494..ccbcc36bc 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -10,6 +10,8 @@ nav = [ ### External Tutorials +Some of our tutorials work better as standalone repos, since you can `git clone` the instructions, code, and data all at once. Officially supported tutorials are listed here. + - [Run Pilosa with Microsoft's Azure Cosmos DB](https://github.com/pilosa/cosmosa) ## Tutorials From 9e5d011b4a70bc3d9de1a12f7931b4b16b9ece69 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 14:44:51 -0500 Subject: [PATCH 173/311] Remove 'go get' in favor of 'git clone' or curlbash --- CONTRIBUTING.md | 4 ++-- docs/installation.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10b19eae1..8a6ff2b4f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,10 +39,10 @@ If you want to help but you aren't sure where to start, check out our [github la cd ${GOPATH}/src/github.com/pilosa/pilosa ``` -- Install `dep` to manage dependencies: +- [Install](https://github.com/golang/dep/#installation) `dep` to manage dependencies: ```sh - go get -u github.com/golang/dep/cmd/dep + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh ``` - Install Pilosa command line tools: diff --git a/docs/installation.md b/docs/installation.md index 5a0ea3086..e26ddf94a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -141,7 +141,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 2. Clone the repo: ``` - go get -d github.com/pilosa/pilosa + git clone https://github.com/pilosa/pilosa.git $GOPATH/src/github.com/pilosa/pilosa ``` 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa): @@ -294,7 +294,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 2. Clone the repo: ``` - go get -d github.com/pilosa/pilosa + git clone https://github.com/pilosa/pilosa.git $GOPATH/src/github.com/pilosa/pilosa ``` 3. Build the Pilosa repo: From 23e65cc225988b12ca334898d06d3ae11f2a4ebc Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 14:45:27 -0500 Subject: [PATCH 174/311] Clean up a few links --- CONTRIBUTING.md | 12 ++++++++---- docs/installation.md | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a6ff2b4f..402895f1d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,9 +20,9 @@ If you want to help but you aren't sure where to start, check out our [github la ### Development Environment -- Ensure you have a recent version of [Go](https://golang.org/dl/) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information. +- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information. -- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`. +- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). - Fork the [Pilosa repository][2] to your own account. @@ -49,8 +49,12 @@ If you want to help but you aren't sure where to start, check out our [github la ```sh make install - # or: - # dep ensure && go install github.com/pilosa/pilosa/cmd/... + ``` + + or + + ``` + dep ensure && go install github.com/pilosa/pilosa/cmd/... ``` Running `pilosa` should now run a Pilosa instance. diff --git a/docs/installation.md b/docs/installation.md index e26ddf94a..71904b147 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -136,7 +136,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Install the prerequisites: - * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). + * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) 2. Clone the repo: @@ -289,7 +289,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. Install the prerequisites: - * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). + * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) 2. Clone the repo: From 96c7f285e54b9f5c7770ee59f8410529edc1369d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 14:52:15 -0500 Subject: [PATCH 175/311] Ensure directory exists before cloning --- docs/installation.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 71904b147..67558d904 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -141,7 +141,8 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 2. Clone the repo: ``` - git clone https://github.com/pilosa/pilosa.git $GOPATH/src/github.com/pilosa/pilosa + mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ + git clone https://github.com/pilosa/pilosa.git ``` 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa): @@ -294,7 +295,8 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 2. Clone the repo: ``` - git clone https://github.com/pilosa/pilosa.git $GOPATH/src/github.com/pilosa/pilosa + mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ + git clone https://github.com/pilosa/pilosa.git ``` 3. Build the Pilosa repo: From edb63e70b19029c5442cf3a1a46f89d0b72a409d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 15:17:48 -0500 Subject: [PATCH 176/311] Synchronize mac and linux install sections --- docs/installation.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 67558d904..9ca90ccf2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -145,7 +145,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa): +3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa make generate-statik @@ -204,7 +204,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) docker version ``` -If you don't see the server listed, start the Docker application. + If you don't see the server listed, start the Docker application. 3. Pull the official Pilosa image from Docker Hub: ``` @@ -299,9 +299,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo: +3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make generate-statik make install ``` From 38812a38aa166766c0c39991948c1b87abd19e72 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 15:21:48 -0500 Subject: [PATCH 177/311] Improve links --- docs/api-reference.md | 3 ++- docs/webui.md | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 2c5309da0..9092c92bc 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -107,9 +107,10 @@ The request payload is in JSON, and may contain the `options` field. The `option * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `rangeEnabled` (boolean): Enables range-encoded fields in this frame. -* `fields` (array): List of range-encoded fields. +* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: + * `name` (string): Field name. * `type` (string): Field type, currently only "int" is supported. * `min` (int): Minimum value allowed for this field. diff --git a/docs/webui.md b/docs/webui.md index 97fa2ce8f..525f1aed9 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -9,7 +9,8 @@ nav = [ ## WebUI -The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101) +The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101). + This can be used for constructing queries and viewing the cluster status. ### Console @@ -31,7 +32,7 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/). +Index and frame creation also supports options like timeQuantum or inverseEnabled. Check the API reference sections for [index](../api-reference/#change-index-time-quantum) and [frame](../api-reference/#create-frame) options to see other available keys. - `:create index timeQuantum=YM` - `:create frame inverseEnabled=true cacheSize=10000` From daca793c7f7b7782878f5849cda4e7d879ac1bfd Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 20 Mar 2018 16:00:51 -0500 Subject: [PATCH 178/311] Remove reference to deprecated index timeQuantum --- docs/webui.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/webui.md b/docs/webui.md index 525f1aed9..738fb20fd 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -32,9 +32,8 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Index and frame creation also supports options like timeQuantum or inverseEnabled. Check the API reference sections for [index](../api-reference/#change-index-time-quantum) and [frame](../api-reference/#create-frame) options to see other available keys. +Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). -- `:create index timeQuantum=YM` - `:create frame inverseEnabled=true cacheSize=10000` From a95993fb69cb26d2f9c7c59cd3f714e084be94d1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 20 Mar 2018 17:17:35 -0500 Subject: [PATCH 179/311] Remove unused cgo preamble --- bitmap.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bitmap.go b/bitmap.go index 72296efab..117d020f7 100644 --- a/bitmap.go +++ b/bitmap.go @@ -14,8 +14,6 @@ package pilosa -// #cgo CFLAGS:-mpopcnt - import ( "encoding/json" "sort" From cfd083042f9d8c69f86f5646b27b92925055d24c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 21 Mar 2018 11:44:59 -0500 Subject: [PATCH 180/311] Add CHANGELOG changes from v0.8 to master --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9676b4f89..ac62abc67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,53 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.8.8] - 2018-02-19 + +This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions. + +### Fixed + +- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) + +## [0.8.7] - 2018-02-12 + +This version contains 1 contribution from 1 contributors. There are 2 files changed, 84 insertions, and 4 deletions. + +### Fixed + +- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) + +## [0.8.6] - 2018-02-09 + +This version contains 2 contributions from 2 contributors. There are 3 files changed, 171 insertions, and 6 deletions. + +### Fixed + +- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) +- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) + +## [0.8.5] - 2018-01-18 + +This version contains 1 contribution from 1 contributor. There is 1 file changed, 1 insertion, and 0 deletions. + +### Fixed + +- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) + +## [0.8.4] - 2018-01-10 + +This version contains 4 contributions from 3 contributors. There are 17 files changed, 974 insertions, and 221 deletions. + +### Fixed + +- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) +- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) + +### Performance + +- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) +- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) + ## [0.8.3] - 2017-12-12 This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions. From 48081c12659f6513ecc55efae0012491db24760a Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 22 Mar 2018 11:30:51 -0500 Subject: [PATCH 181/311] Move external tutorial list to a note under the main heading --- docs/tutorials.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index ccbcc36bc..2401cdfe4 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -8,14 +8,18 @@ nav = [ ] +++ -### External Tutorials - -Some of our tutorials work better as standalone repos, since you can `git clone` the instructions, code, and data all at once. Officially supported tutorials are listed here. - -- [Run Pilosa with Microsoft's Azure Cosmos DB](https://github.com/pilosa/cosmosa) - ## Tutorials +
+ +Some of our tutorials work better as standalone repos, since you can git clone the instructions, code, and data all at once. Officially supported tutorials are listed here.
+
+ + +
+ ### Setting Up a Secure Cluster #### Introduction From f9efa6c57928597fb3d0aa62aa757fa0562d338d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 23 Mar 2018 13:51:50 -0500 Subject: [PATCH 182/311] Use type `func(string) AttrStore` in place of interface AttrStoreGenerator for simplicity --- attr.go | 19 +------------------ boltdb/attrstore.go | 15 +-------------- fragment_test.go | 2 +- holder.go | 8 ++++---- index.go | 8 ++++---- server.go | 4 ++-- server/server.go | 4 ++-- test/attr.go | 12 ++++++------ test/fragment.go | 6 +++--- test/holder.go | 4 ++-- test/pilosa.go | 8 ++++---- view_test.go | 6 +++--- 12 files changed, 33 insertions(+), 63 deletions(-) diff --git a/attr.go b/attr.go index 7e025ea5b..03ea4f43f 100644 --- a/attr.go +++ b/attr.go @@ -30,11 +30,6 @@ const ( AttrTypeFloat = 4 ) -// AttrStoreGenerator represents an interface for generating an AttrStore. -type AttrStoreGenerator interface { - New(string) AttrStore -} - // AttrStore represents an interface for handling row/column attributes. type AttrStore interface { Path() string @@ -48,25 +43,13 @@ type AttrStore interface { } func init() { - NopAttrStoreGenerator = &nopAttrStoreGenerator{ - Name: "NooP", - } NopAttrStore = &nopAttrStore{} } -// NopAttrStoreGenerator represents an AttrStoreGenerator that return a no-op AttrStore. -var NopAttrStoreGenerator AttrStoreGenerator - // NopAttrStore represents an AttrStore that doesn't do anything. var NopAttrStore AttrStore -// nopAttrStoreGenerator represents a no-op implementation of the AttrStoreGenerator interface. -type nopAttrStoreGenerator struct { - Name string -} - -// New is a no-op implementation of AttrStoreGenerator New method. -func (g *nopAttrStoreGenerator) New(string) AttrStore { +func NewNopAttrStore(string) AttrStore { return &nopAttrStore{} } diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 0ea35e663..ebb539903 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -62,14 +62,6 @@ func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { c.attrs[id] = attrs } -// AttrStoreGenerator represents a bolt implementation of the AttrStoreGenerator interface. -type AttrStoreGenerator struct{} - -// New is a bolt implementation of AttrStoreGenerator New method. -func (g *AttrStoreGenerator) New(path string) pilosa.AttrStore { - return NewAttrStore(path) -} - // AttrStore represents a storage layer for attributes. type AttrStore struct { mu sync.RWMutex @@ -78,11 +70,6 @@ type AttrStore struct { attrCache *AttrCache } -// NewAttrStoreGenerator returns a new instance of AttrStoreGenerator. -func NewAttrStoreGenerator() *AttrStoreGenerator { - return &AttrStoreGenerator{} -} - // NewAttrCache returns a new instance of AttrCache. func NewAttrCache() *AttrCache { return &AttrCache{ @@ -91,7 +78,7 @@ func NewAttrCache() *AttrCache { } // NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(path string) *AttrStore { +func NewAttrStore(path string) pilosa.AttrStore { return &AttrStore{ path: path, attrCache: NewAttrCache(), diff --git a/fragment_test.go b/fragment_test.go index 8a54d3b07..b86c10e3f 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -702,7 +702,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { Fragment: frag, RowAttrStore: test.MustOpenAttrStore(), } - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore if err := f.Open(); err != nil { panic(err) } diff --git a/holder.go b/holder.go index 2ebc1a639..63bbbbc86 100644 --- a/holder.go +++ b/holder.go @@ -56,7 +56,7 @@ type Holder struct { Broadcaster Broadcaster - AttrStoreGenerator AttrStoreGenerator + NewAttrStore func(string) AttrStore // Close management wg sync.WaitGroup @@ -85,7 +85,7 @@ func NewHolder() *Holder { Broadcaster: NopBroadcaster, Stats: NopStatsClient, - AttrStoreGenerator: NopAttrStoreGenerator, + NewAttrStore: NewNopAttrStore, CacheFlushInterval: DefaultCacheFlushInterval, @@ -377,8 +377,8 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { index.LogOutput = h.LogOutput index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.Broadcaster - index.AttrStoreGenerator = h.AttrStoreGenerator - index.columnAttrStore = h.AttrStoreGenerator.New(filepath.Join(index.path, ".data")) + index.NewAttrStore = h.NewAttrStore + index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data")) return index, nil } diff --git a/index.go b/index.go index ba50f47e1..707b87213 100644 --- a/index.go +++ b/index.go @@ -55,7 +55,7 @@ type Index struct { remoteMaxSlice uint64 remoteMaxInverseSlice uint64 - AttrStoreGenerator AttrStoreGenerator + NewAttrStore func(string) AttrStore // Column attribute storage and cache. columnAttrStore AttrStore @@ -85,8 +85,8 @@ func NewIndex(path, name string) (*Index, error) { remoteMaxSlice: 0, remoteMaxInverseSlice: 0, - AttrStoreGenerator: NopAttrStoreGenerator, - columnAttrStore: NopAttrStore, + NewAttrStore: NewNopAttrStore, + columnAttrStore: NopAttrStore, columnLabel: DefaultColumnLabel, @@ -531,7 +531,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { f.LogOutput = i.LogOutput f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) f.broadcaster = i.broadcaster - f.rowAttrStore = i.AttrStoreGenerator.New(filepath.Join(f.path, ".data")) + f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) return f, nil } diff --git a/server.go b/server.go index c939b5ba1..7f6eeb6cb 100644 --- a/server.go +++ b/server.go @@ -74,7 +74,7 @@ type Server struct { GCNotifier GCNotifier - AttrStoreGenerator AttrStoreGenerator + NewAttrStore func(string) AttrStore // Background monitoring intervals. AntiEntropyInterval time.Duration @@ -109,7 +109,7 @@ func NewServer() *Server { GCNotifier: NopGCNotifier, - AttrStoreGenerator: NopAttrStoreGenerator, + NewAttrStore: NewNopAttrStore, AntiEntropyInterval: DefaultAntiEntropyInterval, MetricInterval: 0, diff --git a/server/server.go b/server/server.go index 976e277c4..8e1bb6b97 100644 --- a/server/server.go +++ b/server/server.go @@ -147,8 +147,8 @@ func (m *Command) SetupServer() error { // Configure data directory (for Cluster .topology) m.Server.Cluster.Path = m.Config.DataDir - m.Server.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() - m.Server.Holder.AttrStoreGenerator = m.Server.AttrStoreGenerator + m.Server.NewAttrStore = boltdb.NewAttrStore + m.Server.Holder.NewAttrStore = boltdb.NewAttrStore // Configure holder. m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) diff --git a/test/attr.go b/test/attr.go index de87144af..16e8ff334 100644 --- a/test/attr.go +++ b/test/attr.go @@ -21,17 +21,17 @@ import ( "sync" "testing" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" ) // AttrStore represents a test wrapper for pilosa.AttrStore. type AttrStore struct { - //*pilosa.AttrStore - *boltdb.AttrStore + pilosa.AttrStore } // NewAttrStore returns a new instance of AttrStore. -func NewAttrStore() *AttrStore { +func NewAttrStore(string) pilosa.AttrStore { f, err := ioutil.TempFile("", "pilosa-attr-") if err != nil { panic(err) @@ -39,7 +39,7 @@ func NewAttrStore() *AttrStore { f.Close() os.Remove(f.Name()) - return &AttrStore{AttrStore: boltdb.NewAttrStore(f.Name())} + return &AttrStore{boltdb.NewAttrStore(f.Name())} } func BenchmarkAttrStore_Duplicate(b *testing.B) { @@ -75,8 +75,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) { } // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() *AttrStore { - s := NewAttrStore() +func MustOpenAttrStore() pilosa.AttrStore { + s := NewAttrStore("") if err := s.Open(); err != nil { panic(err) } diff --git a/test/fragment.go b/test/fragment.go index b208e7a3a..39caa8db2 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -27,7 +27,7 @@ const SliceWidth = pilosa.SliceWidth // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { *pilosa.Fragment - RowAttrStore *AttrStore + RowAttrStore pilosa.AttrStore } // NewFragment returns a new instance of Fragment with a temporary path. @@ -43,7 +43,7 @@ func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fra RowAttrStore: MustOpenAttrStore(), } f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore return f } @@ -78,7 +78,7 @@ func (f *Fragment) Reopen() error { f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore if err := f.Open(); err != nil { return err } diff --git a/test/holder.go b/test/holder.go index 2778cf8c2..59402cea0 100644 --- a/test/holder.go +++ b/test/holder.go @@ -39,7 +39,7 @@ func NewHolder() *Holder { h := &Holder{Holder: pilosa.NewHolder()} h.Path = path h.Holder.LogOutput = &h.LogOutput - h.Holder.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() + h.Holder.NewAttrStore = boltdb.NewAttrStore return h } @@ -66,7 +66,7 @@ func (h *Holder) Reopen() error { h.Holder = pilosa.NewHolder() h.Holder.Path = path h.Holder.LogOutput = logOutput - h.Holder.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() + h.Holder.NewAttrStore = boltdb.NewAttrStore if err := h.Holder.Open(); err != nil { return err } diff --git a/test/pilosa.go b/test/pilosa.go index f798e4a85..86623c239 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -50,8 +50,8 @@ func NewMain() *Main { m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} m.Server.Network = *Network - m.Server.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() - m.Server.Holder.AttrStoreGenerator = m.Server.AttrStoreGenerator + m.Server.NewAttrStore = NewAttrStore + m.Server.Holder.NewAttrStore = NewAttrStore m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true @@ -139,8 +139,8 @@ func (m *Main) Reopen() error { config := m.Config m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) m.Server.Network = *Network - m.Server.AttrStoreGenerator = boltdb.NewAttrStoreGenerator() - m.Server.Holder.AttrStoreGenerator = m.Server.AttrStoreGenerator + m.Server.NewAttrStore = boltdb.NewAttrStore + m.Server.Holder.NewAttrStore = m.Server.NewAttrStore m.Config = config // Run new program. diff --git a/view_test.go b/view_test.go index 64041aea0..87ed8e628 100644 --- a/view_test.go +++ b/view_test.go @@ -26,7 +26,7 @@ import ( // View is a test wrapper for pilosa.View. type View struct { *pilosa.View - RowAttrStore *test.AttrStore + RowAttrStore pilosa.AttrStore } // NewView returns a new instance of View with a temporary path. @@ -40,7 +40,7 @@ func NewView(index, frame, name string) *View { View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize), RowAttrStore: test.MustOpenAttrStore(), } - v.View.RowAttrStore = v.RowAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore return v } @@ -68,7 +68,7 @@ func (v *View) Reopen() error { } v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) - v.View.RowAttrStore = v.RowAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore if err := v.Open(); err != nil { return err } From 8686a8645cb513b15b70b636c6ae0b0dbceeb3b2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 23 Mar 2018 15:11:45 -0500 Subject: [PATCH 183/311] Default gossip seed should be empty instead of local bind address --- gossip/gossip.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 866d2e41e..b1c6902d8 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -63,6 +63,9 @@ func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { // Seeds returns the gossipSeeds determined by the config. func (g *GossipMemberSet) Seeds() []string { + if len(g.config.gossipSeeds) == 0 { + return []string{fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort)} + } return g.config.gossipSeeds } @@ -202,11 +205,6 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport g.statusHandler = server - // If no gossipSeeds is provided, use local host:port. - if len(cfg.Gossip.Seeds) == 0 { - g.config.gossipSeeds = []string{fmt.Sprintf("%s:%d", host, port)} - } - return g, nil } From 0ca7cfe498c51cfc415b73cb90bcaf11a7a4c1dc Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 23 Mar 2018 16:34:19 -0500 Subject: [PATCH 184/311] Change Seeds() to GetBindAddr() to clarify GossipMemberSet testing. --- gossip/gossip.go | 11 +++++------ server/cluster_test.go | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index b1c6902d8..62c794560 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -61,12 +61,11 @@ func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { return nil } -// Seeds returns the gossipSeeds determined by the config. -func (g *GossipMemberSet) Seeds() []string { - if len(g.config.gossipSeeds) == 0 { - return []string{fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort)} - } - return g.config.gossipSeeds +// GetBindAddr returns the gossip bind address based on config and auto bind port. +// This method is currently only used in a test scenario where a second node needs +// the auto-bind address of the first node to use as its gossip seed. +func (g *GossipMemberSet) GetBindAddr() string { + return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort) } // Open implements the MemberSet interface to start network activity. diff --git a/server/cluster_test.go b/server/cluster_test.go index 81a410e83..fbb784691 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -78,7 +78,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // get the host portion of addr to use for binding m1.Config.Gossip.Port = "0" - m1.Config.Gossip.Seeds = gossipMemberSet0.Seeds() + m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()} m1.Server.Cluster.Coordinator = m0.Server.NodeID m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput) From 59bb134a52718759fe90bb843a6583520115b550 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 21 Mar 2018 15:59:09 -0500 Subject: [PATCH 185/311] Add unfinished release notes for v0.9 --- CHANGELOG.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac62abc67..9df22a5e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,65 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased + +### Added + +- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), ([#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) +- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) +- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) +- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) +- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) +- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) +- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) +- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) +- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) +- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) +- Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) +- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013)) +- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) +- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), ([#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)) + +### Changed + +- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) +- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) +- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) +- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) +- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) +- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) +- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) +- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) +- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) +- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) +- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) +- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) +- Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) +- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) + +### Removed + +- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) +- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) +- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) + +### Fixed + +- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) +- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) +- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) +- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) +- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) +- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) +- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) +- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) +- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) + +### Performance + +- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) + ## [0.8.8] - 2018-02-19 This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions. From 684e8743968db0e87fa20fa53132629300e3efae Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 21 Mar 2018 16:08:56 -0500 Subject: [PATCH 186/311] Remove extra parentheses. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9df22a5e2..2a4d17174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added -- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), ([#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) +- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) - Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) - Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) - Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), ([#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)) ### Changed From 99eb00bbdc7787c551996b14575fd981463a8435 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 23 Mar 2018 14:20:23 -0500 Subject: [PATCH 187/311] Remove obsolete coverage tools from Makefile Go 1.10 allows -coverprofile when testing multiple packages, so these hacks are no longer needed. --- Makefile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8fdbcea3d..9eb2df7e6 100644 --- a/Makefile +++ b/Makefile @@ -40,20 +40,10 @@ test: vendor go test $(PKGS) $(TESTFLAGS) cover: vendor - mkdir -p build/coverage - echo "mode: set" > build/coverage/all.out - for pkg in $(PKGS) ; do \ - make cover-pkg PKG=$$pkg ; \ - done - -cover-pkg: - mkdir -p build/coverage - touch build/coverage/$(subst /,-,$(PKG)).out - go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) - tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + make test TESTFLAGS="-coverprofile=build/coverage.out" cover-viz: cover - go tool cover -html=build/coverage/all.out + go tool cover -html=build/coverage.out pilosa: vendor go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From b8b2b5880406b97c7b107fd91848050eda3e3612 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 26 Mar 2018 10:59:48 -0500 Subject: [PATCH 188/311] add benchmark for import and snapshot --- fragment_test.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/fragment_test.go b/fragment_test.go index 8a54d3b07..392607bcc 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1098,3 +1098,68 @@ func BenchmarkFragment_Snapshot(b *testing.B) { } } } +func BenchmarkFragment_FullSnapshot(b *testing.B) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + // Generate some intersecting data. + maxX := 1048576 / 2 + sz := maxX + rows := make([]uint64, sz, sz) + cols := make([]uint64, sz, sz) + + max := 0 + for row := 0; row < 100; row++ { + val := 1 + i := 0 + for col := 0; col < SliceWidth/2; col++ { + rows[i] = uint64(row) + cols[i] = uint64(val) + val += 2 + i++ + } + if err := f.Import(rows, cols); err != nil { + b.Fatalf("Error Building Sample: %s", err) + } + if row > max { + max = row + } + } + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + if err := f.Snapshot(); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkFragment_Import(b *testing.B) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + maxX := 1048576 * 5 * 2 + sz := maxX + rows := make([]uint64, sz, sz) + cols := make([]uint64, sz, sz) + i := 0 + for row := 0; row < 100; row++ { + val := 1 + for col := 0; col < SliceWidth/2; col++ { + rows[i] = uint64(row) + cols[i] = uint64(val) + val += 2 + i++ + } + if i == maxX { + break + } + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := f.Import(rows, cols); err != nil { + b.Fatalf("Error Building Sample: %s", err) + } + } +} From 2a8172b2a3100ead64830a93159cc490a73710e4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 26 Mar 2018 14:48:56 -0500 Subject: [PATCH 189/311] Remove RowLabel support --- executor.go | 47 ++++----- executor_test.go | 110 ++++++++++---------- fragment.go | 4 +- frame.go | 41 +------- frame_test.go | 43 -------- handler_internal_test.go | 9 +- handler_test.go | 4 +- index.go | 10 -- index_test.go | 34 +------ input_definition_test.go | 8 +- internal/private.pb.go | 214 +++++++++++++++------------------------ internal/private.proto | 1 - pilosa.go | 1 - server/cluster_test.go | 14 +-- server/server_test.go | 66 ++++++------ stats_test.go | 4 +- 16 files changed, 216 insertions(+), 394 deletions(-) diff --git a/executor.go b/executor.go index 854a9d8b9..de0d65df0 100644 --- a/executor.go +++ b/executor.go @@ -80,7 +80,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // MaxSlice can differ between inverse and standard views, so we need // to send queries to different slices based on orientation. var inverseSlices []uint64 - rowLabel := DefaultRowLabel columnLabel := DefaultColumnLabel // If slices aren't specified, then include all of them. @@ -131,10 +130,9 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic if f == nil { return nil, ErrFrameNotFound } - rowLabel = f.RowLabel() // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(rowLabel, columnLabel) { + if call.IsInverse(DefaultRowLabel, columnLabel) { slices = inverseSlices } } @@ -280,8 +278,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { frame, _ := c.Args["frame"].(string) if fr := idx.Frame(frame); fr != nil { - rowLabel := fr.RowLabel() - rowID, _, err := c.UintArg(rowLabel) + rowID, _, err := c.UintArg(DefaultRowLabel) if err != nil { return nil, err } @@ -536,18 +533,17 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() // Return an error if both the row and column label are specified. - rowID, rowOK, rowErr := c.UintArg(rowLabel) + rowID, rowOK, rowErr := c.UintArg(DefaultRowLabel) columnID, columnOK, columnErr := c.UintArg(columnLabel) if rowErr != nil || columnErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) } if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, columnLabel) } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, columnLabel) } // Determine row or column orientation. @@ -613,14 +609,13 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() // Read row & column id. columnID, columnOK, err := c.UintArg(columnLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) } - rowID, rowOK, err := c.UintArg(rowLabel) + rowID, rowOK, err := c.UintArg(DefaultRowLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } @@ -629,9 +624,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C var id uint64 var viewName string if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel) + return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, DefaultRowLabel) } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel) + return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, DefaultRowLabel) } else if columnOK { viewName, id = ViewInverse, columnID } else { @@ -906,14 +901,13 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal // Retrieve labels. columnLabel := idx.ColumnLabel() - rowLabel := f.RowLabel() // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("ClearBit() row field '%v' required", DefaultRowLabel) } colID, ok, err := c.UintArg(columnLabel) @@ -1000,14 +994,13 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, // Retrieve labels. columnLabel := idx.ColumnLabel() - rowLabel := f.RowLabel() // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { return false, fmt.Errorf("reading SetBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("SetBit() row field '%v' required", DefaultRowLabel) } colID, ok, err := c.UintArg(columnLabel) @@ -1171,20 +1164,19 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if frame == nil { return ErrFrameNotFound } - rowLabel := frame.RowLabel() // Parse labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetRowAttrs() row field '%v' required", rowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required", DefaultRowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, rowLabel) + delete(attrs, DefaultRowLabel) // Set attributes. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { @@ -1232,19 +1224,18 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { - return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) + return nil, fmt.Errorf("reading SetRowAttrs() row: %v", DefaultRowLabel) } else if !ok { - return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required", DefaultRowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, rowLabel) + delete(attrs, DefaultRowLabel) // Create frame group, if not exists. frameMap := m[frame] diff --git a/executor_test.go b/executor_test.go index c98239137..d90c8048f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit bits. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected bits: %+v", bits) @@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -122,7 +122,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { t.Fatalf("unexpected bits: %+v", bits) @@ -154,7 +154,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -184,7 +184,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -217,7 +217,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -233,7 +233,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -384,16 +384,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -419,15 +419,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=1) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, rowID=10, columnID=0) - SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, rowID=0, columnID=0) + SetBit(frame=f, row=0, columnID=0) + SetBit(frame=f, row=0, columnID=1) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, row=10, columnID=0) + SetBit(frame=f, row=10, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=20, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, row=0, columnID=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -543,7 +543,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -590,7 +590,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -630,8 +630,8 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, columnID=0) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) @@ -652,7 +652,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -679,22 +679,22 @@ func TestExecutor_Execute_Range(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, rowID=1, columnID=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, rowID=1, columnID=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, rowID=1, columnID=7, timestamp="2002-01-01T02:00") + SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-31T00:00") + SetBit(frame=f, row=1, columnID=3, timestamp="2000-01-01T00:00") + SetBit(frame=f, row=1, columnID=4, timestamp="2000-01-02T00:00") + SetBit(frame=f, row=1, columnID=5, timestamp="2000-02-01T00:00") + SetBit(frame=f, row=1, columnID=6, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, columnID=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, rowID=1, columnID=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, rowID=10, columnID=2, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-30T00:00") + SetBit(frame=f, row=1, columnID=2, timestamp="2002-02-01T00:00") + SetBit(frame=f, row=10, columnID=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) @@ -751,8 +751,8 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, columnID=0) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) @@ -908,7 +908,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", rowID=10)` { + } else if query.String() != `Bitmap(frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -931,7 +931,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected bits: %+v", bits) @@ -966,7 +966,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -994,7 +994,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10)` { + } else if query.String() != `SetBit(columnID=2, frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1012,7 +1012,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -1046,7 +1046,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(columnID=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1066,7 +1066,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -1169,7 +1169,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=10)"), nil, nil) + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=10)"), nil, nil) if err != nil { t.Fatal(err) } @@ -1186,7 +1186,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=20)"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=20)"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index ab397af47..8b32182d6 100644 --- a/fragment.go +++ b/fragment.go @@ -1841,11 +1841,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/frame.go b/frame.go index 466d7ae96..060125385 100644 --- a/frame.go +++ b/frame.go @@ -32,7 +32,7 @@ import ( // Default frame settings. const ( - DefaultRowLabel = "rowID" + DefaultRowLabel = "row" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false DefaultRangeEnabled = false @@ -87,7 +87,6 @@ func NewFrame(path, index, name string) (*Frame, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, @@ -139,39 +138,6 @@ func (f *Frame) MaxInverseSlice() uint64 { return view.MaxSlice() } -// SetRowLabel sets the row labels. Persists to meta file on update. -func (f *Frame) SetRowLabel(v string) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || f.rowLabel == v { - return nil - } - - // Make sure rowLabel is valid name - err := ValidateLabel(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - f.rowLabel = v - if err := f.saveMeta(); err != nil { - return err - } - - return nil -} - -// RowLabel returns the row label. -func (f *Frame) RowLabel() string { - f.mu.RLock() - v := f.rowLabel - f.mu.RUnlock() - return v -} - // CacheType returns the caching mode for the frame. func (f *Frame) CacheType() string { return f.cacheType @@ -224,7 +190,6 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ - RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, @@ -319,7 +284,6 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled f.cacheType = pb.CacheType if f.cacheType == "" { @@ -1030,7 +994,6 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` RangeEnabled bool `json:"rangeEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` @@ -1049,7 +1012,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { return nil } return &internal.FrameMeta{ - RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, @@ -1064,7 +1026,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { return nil } return &FrameOptions{ - RowLabel: options.RowLabel, InverseEnabled: options.InverseEnabled, RangeEnabled: options.RangeEnabled, CacheType: options.CacheType, diff --git a/frame_test.go b/frame_test.go index 6e4ade84a..2107bd703 100644 --- a/frame_test.go +++ b/frame_test.go @@ -267,49 +267,6 @@ func TestFrame_NameValidation(t *testing.T) { } } -// Ensure that frame RowLable validation is consistent. -func TestFrame_RowLabelValidation(t *testing.T) { - validRowLabels := []string{ - "", - "foo", - "hyphen-ated", - "under_score", - "abc123", - "trailing_", - "camelCase", - "UPPERCASE", - } - invalidRowLabels := []string{ - "123abc", - "x.y", - "_foo", - "-bar", - "abc def", - "a12345678901234567890123456789012345678901234567890123456789012345", - } - - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - f, err := pilosa.NewFrame(path, "i", "f") - if err != nil { - t.Fatalf("unexpected frame error: %s", err) - } - - for _, label := range validRowLabels { - if err := f.SetRowLabel(label); err != nil { - t.Fatalf("unexpected row label: %s %s", label, err) - } - } - for _, label := range invalidRowLabels { - if err := f.SetRowLabel(label); err == nil { - t.Fatalf("expected error on row label: %s", label) - } - } - -} - // Ensure frame can open and retrieve a view. func TestFrame_DeleteView(t *testing.T) { f := test.MustOpenFrame() diff --git a/handler_internal_test.go b/handler_internal_test.go index 169267c38..fb6226ea9 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -66,11 +66,10 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}}, - {json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, - {json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, + {json: `{"options": {"rowLabel": "test"}}`, err: "Unknown key: rowLabel:test"}, + {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, + {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { actual := &postFrameRequest{} diff --git a/handler_test.go b/handler_test.go index 551c81d66..3c9c08094 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1433,7 +1433,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { // Test input definition is deleted. index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -1470,7 +1470,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { h.Holder = hldr.Holder h.Cluster = test.NewCluster(1) - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} diff --git a/index.go b/index.go index 707b87213..968bac847 100644 --- a/index.go +++ b/index.go @@ -446,11 +446,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrInvalidCacheType } - // Validate that row label does not match column label. - if i.columnLabel == opt.RowLabel || (opt.RowLabel == "" && i.columnLabel == DefaultRowLabel) { - return nil, ErrColumnRowLabelEqual - } - // Validate mutually exclusive options if ranges are enabled. if opt.RangeEnabled { if opt.InverseEnabled { @@ -496,10 +491,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } f.cacheType = opt.CacheType - // Set options. - if opt.RowLabel != "" { - f.rowLabel = opt.RowLabel - } if opt.CacheSize != 0 { f.cacheSize = opt.CacheSize } @@ -694,7 +685,6 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin for _, fr := range pb.Frames { opt := FrameOptions{ // Deprecating row labels per #810. So, setting the default row label here. - RowLabel: DefaultRowLabel, InverseEnabled: fr.Meta.InverseEnabled, CacheType: fr.Meta.CacheType, CacheSize: fr.Meta.CacheSize, diff --git a/index_test.go b/index_test.go index 23a820f2d..c8cd238e8 100644 --- a/index_test.go +++ b/index_test.go @@ -215,32 +215,6 @@ func TestIndex_CreateFrame(t *testing.T) { } }) }) - - // Ensure frame cannot be created with a matching row label. - t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { - t.Run("Explicit", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - _, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel}) - if err != pilosa.ErrColumnRowLabelEqual { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("Default", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil { - t.Fatal(err) - } - - _, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != pilosa.ErrColumnRowLabelEqual { - t.Fatalf("unexpected error: %s", err) - } - }) - }) } // Ensure index can delete a frame. @@ -303,7 +277,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} @@ -330,7 +304,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { } // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -350,7 +324,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -381,7 +355,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} diff --git a/input_definition_test.go b/input_definition_test.go index 93d0e393d..efd13bec3 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -30,7 +30,7 @@ func TestInputDefinition_Open(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -115,14 +115,14 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err) } - frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{}} action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} @@ -131,7 +131,7 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err) } - frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}} action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} diff --git a/internal/private.pb.go b/internal/private.pb.go index 1b7f71ac4..488aca0fe 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -92,7 +92,6 @@ func (m *IndexMeta) GetTimeQuantum() string { } type FrameMeta struct { - RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` @@ -106,13 +105,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } -func (m *FrameMeta) GetRowLabel() string { - if m != nil { - return m.RowLabel - } - return "" -} - func (m *FrameMeta) GetInverseEnabled() bool { if m != nil { return m.InverseEnabled @@ -1278,12 +1270,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.RowLabel) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.RowLabel))) - i += copy(dAtA[i:], m.RowLabel) - } if m.InverseEnabled { dAtA[i] = 0x10 i++ @@ -2788,10 +2774,6 @@ func (m *IndexMeta) Size() (n int) { func (m *FrameMeta) Size() (n int) { var l int _ = l - l = len(m.RowLabel) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } if m.InverseEnabled { n += 2 } @@ -3583,35 +3565,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.RowLabel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) @@ -8667,89 +8620,88 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1334 bytes of a gzipped FileDescriptorProto + // 1326 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf6, 0x71, 0x9c, 0x38, 0xd3, 0x34, 0xaf, 0x13, 0x45, 0xae, 0x19, - 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0x34, 0x50, 0xa9, 0xc4, 0x76, 0xd5, 0x85, 0x26, 0x94, - 0x71, 0x12, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x24, 0xee, - 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x4f, 0x40, 0xe1, 0x47, 0x20, - 0x71, 0x03, 0x9a, 0xaf, 0xdd, 0xf5, 0x57, 0xd2, 0x04, 0xee, 0xf6, 0x3c, 0x73, 0xce, 0x99, 0x67, - 0xce, 0xd7, 0xcc, 0x42, 0xb9, 0x1f, 0xba, 0xa7, 0x94, 0xb3, 0xcd, 0x7e, 0x18, 0xf0, 0x00, 0x15, - 0x5c, 0x9f, 0xb3, 0xd0, 0xa7, 0x1e, 0xfe, 0x14, 0x8a, 0x8e, 0xdf, 0x65, 0xe7, 0xbb, 0x8c, 0x53, - 0x54, 0x87, 0x52, 0x23, 0xf0, 0x06, 0x3d, 0xff, 0x39, 0x3d, 0x62, 0x5e, 0xd5, 0xaa, 0x5b, 0x1b, - 0x45, 0x92, 0x86, 0x84, 0xc6, 0xbe, 0xdb, 0x63, 0x9f, 0x0d, 0xa8, 0xcf, 0x07, 0xbd, 0x6a, 0x46, - 0x69, 0xa4, 0x20, 0xfc, 0x97, 0x05, 0xc5, 0xa7, 0x21, 0xed, 0x31, 0xe9, 0x71, 0x0d, 0x0a, 0x24, - 0x38, 0x4b, 0xbb, 0x8b, 0x65, 0xf4, 0x16, 0x2c, 0x38, 0xfe, 0x29, 0x0b, 0x23, 0xd6, 0xf2, 0xe9, - 0x91, 0xc7, 0xba, 0xd2, 0x5d, 0x81, 0x8c, 0xa1, 0x68, 0x1d, 0x8a, 0x0d, 0xda, 0x79, 0xc9, 0xf6, - 0x87, 0x7d, 0x56, 0xb5, 0xa5, 0x93, 0x04, 0x88, 0x57, 0xdb, 0xee, 0x2b, 0x56, 0xcd, 0xd6, 0xad, - 0x8d, 0x32, 0x49, 0x80, 0x71, 0xbe, 0xb9, 0x09, 0xbe, 0x08, 0xc3, 0x3c, 0xa1, 0xfe, 0x71, 0xcc, - 0x21, 0x2f, 0x39, 0x8c, 0x60, 0xe8, 0x1e, 0xe4, 0x9f, 0xba, 0xcc, 0xeb, 0x46, 0xd5, 0xb9, 0xba, - 0xbd, 0x51, 0xda, 0x5a, 0xdc, 0x34, 0xf1, 0xdb, 0x94, 0x38, 0xd1, 0xcb, 0x18, 0xc3, 0x82, 0xd3, - 0xeb, 0x07, 0x21, 0x27, 0x2c, 0xea, 0x07, 0x7e, 0xc4, 0x50, 0x05, 0xec, 0x56, 0x18, 0xea, 0xb3, - 0x8b, 0x4f, 0xfc, 0x2d, 0x54, 0x76, 0xbc, 0xa0, 0x73, 0xd2, 0xa4, 0x9c, 0x12, 0xf6, 0xcd, 0x80, - 0x45, 0x1c, 0x2d, 0x43, 0x4e, 0x66, 0x41, 0xeb, 0x29, 0x41, 0xa0, 0x32, 0x92, 0x3a, 0xcc, 0x4a, - 0x10, 0xa8, 0xb4, 0x97, 0xa1, 0xc8, 0x12, 0x25, 0x08, 0xb4, 0xed, 0xb9, 0x1d, 0x15, 0x82, 0x2c, - 0x51, 0x02, 0x42, 0x90, 0x3d, 0x74, 0xd9, 0x99, 0x3e, 0xb7, 0xfc, 0xc6, 0x0e, 0x2c, 0xa5, 0xf6, - 0xd7, 0x34, 0x57, 0x20, 0x4f, 0x82, 0x33, 0xa7, 0x19, 0x55, 0xad, 0xba, 0xbd, 0x91, 0x25, 0x5a, - 0x92, 0xd1, 0x95, 0xe9, 0x17, 0x4b, 0x19, 0xb9, 0x94, 0x00, 0x78, 0x15, 0x72, 0x32, 0xd4, 0xe2, - 0x94, 0x89, 0xad, 0xf8, 0xc4, 0x7f, 0x5b, 0x50, 0xdc, 0xa5, 0xe7, 0x92, 0x46, 0x84, 0x1e, 0x43, - 0xa1, 0xcd, 0xa9, 0xdf, 0xa5, 0x61, 0x57, 0x2a, 0x95, 0xb6, 0xde, 0x48, 0x42, 0x18, 0xab, 0x6d, - 0x1a, 0x9d, 0x96, 0xcf, 0xc3, 0x21, 0x89, 0x4d, 0xd0, 0x36, 0xcc, 0xe9, 0x9a, 0x90, 0x1c, 0x4a, - 0x5b, 0xf5, 0x69, 0xd6, 0x71, 0xd9, 0x08, 0x63, 0x63, 0xb0, 0xf6, 0x01, 0x94, 0x47, 0xdc, 0x0a, - 0xae, 0x27, 0x6c, 0x68, 0x32, 0x72, 0xc2, 0x86, 0x22, 0x76, 0xa7, 0xd4, 0x1b, 0xa8, 0x38, 0x67, - 0x89, 0x12, 0xb6, 0x33, 0xef, 0x5b, 0x6b, 0xdb, 0x30, 0x9f, 0xf6, 0x7a, 0x1d, 0x5b, 0xfc, 0x15, - 0xa0, 0x46, 0xc8, 0x28, 0x67, 0x92, 0xde, 0x2e, 0x8b, 0x22, 0x7a, 0xcc, 0x66, 0x67, 0x5a, 0x65, - 0x2f, 0x93, 0xce, 0xde, 0x3a, 0x14, 0x9d, 0xc8, 0x1c, 0xdc, 0x96, 0x75, 0x99, 0x00, 0xf8, 0x3e, - 0xa0, 0x26, 0xf3, 0x18, 0x67, 0xba, 0x7f, 0x2f, 0xf1, 0x8f, 0xdb, 0x86, 0xcb, 0xd5, 0xba, 0xe8, - 0x1e, 0x64, 0x45, 0xeb, 0x4a, 0x2a, 0xa5, 0xad, 0x5b, 0x49, 0xa4, 0xe3, 0x39, 0x41, 0xa4, 0x02, - 0x76, 0x8d, 0x53, 0xdd, 0xee, 0x57, 0x1c, 0x70, 0x4a, 0x29, 0x9b, 0xad, 0xec, 0xf1, 0xad, 0xe2, - 0x01, 0xa2, 0xb7, 0x7a, 0x62, 0xce, 0x7a, 0xd3, 0xad, 0xf0, 0x71, 0x4c, 0x56, 0x74, 0xea, 0x4d, - 0xc8, 0xbe, 0x09, 0x39, 0x69, 0xab, 0xd9, 0x4e, 0xcc, 0x00, 0xb5, 0x8a, 0x0f, 0x63, 0xaa, 0x37, - 0xdd, 0x68, 0x39, 0xbd, 0x51, 0xd1, 0xf8, 0xfd, 0x42, 0xeb, 0x8a, 0x9e, 0xde, 0x13, 0x36, 0xca, - 0x93, 0xfc, 0x9e, 0x9d, 0xb3, 0xb1, 0x40, 0x0a, 0xdf, 0x62, 0x08, 0x44, 0x55, 0xbb, 0x6e, 0x0b, - 0xdf, 0x52, 0xc0, 0x0f, 0x21, 0xdf, 0xee, 0xbc, 0x64, 0x3d, 0x8a, 0xde, 0x16, 0x9d, 0xd6, 0x65, - 0xe7, 0x2c, 0xd2, 0x7d, 0xba, 0x38, 0x96, 0x7f, 0x62, 0xd6, 0xf1, 0x0f, 0x96, 0x3e, 0xd3, 0x0c, - 0x46, 0x79, 0xb9, 0x77, 0x54, 0xcd, 0x4e, 0x8c, 0x4c, 0x81, 0x13, 0xbd, 0x8c, 0x5a, 0x50, 0x71, - 0xfc, 0xfe, 0x80, 0x37, 0xd9, 0xd7, 0xae, 0xef, 0x72, 0x37, 0xf0, 0xa3, 0x6a, 0x5e, 0x9a, 0xac, - 0xa6, 0xb7, 0x1e, 0xd1, 0x20, 0x13, 0x26, 0xf8, 0x3b, 0x0b, 0x16, 0xc7, 0xc0, 0x2b, 0x78, 0x65, - 0x2e, 0xe7, 0xf5, 0x5e, 0x3c, 0xf3, 0x6d, 0xa9, 0x58, 0x9b, 0xc9, 0x66, 0xf4, 0x0a, 0xf8, 0xc5, - 0x82, 0xe5, 0x69, 0x0a, 0x53, 0xd9, 0xd4, 0x00, 0x5e, 0x84, 0x6e, 0x8f, 0x86, 0xc3, 0x4f, 0xd8, - 0x50, 0x5f, 0x7f, 0x29, 0x04, 0x7d, 0x0e, 0x2b, 0x63, 0xbe, 0x3e, 0xea, 0xa8, 0x10, 0x29, 0x52, - 0x77, 0x66, 0x92, 0x52, 0x7a, 0x64, 0x86, 0x39, 0xfe, 0xd3, 0x82, 0xdb, 0x53, 0x97, 0x92, 0x9a, - 0xb4, 0xd2, 0x35, 0x79, 0x1f, 0x2a, 0x87, 0x62, 0xb2, 0x35, 0x59, 0xc4, 0x5d, 0x9f, 0x0a, 0x4d, - 0x5d, 0xb4, 0x13, 0x38, 0x72, 0xa0, 0x20, 0xb1, 0x5d, 0xda, 0xd7, 0x34, 0xdf, 0xb9, 0x82, 0xe6, - 0xa6, 0xd1, 0xd7, 0x83, 0xdf, 0x88, 0x82, 0x8c, 0xbc, 0x88, 0xcc, 0xad, 0x26, 0x05, 0x31, 0xd2, - 0x47, 0x0c, 0xae, 0x35, 0x96, 0x03, 0x58, 0x37, 0xa3, 0x70, 0x84, 0xc9, 0xe5, 0x9d, 0xfa, 0x08, - 0x20, 0x51, 0xd5, 0x13, 0xe0, 0x92, 0xfa, 0x4c, 0x29, 0xe3, 0x67, 0xb0, 0x6e, 0xe6, 0xf4, 0x35, - 0x36, 0x34, 0xd5, 0x92, 0x49, 0xaa, 0x05, 0xb7, 0xc0, 0x3e, 0x20, 0x8e, 0xb8, 0xab, 0x65, 0xb7, - 0x9a, 0x14, 0x69, 0x49, 0x98, 0x3c, 0x0b, 0x22, 0x6e, 0x4c, 0xc4, 0xb7, 0xc0, 0x5e, 0x04, 0x21, - 0x97, 0x8c, 0xcb, 0x44, 0x7e, 0xe3, 0x2f, 0x21, 0xbb, 0x17, 0x74, 0x19, 0x5a, 0x80, 0x8c, 0xd3, - 0xd4, 0x3e, 0x32, 0x4e, 0x13, 0xdd, 0x91, 0xee, 0xf5, 0x0c, 0x29, 0x27, 0x87, 0x3b, 0x20, 0x0e, - 0x91, 0x1b, 0xdf, 0x85, 0xb2, 0x13, 0x35, 0x82, 0x20, 0xec, 0x8a, 0x54, 0x07, 0xa1, 0xbe, 0x93, - 0x46, 0x41, 0xfc, 0x04, 0x2a, 0xc2, 0x7d, 0x9b, 0x53, 0x1e, 0x4f, 0xea, 0x15, 0xc8, 0x0b, 0x2c, - 0xde, 0x4e, 0x4b, 0xf2, 0xde, 0x13, 0x7a, 0x66, 0x00, 0x4a, 0x01, 0x3f, 0x57, 0x1e, 0x5a, 0xa7, - 0xcc, 0xe7, 0xa9, 0x28, 0x49, 0x59, 0x3a, 0x28, 0x13, 0x25, 0x20, 0xac, 0x8e, 0xa2, 0x39, 0x2f, - 0x24, 0x9c, 0x05, 0x4a, 0xe4, 0x1a, 0xfe, 0xd1, 0x02, 0x30, 0x84, 0x06, 0x51, 0x6c, 0x62, 0xcd, - 0x36, 0x41, 0xef, 0xa6, 0xde, 0x2e, 0x93, 0x33, 0x35, 0x5e, 0x22, 0xa9, 0x17, 0xce, 0x86, 0x19, - 0xa1, 0xba, 0x38, 0x2a, 0x89, 0xbe, 0xc2, 0x75, 0x9a, 0xc4, 0xb5, 0x59, 0x6e, 0x78, 0x83, 0x88, - 0xb3, 0x50, 0x33, 0x12, 0x6f, 0x2c, 0x05, 0xc4, 0xf1, 0x49, 0x80, 0xe9, 0x21, 0x42, 0x77, 0x21, - 0x27, 0x98, 0x9a, 0x39, 0x30, 0x7e, 0x0c, 0xb5, 0x88, 0xdb, 0xfa, 0x26, 0x99, 0x3a, 0x7b, 0x10, - 0x64, 0xe5, 0x8b, 0x5a, 0x97, 0x8b, 0x7c, 0x4c, 0x57, 0xc0, 0xde, 0x75, 0x55, 0x7d, 0xdb, 0x44, - 0x7c, 0x4a, 0x84, 0x9e, 0xcb, 0xfe, 0x13, 0x08, 0x15, 0x6f, 0x89, 0x25, 0xd5, 0x40, 0xe2, 0xee, - 0xb8, 0xc9, 0xfd, 0x66, 0x1e, 0xa5, 0x76, 0xea, 0x51, 0xda, 0x86, 0x25, 0xd5, 0x24, 0xff, 0xa5, - 0xd3, 0x9f, 0x33, 0xb0, 0x44, 0x58, 0xe4, 0xbe, 0x62, 0x8e, 0x1f, 0xf1, 0x70, 0x10, 0x0f, 0xb8, - 0x8f, 0x83, 0x23, 0x1d, 0x6a, 0x9b, 0x28, 0xe1, 0x75, 0x2a, 0x09, 0x3d, 0x10, 0xbf, 0x47, 0xa3, - 0xd5, 0x3f, 0xa9, 0x9a, 0x56, 0x41, 0x0f, 0x60, 0xae, 0x1d, 0x0c, 0xc2, 0x4e, 0x7c, 0x0d, 0xae, - 0x24, 0xda, 0x8a, 0x99, 0x5a, 0x26, 0x46, 0x2d, 0x55, 0x47, 0xb9, 0xcb, 0xeb, 0x08, 0x3d, 0x1e, - 0xab, 0x23, 0xf9, 0xe7, 0x52, 0xda, 0xfa, 0x7f, 0x62, 0x30, 0xb2, 0x4c, 0x46, 0xb5, 0xf1, 0xf7, - 0x16, 0xcc, 0xa7, 0x29, 0xbc, 0x56, 0x63, 0xc4, 0x19, 0xc9, 0x4c, 0xcd, 0x88, 0x3d, 0x2d, 0x23, - 0xd9, 0x24, 0x23, 0xc9, 0x3b, 0x37, 0x97, 0x7a, 0xe7, 0xe2, 0x13, 0x58, 0x9d, 0x48, 0x53, 0x23, - 0xe8, 0xf5, 0x45, 0x3d, 0xfc, 0x8b, 0x74, 0x89, 0x91, 0x11, 0x86, 0x3a, 0x51, 0x45, 0xa2, 0x04, - 0xfc, 0x08, 0x6e, 0xb7, 0x19, 0x4f, 0x25, 0xc9, 0x54, 0x5b, 0x1d, 0xec, 0x3d, 0x76, 0x36, 0xe3, - 0xf8, 0x62, 0x09, 0x7f, 0x08, 0xd5, 0x83, 0x7e, 0x97, 0x72, 0x76, 0x23, 0xeb, 0x1d, 0x28, 0xec, - 0x07, 0xfd, 0xc0, 0x0b, 0x8e, 0x87, 0x57, 0xb4, 0x7c, 0x15, 0xe6, 0xd4, 0x7c, 0x54, 0x8f, 0x94, - 0x22, 0x31, 0x22, 0xbe, 0x25, 0x0a, 0xba, 0x43, 0xbd, 0xce, 0xc0, 0x13, 0x34, 0xc4, 0xbf, 0x57, - 0xb4, 0x53, 0xf9, 0xf5, 0xa2, 0x66, 0xfd, 0x76, 0x51, 0xb3, 0x7e, 0xbf, 0xa8, 0x59, 0x3f, 0xfd, - 0x51, 0xfb, 0xdf, 0x51, 0x5e, 0xfe, 0xe5, 0x3f, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x66, 0x19, - 0x3d, 0xd2, 0xf6, 0x0f, 0x00, 0x00, + 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf1, 0x71, 0x9d, 0x3a, 0xd3, 0x36, 0xaf, 0x5b, 0x45, 0xae, 0x19, + 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0xb4, 0x50, 0xa9, 0xd4, 0x76, 0xd5, 0x85, 0xa6, 0x94, + 0x71, 0x5b, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x34, 0xee, + 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x2f, 0x40, 0xa8, 0xfc, 0x08, + 0x2e, 0x41, 0x67, 0x3e, 0x76, 0xd7, 0x5f, 0x49, 0x1b, 0xb8, 0xdb, 0xf3, 0xcc, 0x39, 0x67, 0x9e, + 0x39, 0x5f, 0x33, 0x0b, 0xb5, 0x51, 0xec, 0x1f, 0x72, 0x29, 0xb6, 0x47, 0x71, 0x24, 0x23, 0xb2, + 0xea, 0x87, 0x52, 0xc4, 0x21, 0x0f, 0xe8, 0xa7, 0x50, 0xf1, 0xc2, 0x81, 0x38, 0xda, 0x15, 0x92, + 0x93, 0x16, 0x54, 0xdb, 0x51, 0x30, 0x1e, 0x86, 0x0f, 0xf8, 0x9e, 0x08, 0x1a, 0x4e, 0xcb, 0xd9, + 0xaa, 0xb0, 0x3c, 0x84, 0x1a, 0x8f, 0xfd, 0xa1, 0xf8, 0x6c, 0xcc, 0x43, 0x39, 0x1e, 0x36, 0x0a, + 0x5a, 0x23, 0x07, 0xd1, 0xdf, 0x1d, 0xa8, 0xdc, 0x8b, 0xf9, 0x50, 0x28, 0x8f, 0x6f, 0xc1, 0x9a, + 0x17, 0x1e, 0x8a, 0x38, 0x11, 0xdd, 0x90, 0xef, 0x05, 0x62, 0xa0, 0x4c, 0x56, 0xd9, 0x0c, 0x4a, + 0x36, 0xa1, 0xd2, 0xe6, 0xfd, 0x67, 0xe2, 0xf1, 0x64, 0x24, 0x1a, 0xae, 0xf2, 0x9a, 0x01, 0xe9, + 0x6a, 0xcf, 0x7f, 0x21, 0x1a, 0xc5, 0x96, 0xb3, 0x55, 0x63, 0x19, 0x30, 0xcb, 0xa9, 0x34, 0xc7, + 0x89, 0x50, 0x38, 0xc3, 0x78, 0xb8, 0x9f, 0x72, 0x28, 0x2b, 0x0e, 0x53, 0x18, 0xb9, 0x0a, 0xe5, + 0x7b, 0xbe, 0x08, 0x06, 0x49, 0x63, 0xa5, 0xe5, 0x6e, 0x55, 0x77, 0xce, 0x6e, 0xdb, 0x18, 0x6d, + 0x2b, 0x9c, 0x99, 0x65, 0x4a, 0x61, 0xcd, 0x1b, 0x8e, 0xa2, 0x58, 0x32, 0x91, 0x8c, 0xa2, 0x30, + 0x11, 0xa4, 0x0e, 0x6e, 0x37, 0x8e, 0x4d, 0xb8, 0xf0, 0x93, 0x7e, 0x0b, 0xf5, 0xbb, 0x41, 0xd4, + 0x3f, 0xe8, 0x70, 0xc9, 0x99, 0xf8, 0x66, 0x2c, 0x12, 0x49, 0xce, 0x43, 0x49, 0x45, 0xda, 0xe8, + 0x69, 0x01, 0x51, 0x15, 0x2d, 0x13, 0x4a, 0x2d, 0x20, 0xaa, 0xec, 0x55, 0x28, 0x8a, 0x4c, 0x0b, + 0x88, 0xf6, 0x02, 0xbf, 0xaf, 0x43, 0x50, 0x64, 0x5a, 0x20, 0x04, 0x8a, 0x4f, 0x7d, 0xf1, 0xdc, + 0x9c, 0x5b, 0x7d, 0x53, 0x0f, 0xd6, 0x73, 0xfb, 0x1b, 0x9a, 0x1b, 0x50, 0x66, 0xd1, 0x73, 0xaf, + 0x93, 0x34, 0x9c, 0x96, 0xbb, 0x55, 0x64, 0x46, 0x52, 0xd1, 0x55, 0x29, 0xc6, 0xa5, 0x82, 0x5a, + 0xca, 0x00, 0x7a, 0x11, 0x4a, 0x2a, 0xd4, 0x78, 0xca, 0xcc, 0x16, 0x3f, 0xe9, 0xdf, 0x0e, 0x54, + 0x76, 0xf9, 0x91, 0xa2, 0x91, 0x90, 0xdb, 0xb0, 0xda, 0x93, 0x3c, 0x1c, 0xf0, 0x78, 0xa0, 0x94, + 0xaa, 0x3b, 0x6f, 0x64, 0x21, 0x4c, 0xd5, 0xb6, 0xad, 0x4e, 0x37, 0x94, 0xf1, 0x84, 0xa5, 0x26, + 0xe4, 0x16, 0xac, 0x98, 0x9a, 0x50, 0x1c, 0xaa, 0x3b, 0xad, 0x45, 0xd6, 0x69, 0xd9, 0xa0, 0xb1, + 0x35, 0xb8, 0xf4, 0x01, 0xd4, 0xa6, 0xdc, 0x22, 0xd7, 0x03, 0x31, 0xb1, 0x19, 0x39, 0x10, 0x13, + 0x8c, 0xdd, 0x21, 0x0f, 0xc6, 0x3a, 0xce, 0x45, 0xa6, 0x85, 0x5b, 0x85, 0xf7, 0x9d, 0x4b, 0xb7, + 0xe0, 0x4c, 0xde, 0xeb, 0xeb, 0xd8, 0xd2, 0xaf, 0x80, 0xb4, 0x63, 0xc1, 0xa5, 0x50, 0xf4, 0x76, + 0x45, 0x92, 0xf0, 0x7d, 0xb1, 0x3c, 0xd3, 0x3a, 0x7b, 0x85, 0x7c, 0xf6, 0x36, 0xa1, 0xe2, 0x25, + 0xf6, 0xe0, 0xae, 0xaa, 0xcb, 0x0c, 0xa0, 0xd7, 0x80, 0x74, 0x44, 0x20, 0xa4, 0x30, 0x3d, 0x7a, + 0x8c, 0x7f, 0xda, 0xb3, 0x5c, 0x4e, 0xd6, 0x25, 0x57, 0xa1, 0x88, 0xed, 0xa9, 0xa8, 0x54, 0x77, + 0xce, 0x65, 0x91, 0x4e, 0x67, 0x01, 0x53, 0x0a, 0xd4, 0xb7, 0x4e, 0x4d, 0x4b, 0x9f, 0x70, 0xc0, + 0x05, 0xa5, 0x6c, 0xb7, 0x72, 0x67, 0xb7, 0x4a, 0x87, 0x84, 0xd9, 0xea, 0x8e, 0x3d, 0xeb, 0x69, + 0xb7, 0xa2, 0xfb, 0x29, 0x59, 0xec, 0xd4, 0xd3, 0x90, 0x7d, 0x13, 0x4a, 0xca, 0xd6, 0xb0, 0x9d, + 0x9b, 0x01, 0x7a, 0x95, 0x3e, 0x4d, 0xa9, 0x9e, 0x76, 0xa3, 0xf3, 0xf9, 0x8d, 0x2a, 0xd6, 0xef, + 0x17, 0x46, 0x17, 0x7b, 0xfa, 0x21, 0xda, 0x68, 0x4f, 0xea, 0x7b, 0x79, 0xce, 0x66, 0x02, 0x89, + 0xbe, 0x71, 0x08, 0x24, 0x0d, 0xb7, 0xe5, 0xa2, 0x6f, 0x25, 0xd0, 0x1b, 0x50, 0xee, 0xf5, 0x9f, + 0x89, 0x21, 0x27, 0x6f, 0x63, 0xa7, 0x0d, 0xc4, 0x91, 0x48, 0x4c, 0x9f, 0x9e, 0x9d, 0xc9, 0x3f, + 0xb3, 0xeb, 0xf4, 0x07, 0xc7, 0x9c, 0x69, 0x09, 0xa3, 0xb2, 0xda, 0x3b, 0x69, 0x14, 0xe7, 0x46, + 0x26, 0xe2, 0xcc, 0x2c, 0x93, 0x2e, 0xd4, 0xbd, 0x70, 0x34, 0x96, 0x1d, 0xf1, 0xb5, 0x1f, 0xfa, + 0xd2, 0x8f, 0xc2, 0xa4, 0x51, 0x56, 0x26, 0x17, 0xf3, 0x5b, 0x4f, 0x69, 0xb0, 0x39, 0x13, 0xfa, + 0x9d, 0x03, 0x67, 0x67, 0xc0, 0x13, 0x78, 0x15, 0x8e, 0xe7, 0xf5, 0x5e, 0x3a, 0xf3, 0x5d, 0xa5, + 0xd8, 0x5c, 0xca, 0x66, 0xfa, 0x0a, 0xf8, 0xc5, 0x81, 0xf3, 0x8b, 0x14, 0x16, 0xb2, 0x69, 0x02, + 0x3c, 0x8a, 0xfd, 0x21, 0x8f, 0x27, 0x9f, 0x88, 0x89, 0xb9, 0xfe, 0x72, 0x08, 0xf9, 0x1c, 0x36, + 0x66, 0x7c, 0x7d, 0xd4, 0xd7, 0x21, 0xd2, 0xa4, 0x2e, 0x2f, 0x25, 0xa5, 0xf5, 0xd8, 0x12, 0x73, + 0xfa, 0x97, 0x03, 0x17, 0x16, 0x2e, 0x65, 0x35, 0xe9, 0xe4, 0x6b, 0xf2, 0x1a, 0xd4, 0x9f, 0xe2, + 0x64, 0xeb, 0x88, 0x44, 0xfa, 0x21, 0x47, 0x4d, 0x53, 0xb4, 0x73, 0x38, 0xf1, 0x60, 0x55, 0x61, + 0xbb, 0x7c, 0x64, 0x68, 0xbe, 0x73, 0x02, 0xcd, 0x6d, 0xab, 0x6f, 0x06, 0xbf, 0x15, 0x91, 0x8c, + 0xba, 0x88, 0xec, 0xad, 0xa6, 0x04, 0x1c, 0xe9, 0x53, 0x06, 0xaf, 0x35, 0x96, 0x23, 0xd8, 0xb4, + 0xa3, 0x70, 0x8a, 0xc9, 0xf1, 0x9d, 0x7a, 0x13, 0x20, 0x53, 0x35, 0x13, 0xe0, 0x98, 0xfa, 0xcc, + 0x29, 0xd3, 0xfb, 0xb0, 0x69, 0xe7, 0xf4, 0x6b, 0x6c, 0x68, 0xab, 0xa5, 0x90, 0x55, 0x0b, 0xed, + 0x82, 0xfb, 0x84, 0x79, 0x78, 0x57, 0xab, 0x6e, 0xb5, 0x29, 0x32, 0x12, 0x9a, 0xdc, 0x8f, 0x12, + 0x69, 0x4d, 0xf0, 0x1b, 0xb1, 0x47, 0x51, 0x2c, 0x15, 0xe3, 0x1a, 0x53, 0xdf, 0xf4, 0x4b, 0x28, + 0x3e, 0x8c, 0x06, 0x82, 0xac, 0x41, 0xc1, 0xeb, 0x18, 0x1f, 0x05, 0xaf, 0x43, 0x2e, 0x2b, 0xf7, + 0x66, 0x86, 0xd4, 0xb2, 0xc3, 0x3d, 0x61, 0x1e, 0x53, 0x1b, 0x5f, 0x81, 0x9a, 0x97, 0xb4, 0xa3, + 0x28, 0x1e, 0x60, 0xaa, 0xa3, 0xd8, 0xdc, 0x49, 0xd3, 0x20, 0xbd, 0x03, 0x75, 0x74, 0xdf, 0x93, + 0x5c, 0xa6, 0x93, 0x7a, 0x03, 0xca, 0x88, 0xa5, 0xdb, 0x19, 0x49, 0xdd, 0x7b, 0xa8, 0x67, 0x07, + 0xa0, 0x12, 0xe8, 0x03, 0xed, 0xa1, 0x7b, 0x28, 0x42, 0x99, 0x8b, 0x92, 0x92, 0x95, 0x83, 0x1a, + 0xd3, 0x02, 0xa1, 0xfa, 0x28, 0x86, 0xf3, 0x5a, 0xc6, 0x19, 0x51, 0xa6, 0xd6, 0xe8, 0x8f, 0x0e, + 0x80, 0x25, 0x34, 0x4e, 0x52, 0x13, 0x67, 0xb9, 0x09, 0x79, 0x37, 0xf7, 0x76, 0x99, 0x9f, 0xa9, + 0xe9, 0x12, 0xcb, 0xbd, 0x70, 0xb6, 0xec, 0x08, 0x35, 0xc5, 0x51, 0xcf, 0xf4, 0x35, 0x6e, 0xd2, + 0x84, 0xd7, 0x66, 0xad, 0x1d, 0x8c, 0x13, 0x29, 0x62, 0xc3, 0x08, 0xdf, 0x58, 0x1a, 0x48, 0xe3, + 0x93, 0x01, 0x8b, 0x43, 0x44, 0xae, 0x40, 0x09, 0x99, 0xda, 0x39, 0x30, 0x7b, 0x0c, 0xbd, 0x48, + 0x7b, 0xe6, 0x26, 0x59, 0x38, 0x7b, 0x08, 0x14, 0xd5, 0x8b, 0xda, 0x94, 0x8b, 0x7a, 0x4c, 0xd7, + 0xc1, 0xdd, 0xf5, 0x75, 0x7d, 0xbb, 0x0c, 0x3f, 0x15, 0xc2, 0x8f, 0x54, 0xff, 0x21, 0xc2, 0xf1, + 0x2d, 0xb1, 0xae, 0x1b, 0x08, 0xef, 0x8e, 0xd3, 0xdc, 0x6f, 0xf6, 0x51, 0xea, 0xe6, 0x1e, 0xa5, + 0x3d, 0x58, 0xd7, 0x4d, 0xf2, 0x5f, 0x3a, 0xfd, 0xb9, 0x00, 0xeb, 0x4c, 0x24, 0xfe, 0x0b, 0xe1, + 0x85, 0x89, 0x8c, 0xc7, 0xe9, 0x80, 0xfb, 0x38, 0xda, 0x33, 0xa1, 0x76, 0x99, 0x16, 0x5e, 0xa5, + 0x92, 0xc8, 0x75, 0xfc, 0x05, 0x9a, 0xae, 0xfe, 0x79, 0xd5, 0xbc, 0x0a, 0xb9, 0x0e, 0x2b, 0xbd, + 0x68, 0x1c, 0xf7, 0xd3, 0x6b, 0x70, 0x23, 0xd3, 0xd6, 0xcc, 0xf4, 0x32, 0xb3, 0x6a, 0xb9, 0x3a, + 0x2a, 0x1d, 0x5f, 0x47, 0xe4, 0xf6, 0x4c, 0x1d, 0xa9, 0x3f, 0x97, 0xea, 0xce, 0xff, 0x33, 0x83, + 0xa9, 0x65, 0x36, 0xad, 0x4d, 0xbf, 0x77, 0xe0, 0x4c, 0x9e, 0xc2, 0x2b, 0x35, 0x46, 0x9a, 0x91, + 0xc2, 0xc2, 0x8c, 0xb8, 0x8b, 0x32, 0x52, 0xcc, 0x32, 0x92, 0xbd, 0x73, 0x4b, 0xb9, 0x77, 0x2e, + 0x3d, 0x80, 0x8b, 0x73, 0x69, 0x6a, 0x47, 0xc3, 0x11, 0xd6, 0xc3, 0xbf, 0x48, 0x17, 0x8e, 0x8c, + 0x38, 0x36, 0x89, 0xaa, 0x30, 0x2d, 0xd0, 0x9b, 0x70, 0xa1, 0x27, 0x64, 0x2e, 0x49, 0xb6, 0xda, + 0x5a, 0xe0, 0x3e, 0x14, 0xcf, 0x97, 0x1c, 0x1f, 0x97, 0xe8, 0x87, 0xd0, 0x78, 0x32, 0x1a, 0x70, + 0x29, 0x4e, 0x65, 0x7d, 0x17, 0x56, 0x1f, 0x47, 0xa3, 0x28, 0x88, 0xf6, 0x27, 0x27, 0xb4, 0x7c, + 0x03, 0x56, 0xf4, 0x7c, 0xd4, 0x8f, 0x94, 0x0a, 0xb3, 0x22, 0x3d, 0x87, 0x05, 0xdd, 0xe7, 0x41, + 0x7f, 0x1c, 0x20, 0x0d, 0xfc, 0xf7, 0x4a, 0xee, 0xd6, 0x7f, 0x7d, 0xd9, 0x74, 0x7e, 0x7b, 0xd9, + 0x74, 0xfe, 0x78, 0xd9, 0x74, 0x7e, 0xfa, 0xb3, 0xf9, 0xbf, 0xbd, 0xb2, 0xfa, 0x93, 0xbf, 0xf1, + 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x13, 0x1d, 0x50, 0xda, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 8642d9a13..ca987ec51 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -8,7 +8,6 @@ message IndexMeta { } message FrameMeta { - string RowLabel = 1; bool InverseEnabled = 2; string CacheType = 3; uint32 CacheSize = 4; diff --git a/pilosa.go b/pilosa.go index 98c67203f..71b63fcaa 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,7 +36,6 @@ var ( ErrFrameExists = errors.New("frame already exists") ErrFrameNotFound = errors.New("frame not found") ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") diff --git a/server/cluster_test.go b/server/cluster_test.go index fbb784691..de623bbd4 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -153,8 +153,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", columnID=1) + SetBit(row=1, frame="f", columnID=2400000) `); err != nil { t.Fatal(err) } @@ -351,8 +351,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=1300000) + SetBit(row=1, frame="f", columnID=1) + SetBit(row=1, frame="f", columnID=1300000) `); err != nil { t.Fatal(err) } @@ -406,8 +406,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", columnID=1) + SetBit(row=1, frame="f", columnID=2400000) `); err != nil { t.Fatal(err) } @@ -560,7 +560,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setBits := "" for i := 0; i < 20; i++ { - setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) + setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setBits); err != nil { diff --git a/server/server_test.go b/server/server_test.go index 3852845a6..28155bab4 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -56,7 +56,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(rowID=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -72,7 +72,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -95,7 +95,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -131,36 +131,36 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set bits on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="x", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", columnID=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(rowID=1, frame="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -171,19 +171,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -204,9 +204,9 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=101)`); err != nil { t.Fatal(err) } @@ -216,7 +216,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -227,7 +227,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -248,9 +248,9 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { t.Fatal(err) } @@ -260,7 +260,7 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -285,19 +285,19 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m10.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=100) - SetBit(rowID=1, frame="f", columnID=1000) - SetBit(rowID=1, frame="f", columnID=100000) - SetBit(rowID=1, frame="f", columnID=200000) - SetBit(rowID=1, frame="f", columnID=400000) - SetBit(rowID=1, frame="f", columnID=600000) - SetBit(rowID=1, frame="f", columnID=800000) + SetBit(row=1, frame="f", columnID=100) + SetBit(row=1, frame="f", columnID=1000) + SetBit(row=1, frame="f", columnID=100000) + SetBit(row=1, frame="f", columnID=200000) + SetBit(row=1, frame="f", columnID=400000) + SetBit(row=1, frame="f", columnID=600000) + SetBit(row=1, frame="f", columnID=800000) `); err != nil { t.Fatal("setting bits:", err) } // Query row on first cluster. - if res, err := m10.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -335,7 +335,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Query row on second cluster. - if res, err := m20.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("another bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("2unexpected result: %s", res) @@ -401,7 +401,7 @@ func TestMain_RecalculateHashes(t *testing.T) { data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(rowID=%d, frame="f", columnID=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", columnID=%d)`, rowID, columnID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { diff --git a/stats_test.go b/stats_test.go index 93d7d9bb1..a03407a3d 100644 --- a/stats_test.go +++ b/stats_test.go @@ -143,7 +143,7 @@ func TestStatsCount_Bitmap(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -174,7 +174,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { From b2b0fd081afd96b5c4fdd9e23d7f7a20bf4f74ec Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 26 Mar 2018 15:35:56 -0500 Subject: [PATCH 190/311] Remove ColumnLabel support --- docs/examples.md | 11 +- executor.go | 70 ++++--------- executor_test.go | 120 +++++++++++----------- fragment.go | 4 +- frame.go | 2 - handler_internal_test.go | 3 +- handler_test.go | 4 +- holder.go | 1 - holder_test.go | 2 +- index.go | 46 +-------- input_definition.go | 2 +- internal/private.pb.go | 214 +++++++++++++++------------------------ internal/private.proto | 1 - pilosa.go | 1 - server.go | 1 - server/cluster_test.go | 16 +-- server/server_test.go | 66 +++--------- stats_test.go | 2 +- 18 files changed, 201 insertions(+), 365 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index cf9885a3f..225e0f07c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -267,12 +267,11 @@ First, follow the instruction in the [getting started](../getting-started/) guid The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 ``` curl localhost:10101/index/mole \ - -X POST \ - -d '{"options": {"columnLabel": "position_id"}}' + -X POST curl localhost:10101/index/mole/frame/fingerprint \ -X POST \ - -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' + -d '{"options": {"inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' ``` @@ -302,7 +301,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a * Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id ```python - bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] + bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] bitmap_string = ', '.join(bit_maps) intersection = "Intersect(%s)" % bitmap_string mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] @@ -312,7 +311,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a ```python for m in mole_ids: - mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] + mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(row=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] existed_mol = False if len(mol) == len(fp): found = m @@ -331,7 +330,7 @@ Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN * Query Pilosa’s TopN to get list of similarity chembl_id ```python - query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' + query_string = 'TopN(Bitmap(row=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) ``` diff --git a/executor.go b/executor.go index de0d65df0..0ed6dbaed 100644 --- a/executor.go +++ b/executor.go @@ -80,7 +80,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // MaxSlice can differ between inverse and standard views, so we need // to send queries to different slices based on orientation. var inverseSlices []uint64 - columnLabel := DefaultColumnLabel // If slices aren't specified, then include all of them. if len(slices) == 0 { @@ -105,9 +104,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic for i := range inverseSlices { inverseSlices[i] = uint64(i) } - - // Fetch column label from index. - columnLabel = idx.ColumnLabel() } } @@ -132,7 +128,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(DefaultRowLabel, columnLabel) { + if call.IsInverse(DefaultRowLabel, DefaultColumnLabel) { slices = inverseSlices } } @@ -266,8 +262,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { idx := e.Holder.Index(index) if idx != nil { - columnLabel := idx.ColumnLabel() - if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { + if columnID, ok, err := c.UintArg(DefaultColumnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err @@ -522,7 +517,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if idx == nil { return nil, ErrIndexNotFound } - columnLabel := idx.ColumnLabel() // Fetch frame & row label based on argument. frame, _ := c.Args["frame"].(string) @@ -536,14 +530,14 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. // Return an error if both the row and column label are specified. rowID, rowOK, rowErr := c.UintArg(DefaultRowLabel) - columnID, columnOK, columnErr := c.UintArg(columnLabel) + columnID, columnOK, columnErr := c.UintArg(DefaultColumnLabel) if rowErr != nil || columnErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) } if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, DefaultColumnLabel) } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, DefaultColumnLabel) } // Determine row or column orientation. @@ -602,7 +596,6 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if idx == nil { return nil, ErrIndexNotFound } - columnLabel := idx.ColumnLabel() // Retrieve base frame. f := idx.Frame(frame) @@ -611,7 +604,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Read row & column id. - columnID, columnOK, err := c.UintArg(columnLabel) + columnID, columnOK, err := c.UintArg(DefaultColumnLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) } @@ -624,9 +617,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C var id uint64 var viewName string if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() cannot contain both %q and %q", DefaultColumnLabel, DefaultRowLabel) } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() must specify either %q or %q", DefaultColumnLabel, DefaultRowLabel) } else if columnOK { viewName, id = ViewInverse, columnID } else { @@ -899,9 +892,6 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, ErrFrameNotFound } - // Retrieve labels. - columnLabel := idx.ColumnLabel() - // Read fields using labels. rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { @@ -910,11 +900,11 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("ClearBit() row field '%v' required", DefaultRowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg(DefaultColumnLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) + return false, fmt.Errorf("ClearBit col field '%v' required", DefaultColumnLabel) } // Clear bits for each view. @@ -992,9 +982,6 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, ErrFrameNotFound } - // Retrieve labels. - columnLabel := idx.ColumnLabel() - // Read fields using labels. rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { @@ -1003,11 +990,11 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, fmt.Errorf("SetBit() row field '%v' required", DefaultRowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg(DefaultColumnLabel) if err != nil { return false, fmt.Errorf("reading SetBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) + return false, fmt.Errorf("SetBit() column field '%v' required", DefaultColumnLabel) } var timestamp *time.Time @@ -1086,13 +1073,6 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq return errors.New("SetFieldValue() frame required") } - // Retrieve column label. - idx := e.Holder.Index(index) - if idx == nil { - return ErrIndexNotFound - } - columnLabel := idx.ColumnLabel() - // Retrieve frame. frame := e.Holder.Frame(index, frameName) if frame == nil { @@ -1100,11 +1080,11 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Parse labels. - columnID, ok, err := c.UintArg(columnLabel) + columnID, ok, err := c.UintArg(DefaultColumnLabel) if err != nil { return fmt.Errorf("reading SetFieldValue() column: %v", err) } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) + return fmt.Errorf("SetFieldValue() column field '%v' required", DefaultColumnLabel) } // Copy args and remove reserved fields. @@ -1112,7 +1092,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq delete(args, "frame") // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. - delete(args, columnLabel) + delete(args, DefaultColumnLabel) // Set values. for name, value := range args { @@ -1304,28 +1284,18 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return ErrIndexNotFound } - var colName string - id, okID, errID := c.UintArg("id") - if errID != nil || !okID { - // Retrieve columnLabel - columnLabel := idx.columnLabel - col, okCol, errCol := c.UintArg(columnLabel) - if errCol != nil || !okCol { - return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) - } - id = col - colName = columnLabel - } else { - colName = "id" + col, okCol, errCol := c.UintArg(DefaultColumnLabel) + if errCol != nil || !okCol { + return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, colName) + delete(attrs, DefaultColumnLabel) delete(attrs, "frame") // Set attributes. - if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil { + if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil { return err } idx.Stats.Count("SetProfileAttrs", 1, 1.0) diff --git a/executor_test.go b/executor_test.go index d90c8048f..1a23eba14 100644 --- a/executor_test.go +++ b/executor_test.go @@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -293,9 +293,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Set field values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=100, frame=f, field0=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil { t.Fatal(err) } @@ -340,28 +340,28 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Run("ErrFrameRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { t.Fatalf("unexpected error: %s", err) } }) @@ -419,15 +419,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, columnID=0) - SetBit(frame=f, row=0, columnID=1) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, row=10, columnID=0) - SetBit(frame=f, row=10, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=20, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, row=0, columnID=0) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=1) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, row=10, col=0) + SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, row=0, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -630,15 +630,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, columnID=0) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) - SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, columnID=0) + SetFieldValue(frame=f, foo=20, bar=2000, col=0) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -679,16 +679,16 @@ func TestExecutor_Execute_Range(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, row=1, columnID=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, row=1, columnID=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, row=1, columnID=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, row=1, columnID=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, row=1, columnID=7, timestamp="2002-01-01T02:00") + SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") + SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") + SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00") + SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00") + SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, row=1, columnID=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, row=10, columnID=2, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00") + SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00") + SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } @@ -703,7 +703,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Run("Inverse", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { t.Fatalf("unexpected bits: %+v", bits) @@ -751,17 +751,17 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, columnID=0) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) - SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, columnID=0) - SetFieldValue(frame=edge, foo=100, columnID=0) - SetFieldValue(frame=edge, foo=-100, columnID=1) + SetFieldValue(frame=f, foo=20, bar=2000, col=50) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, col=0) + SetFieldValue(frame=edge, foo=100, col=0) + SetFieldValue(frame=edge, foo=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } @@ -994,7 +994,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", row=10)` { + } else if query.String() != `SetBit(col=2, frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1012,7 +1012,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -1046,7 +1046,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1066,7 +1066,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -1169,11 +1169,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=10)"), nil, nil) + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', columnID=10, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } @@ -1186,11 +1186,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=20)"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(columnID=20, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index 8b32182d6..093b14d34 100644 --- a/fragment.go +++ b/fragment.go @@ -1841,11 +1841,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/frame.go b/frame.go index 060125385..f786ac0b0 100644 --- a/frame.go +++ b/frame.go @@ -57,7 +57,6 @@ type Frame struct { Stats StatsClient // Frame options. - rowLabel string inverseEnabled bool cacheType string cacheSize uint32 @@ -267,7 +266,6 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.rowLabel = DefaultRowLabel f.inverseEnabled = DefaultInverseEnabled f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize diff --git a/handler_internal_test.go b/handler_internal_test.go index fb6226ea9..395ec73b6 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -30,8 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}}, - {json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"}, + {json: `{"options": {"columnLabel": "test"}}`, err: "Unknown key: columnLabel:test"}, } for _, test := range tests { actual := &postIndexRequest{} diff --git a/handler_test.go b/handler_test.go index 3c9c08094..5e9b993f9 100644 --- a/handler_test.go +++ b/handler_test.go @@ -107,7 +107,7 @@ func TestHandler_Schema(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -1349,7 +1349,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { } // Ensure throwing error if there's no primary key - hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) unmatchColumnBody := []byte(` { "frames":[{ diff --git a/holder.go b/holder.go index 63bbbbc86..f418a145d 100644 --- a/holder.go +++ b/holder.go @@ -361,7 +361,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } // Update options. - index.SetColumnLabel(opt.ColumnLabel) index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index diff --git a/holder_test.go b/holder_test.go index 5a06a9272..267a10372 100644 --- a/holder_test.go +++ b/holder_test.go @@ -68,7 +68,7 @@ func TestHolder_Open(t *testing.T) { h := test.MustOpenHolder() defer h.Close() - if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) diff --git a/index.go b/index.go index 968bac847..f17854e1e 100644 --- a/index.go +++ b/index.go @@ -31,7 +31,7 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "columnID" + DefaultColumnLabel = "col" InputDefinitionDir = ".input-definitions" ) @@ -45,9 +45,6 @@ type Index struct { // This can be overridden by individual frames. timeQuantum TimeQuantum - // Label used for referring to columns in index. - columnLabel string - // Frames by name. frames map[string]*Frame @@ -88,8 +85,6 @@ func NewIndex(path, name string) (*Index, error) { NewAttrStore: NewNopAttrStore, columnAttrStore: NopAttrStore, - columnLabel: DefaultColumnLabel, - broadcaster: NopBroadcaster, Stats: NopStatsClient, LogOutput: ioutil.Discard, @@ -105,39 +100,6 @@ func (i *Index) Path() string { return i.path } // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } -// SetColumnLabel sets the column label. Persists to meta file on update. -func (i *Index) SetColumnLabel(v string) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || i.columnLabel == v { - return nil - } - - // Make sure columnLabel is valid name - err := ValidateLabel(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - i.columnLabel = v - if err := i.saveMeta(); err != nil { - return err - } - - return nil -} - -// ColumnLabel returns the column label. -func (i *Index) ColumnLabel() string { - i.mu.RLock() - v := i.columnLabel - i.mu.RUnlock() - return v -} - // Options returns all options for this index. func (i *Index) Options() IndexOptions { i.mu.RLock() @@ -147,7 +109,6 @@ func (i *Index) Options() IndexOptions { func (i *Index) options() IndexOptions { return IndexOptions{ - ColumnLabel: i.columnLabel, TimeQuantum: i.timeQuantum, } } @@ -217,7 +178,6 @@ func (i *Index) loadMeta() error { buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta")) if os.IsNotExist(err) { i.timeQuantum = "" - i.columnLabel = DefaultColumnLabel return nil } else if err != nil { return err @@ -229,7 +189,6 @@ func (i *Index) loadMeta() error { // Copy metadata fields. i.timeQuantum = TimeQuantum(pb.TimeQuantum) - i.columnLabel = pb.ColumnLabel return nil } @@ -239,7 +198,6 @@ func (i *Index) saveMeta() error { // Marshal metadata. buf, err := proto.Marshal(&internal.IndexMeta{ TimeQuantum: string(i.timeQuantum), - ColumnLabel: i.columnLabel, }) if err != nil { return err @@ -631,14 +589,12 @@ func encodeIndex(d *Index) *internal.Index { // IndexOptions represents options to set when initializing an index. type IndexOptions struct { - ColumnLabel string `json:"columnLabel,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { return &internal.IndexMeta{ - ColumnLabel: i.ColumnLabel, TimeQuantum: string(i.TimeQuantum), } } diff --git a/input_definition.go b/input_definition.go index f4571d0bb..b18b4980e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -281,7 +281,7 @@ func (i *InputDefinitionInfo) Validate() error { } } - // Validate columnLabel and duplicate primaryKey. + // Validate duplicate primaryKey. for _, field := range i.Fields { if field.Name == "" { return ErrInputDefinitionNameRequired diff --git a/internal/private.pb.go b/internal/private.pb.go index 488aca0fe..0d0f3124a 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -68,7 +68,6 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } @@ -77,13 +76,6 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -func (m *IndexMeta) GetColumnLabel() string { - if m != nil { - return m.ColumnLabel - } - return "" -} - func (m *IndexMeta) GetTimeQuantum() string { if m != nil { return m.TimeQuantum @@ -1240,12 +1232,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.ColumnLabel) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ColumnLabel))) - i += copy(dAtA[i:], m.ColumnLabel) - } if len(m.TimeQuantum) > 0 { dAtA[i] = 0x12 i++ @@ -2760,10 +2746,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l - l = len(m.ColumnLabel) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } l = len(m.TimeQuantum) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -3457,35 +3439,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ColumnLabel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) @@ -8620,88 +8573,87 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1326 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf1, 0x71, 0x9d, 0x3a, 0xd3, 0x36, 0xaf, 0x5b, 0x45, 0xae, 0x19, - 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0xb4, 0x50, 0xa9, 0xd4, 0x76, 0xd5, 0x85, 0xa6, 0x94, - 0x71, 0x5b, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x34, 0xee, - 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x2f, 0x40, 0xa8, 0xfc, 0x08, - 0x2e, 0x41, 0x67, 0x3e, 0x76, 0xd7, 0x5f, 0x49, 0x1b, 0xb8, 0xdb, 0xf3, 0xcc, 0x39, 0x67, 0x9e, - 0x39, 0x5f, 0x33, 0x0b, 0xb5, 0x51, 0xec, 0x1f, 0x72, 0x29, 0xb6, 0x47, 0x71, 0x24, 0x23, 0xb2, - 0xea, 0x87, 0x52, 0xc4, 0x21, 0x0f, 0xe8, 0xa7, 0x50, 0xf1, 0xc2, 0x81, 0x38, 0xda, 0x15, 0x92, - 0x93, 0x16, 0x54, 0xdb, 0x51, 0x30, 0x1e, 0x86, 0x0f, 0xf8, 0x9e, 0x08, 0x1a, 0x4e, 0xcb, 0xd9, - 0xaa, 0xb0, 0x3c, 0x84, 0x1a, 0x8f, 0xfd, 0xa1, 0xf8, 0x6c, 0xcc, 0x43, 0x39, 0x1e, 0x36, 0x0a, - 0x5a, 0x23, 0x07, 0xd1, 0xdf, 0x1d, 0xa8, 0xdc, 0x8b, 0xf9, 0x50, 0x28, 0x8f, 0x6f, 0xc1, 0x9a, - 0x17, 0x1e, 0x8a, 0x38, 0x11, 0xdd, 0x90, 0xef, 0x05, 0x62, 0xa0, 0x4c, 0x56, 0xd9, 0x0c, 0x4a, - 0x36, 0xa1, 0xd2, 0xe6, 0xfd, 0x67, 0xe2, 0xf1, 0x64, 0x24, 0x1a, 0xae, 0xf2, 0x9a, 0x01, 0xe9, - 0x6a, 0xcf, 0x7f, 0x21, 0x1a, 0xc5, 0x96, 0xb3, 0x55, 0x63, 0x19, 0x30, 0xcb, 0xa9, 0x34, 0xc7, - 0x89, 0x50, 0x38, 0xc3, 0x78, 0xb8, 0x9f, 0x72, 0x28, 0x2b, 0x0e, 0x53, 0x18, 0xb9, 0x0a, 0xe5, - 0x7b, 0xbe, 0x08, 0x06, 0x49, 0x63, 0xa5, 0xe5, 0x6e, 0x55, 0x77, 0xce, 0x6e, 0xdb, 0x18, 0x6d, - 0x2b, 0x9c, 0x99, 0x65, 0x4a, 0x61, 0xcd, 0x1b, 0x8e, 0xa2, 0x58, 0x32, 0x91, 0x8c, 0xa2, 0x30, - 0x11, 0xa4, 0x0e, 0x6e, 0x37, 0x8e, 0x4d, 0xb8, 0xf0, 0x93, 0x7e, 0x0b, 0xf5, 0xbb, 0x41, 0xd4, - 0x3f, 0xe8, 0x70, 0xc9, 0x99, 0xf8, 0x66, 0x2c, 0x12, 0x49, 0xce, 0x43, 0x49, 0x45, 0xda, 0xe8, - 0x69, 0x01, 0x51, 0x15, 0x2d, 0x13, 0x4a, 0x2d, 0x20, 0xaa, 0xec, 0x55, 0x28, 0x8a, 0x4c, 0x0b, - 0x88, 0xf6, 0x02, 0xbf, 0xaf, 0x43, 0x50, 0x64, 0x5a, 0x20, 0x04, 0x8a, 0x4f, 0x7d, 0xf1, 0xdc, - 0x9c, 0x5b, 0x7d, 0x53, 0x0f, 0xd6, 0x73, 0xfb, 0x1b, 0x9a, 0x1b, 0x50, 0x66, 0xd1, 0x73, 0xaf, - 0x93, 0x34, 0x9c, 0x96, 0xbb, 0x55, 0x64, 0x46, 0x52, 0xd1, 0x55, 0x29, 0xc6, 0xa5, 0x82, 0x5a, - 0xca, 0x00, 0x7a, 0x11, 0x4a, 0x2a, 0xd4, 0x78, 0xca, 0xcc, 0x16, 0x3f, 0xe9, 0xdf, 0x0e, 0x54, - 0x76, 0xf9, 0x91, 0xa2, 0x91, 0x90, 0xdb, 0xb0, 0xda, 0x93, 0x3c, 0x1c, 0xf0, 0x78, 0xa0, 0x94, - 0xaa, 0x3b, 0x6f, 0x64, 0x21, 0x4c, 0xd5, 0xb6, 0xad, 0x4e, 0x37, 0x94, 0xf1, 0x84, 0xa5, 0x26, - 0xe4, 0x16, 0xac, 0x98, 0x9a, 0x50, 0x1c, 0xaa, 0x3b, 0xad, 0x45, 0xd6, 0x69, 0xd9, 0xa0, 0xb1, - 0x35, 0xb8, 0xf4, 0x01, 0xd4, 0xa6, 0xdc, 0x22, 0xd7, 0x03, 0x31, 0xb1, 0x19, 0x39, 0x10, 0x13, - 0x8c, 0xdd, 0x21, 0x0f, 0xc6, 0x3a, 0xce, 0x45, 0xa6, 0x85, 0x5b, 0x85, 0xf7, 0x9d, 0x4b, 0xb7, - 0xe0, 0x4c, 0xde, 0xeb, 0xeb, 0xd8, 0xd2, 0xaf, 0x80, 0xb4, 0x63, 0xc1, 0xa5, 0x50, 0xf4, 0x76, - 0x45, 0x92, 0xf0, 0x7d, 0xb1, 0x3c, 0xd3, 0x3a, 0x7b, 0x85, 0x7c, 0xf6, 0x36, 0xa1, 0xe2, 0x25, - 0xf6, 0xe0, 0xae, 0xaa, 0xcb, 0x0c, 0xa0, 0xd7, 0x80, 0x74, 0x44, 0x20, 0xa4, 0x30, 0x3d, 0x7a, - 0x8c, 0x7f, 0xda, 0xb3, 0x5c, 0x4e, 0xd6, 0x25, 0x57, 0xa1, 0x88, 0xed, 0xa9, 0xa8, 0x54, 0x77, - 0xce, 0x65, 0x91, 0x4e, 0x67, 0x01, 0x53, 0x0a, 0xd4, 0xb7, 0x4e, 0x4d, 0x4b, 0x9f, 0x70, 0xc0, - 0x05, 0xa5, 0x6c, 0xb7, 0x72, 0x67, 0xb7, 0x4a, 0x87, 0x84, 0xd9, 0xea, 0x8e, 0x3d, 0xeb, 0x69, - 0xb7, 0xa2, 0xfb, 0x29, 0x59, 0xec, 0xd4, 0xd3, 0x90, 0x7d, 0x13, 0x4a, 0xca, 0xd6, 0xb0, 0x9d, - 0x9b, 0x01, 0x7a, 0x95, 0x3e, 0x4d, 0xa9, 0x9e, 0x76, 0xa3, 0xf3, 0xf9, 0x8d, 0x2a, 0xd6, 0xef, - 0x17, 0x46, 0x17, 0x7b, 0xfa, 0x21, 0xda, 0x68, 0x4f, 0xea, 0x7b, 0x79, 0xce, 0x66, 0x02, 0x89, - 0xbe, 0x71, 0x08, 0x24, 0x0d, 0xb7, 0xe5, 0xa2, 0x6f, 0x25, 0xd0, 0x1b, 0x50, 0xee, 0xf5, 0x9f, - 0x89, 0x21, 0x27, 0x6f, 0x63, 0xa7, 0x0d, 0xc4, 0x91, 0x48, 0x4c, 0x9f, 0x9e, 0x9d, 0xc9, 0x3f, - 0xb3, 0xeb, 0xf4, 0x07, 0xc7, 0x9c, 0x69, 0x09, 0xa3, 0xb2, 0xda, 0x3b, 0x69, 0x14, 0xe7, 0x46, - 0x26, 0xe2, 0xcc, 0x2c, 0x93, 0x2e, 0xd4, 0xbd, 0x70, 0x34, 0x96, 0x1d, 0xf1, 0xb5, 0x1f, 0xfa, - 0xd2, 0x8f, 0xc2, 0xa4, 0x51, 0x56, 0x26, 0x17, 0xf3, 0x5b, 0x4f, 0x69, 0xb0, 0x39, 0x13, 0xfa, - 0x9d, 0x03, 0x67, 0x67, 0xc0, 0x13, 0x78, 0x15, 0x8e, 0xe7, 0xf5, 0x5e, 0x3a, 0xf3, 0x5d, 0xa5, - 0xd8, 0x5c, 0xca, 0x66, 0xfa, 0x0a, 0xf8, 0xc5, 0x81, 0xf3, 0x8b, 0x14, 0x16, 0xb2, 0x69, 0x02, - 0x3c, 0x8a, 0xfd, 0x21, 0x8f, 0x27, 0x9f, 0x88, 0x89, 0xb9, 0xfe, 0x72, 0x08, 0xf9, 0x1c, 0x36, - 0x66, 0x7c, 0x7d, 0xd4, 0xd7, 0x21, 0xd2, 0xa4, 0x2e, 0x2f, 0x25, 0xa5, 0xf5, 0xd8, 0x12, 0x73, - 0xfa, 0x97, 0x03, 0x17, 0x16, 0x2e, 0x65, 0x35, 0xe9, 0xe4, 0x6b, 0xf2, 0x1a, 0xd4, 0x9f, 0xe2, - 0x64, 0xeb, 0x88, 0x44, 0xfa, 0x21, 0x47, 0x4d, 0x53, 0xb4, 0x73, 0x38, 0xf1, 0x60, 0x55, 0x61, - 0xbb, 0x7c, 0x64, 0x68, 0xbe, 0x73, 0x02, 0xcd, 0x6d, 0xab, 0x6f, 0x06, 0xbf, 0x15, 0x91, 0x8c, - 0xba, 0x88, 0xec, 0xad, 0xa6, 0x04, 0x1c, 0xe9, 0x53, 0x06, 0xaf, 0x35, 0x96, 0x23, 0xd8, 0xb4, - 0xa3, 0x70, 0x8a, 0xc9, 0xf1, 0x9d, 0x7a, 0x13, 0x20, 0x53, 0x35, 0x13, 0xe0, 0x98, 0xfa, 0xcc, - 0x29, 0xd3, 0xfb, 0xb0, 0x69, 0xe7, 0xf4, 0x6b, 0x6c, 0x68, 0xab, 0xa5, 0x90, 0x55, 0x0b, 0xed, - 0x82, 0xfb, 0x84, 0x79, 0x78, 0x57, 0xab, 0x6e, 0xb5, 0x29, 0x32, 0x12, 0x9a, 0xdc, 0x8f, 0x12, - 0x69, 0x4d, 0xf0, 0x1b, 0xb1, 0x47, 0x51, 0x2c, 0x15, 0xe3, 0x1a, 0x53, 0xdf, 0xf4, 0x4b, 0x28, - 0x3e, 0x8c, 0x06, 0x82, 0xac, 0x41, 0xc1, 0xeb, 0x18, 0x1f, 0x05, 0xaf, 0x43, 0x2e, 0x2b, 0xf7, - 0x66, 0x86, 0xd4, 0xb2, 0xc3, 0x3d, 0x61, 0x1e, 0x53, 0x1b, 0x5f, 0x81, 0x9a, 0x97, 0xb4, 0xa3, - 0x28, 0x1e, 0x60, 0xaa, 0xa3, 0xd8, 0xdc, 0x49, 0xd3, 0x20, 0xbd, 0x03, 0x75, 0x74, 0xdf, 0x93, - 0x5c, 0xa6, 0x93, 0x7a, 0x03, 0xca, 0x88, 0xa5, 0xdb, 0x19, 0x49, 0xdd, 0x7b, 0xa8, 0x67, 0x07, - 0xa0, 0x12, 0xe8, 0x03, 0xed, 0xa1, 0x7b, 0x28, 0x42, 0x99, 0x8b, 0x92, 0x92, 0x95, 0x83, 0x1a, - 0xd3, 0x02, 0xa1, 0xfa, 0x28, 0x86, 0xf3, 0x5a, 0xc6, 0x19, 0x51, 0xa6, 0xd6, 0xe8, 0x8f, 0x0e, - 0x80, 0x25, 0x34, 0x4e, 0x52, 0x13, 0x67, 0xb9, 0x09, 0x79, 0x37, 0xf7, 0x76, 0x99, 0x9f, 0xa9, - 0xe9, 0x12, 0xcb, 0xbd, 0x70, 0xb6, 0xec, 0x08, 0x35, 0xc5, 0x51, 0xcf, 0xf4, 0x35, 0x6e, 0xd2, - 0x84, 0xd7, 0x66, 0xad, 0x1d, 0x8c, 0x13, 0x29, 0x62, 0xc3, 0x08, 0xdf, 0x58, 0x1a, 0x48, 0xe3, - 0x93, 0x01, 0x8b, 0x43, 0x44, 0xae, 0x40, 0x09, 0x99, 0xda, 0x39, 0x30, 0x7b, 0x0c, 0xbd, 0x48, - 0x7b, 0xe6, 0x26, 0x59, 0x38, 0x7b, 0x08, 0x14, 0xd5, 0x8b, 0xda, 0x94, 0x8b, 0x7a, 0x4c, 0xd7, - 0xc1, 0xdd, 0xf5, 0x75, 0x7d, 0xbb, 0x0c, 0x3f, 0x15, 0xc2, 0x8f, 0x54, 0xff, 0x21, 0xc2, 0xf1, - 0x2d, 0xb1, 0xae, 0x1b, 0x08, 0xef, 0x8e, 0xd3, 0xdc, 0x6f, 0xf6, 0x51, 0xea, 0xe6, 0x1e, 0xa5, - 0x3d, 0x58, 0xd7, 0x4d, 0xf2, 0x5f, 0x3a, 0xfd, 0xb9, 0x00, 0xeb, 0x4c, 0x24, 0xfe, 0x0b, 0xe1, - 0x85, 0x89, 0x8c, 0xc7, 0xe9, 0x80, 0xfb, 0x38, 0xda, 0x33, 0xa1, 0x76, 0x99, 0x16, 0x5e, 0xa5, - 0x92, 0xc8, 0x75, 0xfc, 0x05, 0x9a, 0xae, 0xfe, 0x79, 0xd5, 0xbc, 0x0a, 0xb9, 0x0e, 0x2b, 0xbd, - 0x68, 0x1c, 0xf7, 0xd3, 0x6b, 0x70, 0x23, 0xd3, 0xd6, 0xcc, 0xf4, 0x32, 0xb3, 0x6a, 0xb9, 0x3a, - 0x2a, 0x1d, 0x5f, 0x47, 0xe4, 0xf6, 0x4c, 0x1d, 0xa9, 0x3f, 0x97, 0xea, 0xce, 0xff, 0x33, 0x83, - 0xa9, 0x65, 0x36, 0xad, 0x4d, 0xbf, 0x77, 0xe0, 0x4c, 0x9e, 0xc2, 0x2b, 0x35, 0x46, 0x9a, 0x91, - 0xc2, 0xc2, 0x8c, 0xb8, 0x8b, 0x32, 0x52, 0xcc, 0x32, 0x92, 0xbd, 0x73, 0x4b, 0xb9, 0x77, 0x2e, - 0x3d, 0x80, 0x8b, 0x73, 0x69, 0x6a, 0x47, 0xc3, 0x11, 0xd6, 0xc3, 0xbf, 0x48, 0x17, 0x8e, 0x8c, - 0x38, 0x36, 0x89, 0xaa, 0x30, 0x2d, 0xd0, 0x9b, 0x70, 0xa1, 0x27, 0x64, 0x2e, 0x49, 0xb6, 0xda, - 0x5a, 0xe0, 0x3e, 0x14, 0xcf, 0x97, 0x1c, 0x1f, 0x97, 0xe8, 0x87, 0xd0, 0x78, 0x32, 0x1a, 0x70, - 0x29, 0x4e, 0x65, 0x7d, 0x17, 0x56, 0x1f, 0x47, 0xa3, 0x28, 0x88, 0xf6, 0x27, 0x27, 0xb4, 0x7c, - 0x03, 0x56, 0xf4, 0x7c, 0xd4, 0x8f, 0x94, 0x0a, 0xb3, 0x22, 0x3d, 0x87, 0x05, 0xdd, 0xe7, 0x41, - 0x7f, 0x1c, 0x20, 0x0d, 0xfc, 0xf7, 0x4a, 0xee, 0xd6, 0x7f, 0x7d, 0xd9, 0x74, 0x7e, 0x7b, 0xd9, - 0x74, 0xfe, 0x78, 0xd9, 0x74, 0x7e, 0xfa, 0xb3, 0xf9, 0xbf, 0xbd, 0xb2, 0xfa, 0x93, 0xbf, 0xf1, - 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x13, 0x1d, 0x50, 0xda, 0x0f, 0x00, 0x00, + // 1308 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15, + 0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3, + 0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03, + 0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08, + 0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66, + 0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a, + 0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5, + 0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e, + 0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b, + 0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65, + 0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff, + 0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1, + 0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22, + 0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66, + 0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d, + 0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd, + 0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13, + 0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8, + 0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73, + 0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2, + 0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14, + 0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8, + 0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a, + 0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13, + 0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8, + 0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63, + 0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee, + 0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76, + 0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25, + 0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78, + 0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7, + 0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70, + 0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a, + 0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66, + 0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b, + 0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c, + 0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29, + 0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99, + 0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6, + 0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5, + 0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde, + 0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c, + 0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e, + 0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6, + 0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa, + 0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66, + 0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed, + 0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78, + 0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab, + 0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29, + 0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0, + 0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4, + 0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e, + 0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3, + 0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a, + 0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9, + 0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80, + 0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3, + 0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80, + 0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9, + 0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4, + 0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f, + 0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69, + 0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e, + 0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a, + 0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2, + 0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0, + 0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7, + 0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34, + 0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a, + 0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10, + 0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6, + 0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e, + 0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e, + 0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16, + 0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5, + 0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0, + 0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b, + 0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d, + 0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06, + 0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index ca987ec51..7126811ac 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package internal; message IndexMeta { - string ColumnLabel = 1; string TimeQuantum = 2; } diff --git a/pilosa.go b/pilosa.go index 71b63fcaa..a3e11b826 100644 --- a/pilosa.go +++ b/pilosa.go @@ -40,7 +40,6 @@ var ( ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey") - ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") ErrInputDefinitionNameRequired = errors.New("input-definition name required") ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrInputDefinitionValueMap = errors.New("valueMap required for map") diff --git a/server.go b/server.go index 7f6eeb6cb..cb82de12d 100644 --- a/server.go +++ b/server.go @@ -378,7 +378,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { } case *internal.CreateIndexMessage: opt := IndexOptions{ - ColumnLabel: obj.Meta.ColumnLabel, TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), } _, err := s.Holder.CreateIndex(obj.Index, opt) diff --git a/server/cluster_test.go b/server/cluster_test.go index de623bbd4..828aacdf7 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -153,8 +153,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", columnID=1) - SetBit(row=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { "cacheType": "ranked", "timeQuantum": "YMD" }}], - "fields": [{"name": "columnID", + "fields": [{"name": "col", "primaryKey": true }]} `); err != nil { @@ -351,8 +351,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", columnID=1) - SetBit(row=1, frame="f", columnID=1300000) + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=1300000) `); err != nil { t.Fatal(err) } @@ -406,8 +406,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", columnID=1) - SetBit(row=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -560,7 +560,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setBits := "" for i := 0; i < 20; i++ { - setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) + setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setBits); err != nil { diff --git a/server/server_test.go b/server/server_test.go index 28155bab4..b91cb4317 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -56,7 +56,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -131,13 +131,13 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set bits on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil { t.Fatal(err) } @@ -204,14 +204,14 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { t.Fatal(err) } @@ -234,40 +234,6 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } -// Ensure program can set column attributes with columnLabel option. -func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { - m := test.MustRunMain() - defer m.Close() - - // Create frames. - client := m.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - // Set bits on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { - t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { - t.Fatal(err) - } - - // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { - t.Fatal(err) - } - - // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { - t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { - t.Fatalf("unexpected result: %s", res) - } - -} - // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.MustRunMainWithCluster(t, 2) @@ -285,13 +251,13 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m10.Query("i", "", ` - SetBit(row=1, frame="f", columnID=100) - SetBit(row=1, frame="f", columnID=1000) - SetBit(row=1, frame="f", columnID=100000) - SetBit(row=1, frame="f", columnID=200000) - SetBit(row=1, frame="f", columnID=400000) - SetBit(row=1, frame="f", columnID=600000) - SetBit(row=1, frame="f", columnID=800000) + SetBit(row=1, frame="f", col=100) + SetBit(row=1, frame="f", col=1000) + SetBit(row=1, frame="f", col=100000) + SetBit(row=1, frame="f", col=200000) + SetBit(row=1, frame="f", col=400000) + SetBit(row=1, frame="f", col=600000) + SetBit(row=1, frame="f", col=800000) `); err != nil { t.Fatal("setting bits:", err) } @@ -401,7 +367,7 @@ func TestMain_RecalculateHashes(t *testing.T) { data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", columnID=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { diff --git a/stats_test.go b/stats_test.go index a03407a3d..3e85532a3 100644 --- a/stats_test.go +++ b/stats_test.go @@ -206,7 +206,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { From 9a2aeac6f83e151b5de1dae3c1d1a9c9a9628323 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 20 Mar 2018 16:34:30 -0500 Subject: [PATCH 191/311] Clean up logger; make it honor --log-path flag. Add functional options to NewGossipMemberSet. --- cluster.go | 96 ++++++++++-------------- cmd/server.go | 2 +- diagnostics.go | 21 +----- diagnostics_internal_test.go | 3 - fragment.go | 25 +++---- frame.go | 7 +- gossip/gossip.go | 141 +++++++++++++++++------------------ handler.go | 121 ++++++++++++++---------------- handler_test.go | 6 +- holder.go | 36 ++++----- holder_test.go | 13 +++- index.go | 7 +- logger.go | 69 +++++++++++++++++ server.go | 49 ++++++------ server/cluster_test.go | 4 +- server/server.go | 34 +++++---- stats.go | 9 +-- stats_test.go | 5 +- statsd/statsd.go | 42 +++++------ statsd/statsd_test.go | 2 - test/handler.go | 1 - test/holder.go | 7 +- test/logger.go | 48 ++++++++++++ test/pilosa.go | 2 +- view.go | 16 ++-- 25 files changed, 410 insertions(+), 356 deletions(-) create mode 100644 logger.go create mode 100644 test/logger.go diff --git a/cluster.go b/cluster.go index a5b7b654f..7e53628ac 100644 --- a/cluster.go +++ b/cluster.go @@ -20,9 +20,7 @@ import ( "errors" "fmt" "hash/fnv" - "io" "io/ioutil" - "log" "math/rand" "net/http" "os" @@ -268,8 +266,7 @@ type Cluster struct { closing chan struct{} prefect SecurityManager - // The writer for any logging. - LogOutput io.Writer + Logger Logger // RemoteClient *http.Client @@ -289,16 +286,11 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), - LogOutput: os.Stderr, - prefect: &NopSecurityManager{}, + Logger: NopLogger, + prefect: &NopSecurityManager{}, } } -// logger returns a logger for the cluster. -func (c *Cluster) logger() *log.Logger { - return log.New(c.LogOutput, "", log.LstdFlags) -} - // Coordinator returns the coordinator node. func (c *Cluster) CoordinatorNode() *Node { return c.nodeByID(c.Coordinator) @@ -359,7 +351,7 @@ func (c *Cluster) UpdateCoordinator(n *Node) bool { // AddNode adds a node to the Cluster and updates and saves the // new topology. func (c *Cluster) AddNode(node *Node) error { - c.logger().Printf("add node %s to cluster on %s", node, c.Node) + c.Logger.Printf("add node %s to cluster on %s", node, c.Node) // If the node being added is the coordinator, set it for this node. if node.IsCoordinator { @@ -437,7 +429,7 @@ func (c *Cluster) setState(state string) { return } - c.logger().Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) + c.Logger.Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) var doCleanup bool @@ -471,7 +463,7 @@ func (c *Cluster) setState(state string) { // Clean holder. if err := cleaner.CleanHolder(); err != nil { - c.logger().Printf("holder clean error: err=%s", err) + c.Logger.Printf("holder clean error: err=%s", err) } } } @@ -487,7 +479,7 @@ func (c *Cluster) SetNodeState(state string) error { State: state, } - c.logger().Printf("Sending State %s (%s)", state, c.Coordinator) + c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -509,7 +501,7 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { } c.Topology.nodeStates[nodeID] = state - c.logger().Printf("received state %s (%s)", state, nodeID) + c.Logger.Printf("received state %s (%s)", state, nodeID) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { @@ -951,9 +943,9 @@ func (c *Cluster) Open() error { return fmt.Errorf("sending restart NodeJoin: %v", err) } - c.logger().Printf("wait for joining to complete") + c.Logger.Printf("wait for joining to complete") <-c.joining - c.logger().Printf("joining has completed") + c.Logger.Printf("joining has completed") } return nil @@ -968,7 +960,7 @@ func (c *Cluster) Close() error { } func (c *Cluster) markAsJoined() { - c.logger().Printf("mark node as joined (received coordinator update)") + c.Logger.Printf("mark node as joined (received coordinator update)") if !c.joined { c.joined = true close(c.joining) @@ -1001,9 +993,9 @@ func (c *Cluster) allNodesReady() bool { func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { j, err := c.generateResizeJob(nodeAction) if err != nil { - c.logger().Printf("generateResizeJob error: err=%s", err) + c.Logger.Printf("generateResizeJob error: err=%s", err) if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.logger().Printf("setStateAndBroadcast error: err=%s", err) + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) } return err } @@ -1017,7 +1009,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { }) // Wait for the ResizeJob to finish or be aborted. - c.logger().Printf("wait for jobResult") + c.Logger.Printf("wait for jobResult") jobResult := <-j.result // Make sure j.Run() didn't return an error. @@ -1025,7 +1017,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { return err } - c.logger().Printf("received jobResult: %s", jobResult) + c.Logger.Printf("received jobResult: %s", jobResult) switch jobResult { case ResizeJobStateDone: if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { @@ -1048,7 +1040,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { func (c *Cluster) setStateAndBroadcast(state string) error { c.SetState(state) // Broadcast cluster status changes to the cluster. - c.logger().Printf("broadcasting ClusterStatus: %s", state) + c.Logger.Printf("broadcasting ClusterStatus: %s", state) return c.Broadcaster.SendSync(c.Status()) } @@ -1081,7 +1073,7 @@ func (c *Cluster) listenForJoins() { case nodeAction := <-c.joiningLeavingNodes: err := c.handleNodeAction(nodeAction) if err != nil { - c.logger().Printf("handleNodeAction error: err=%s", err) + c.Logger.Printf("handleNodeAction error: err=%s", err) continue } setNormal = true @@ -1093,7 +1085,7 @@ func (c *Cluster) listenForJoins() { if setNormal { // Put the cluster back to state NORMAL and broadcast. if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.logger().Printf("setStateAndBroadcast error: err=%s", err) + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) } } @@ -1104,7 +1096,7 @@ func (c *Cluster) listenForJoins() { case nodeAction := <-c.joiningLeavingNodes: err := c.handleNodeAction(nodeAction) if err != nil { - c.logger().Printf("handleNodeAction error: err=%s", err) + c.Logger.Printf("handleNodeAction error: err=%s", err) continue } setNormal = true @@ -1117,7 +1109,7 @@ func (c *Cluster) listenForJoins() { // added/removed. It also saves a reference to the ResizeJob in the `jobs` map // for future lookup by JobID. func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { - c.logger().Printf("generateResizeJob: %v", nodeAction) + c.Logger.Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -1125,7 +1117,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, err } - c.logger().Printf("generated ResizeJob: %d", j.ID) + c.Logger.Printf("generated ResizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1215,14 +1207,14 @@ func (c *Cluster) CompleteCurrentJob(state string) error { // FollowResizeInstruction is run by any node that receives a ResizeInstruction. func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - c.logger().Printf("follow resize instruction on %s", c.Node.ID) + c.Logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { return err } - c.logger().Printf("MergeClusterStatus done, start goroutine") + c.Logger.Printf("MergeClusterStatus done, start goroutine") // The actual resizing runs in a goroutine because we don't want to block // the distribution of other ResizeInstructions to the rest of the cluster. @@ -1242,7 +1234,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err if err := func() error { // Sync the schema received in the resize instruction. - c.logger().Printf("Holder ApplySchema") + c.Logger.Printf("Holder ApplySchema") if err := c.Holder.ApplySchema(instr.Schema); err != nil { return err } @@ -1252,7 +1244,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.logger().Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) srcURI := decodeURI(src.Node.URI) @@ -1275,7 +1267,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Stream slice from remote node. - c.logger().Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found @@ -1309,7 +1301,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { - c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) + c.Logger.Printf("sending resizeInstructionComplete error: err=%s", err) } }() return nil @@ -1363,13 +1355,7 @@ type ResizeJob struct { mu sync.RWMutex state string - // The writer for any logging. - LogOutput io.Writer -} - -// logger returns a logger for the resize job. -func (j *ResizeJob) logger() *log.Logger { - return log.New(j.LogOutput, "", log.LstdFlags) + Logger Logger } // NewResizeJob returns a new instance of ResizeJob. @@ -1397,11 +1383,11 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } return &ResizeJob{ - ID: rand.Int63(), - IDs: ids, - action: action, - result: make(chan string), - LogOutput: os.Stderr, + ID: rand.Int63(), + IDs: ids, + action: action, + result: make(chan string), + Logger: NopLogger, } } @@ -1425,18 +1411,18 @@ func (j *ResizeJob) setState(state string) { // Run distributes ResizeInstructions. func (j *ResizeJob) Run() error { - j.logger().Printf("run ResizeJob") + j.Logger.Printf("run ResizeJob") // Set job state to RUNNING. j.SetState(ResizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.nodesArePending() { - j.logger().Printf("ResizeJob contains no pending tasks; mark as done") + j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") j.result <- ResizeJobStateDone return nil } - j.logger().Printf("distribute tasks for ResizeJob") + j.Logger.Printf("distribute tasks for ResizeJob") err := j.distributeResizeInstructions() if err != nil { j.result <- ResizeJobStateAborted @@ -1466,7 +1452,7 @@ func (j *ResizeJob) nodesArePending() bool { } func (j *ResizeJob) distributeResizeInstructions() error { - j.logger().Printf("distributeResizeInstructions for job %d", j.ID) + j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) // Loop through the ResizeInstructions in ResizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create @@ -1475,7 +1461,7 @@ func (j *ResizeJob) distributeResizeInstructions() error { ID: instr.Node.ID, URI: decodeURI(instr.Node.URI), } - j.logger().Printf("send resize instructions: %v", instr) + j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { return err } @@ -1681,7 +1667,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { switch e.Event { case NodeJoin: - c.logger().Printf("received NodeJoin event: %v", e) + c.Logger.Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. if !c.IsCoordinator() { return nil @@ -1701,7 +1687,7 @@ func (c *Cluster) nodeJoin(node *Node) error { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsID(node.ID) { err := fmt.Sprintf("host is not in topology: %s", node.ID) - c.logger().Print(err) + c.Logger.Printf("%v", err) return errors.New(err) } @@ -1816,7 +1802,7 @@ func (c *Cluster) nodeLeave(node *Node) error { } func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { - c.logger().Printf("merge cluster status: %v", cs) + c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). if c.IsCoordinator() { return nil diff --git a/cmd/server.go b/cmd/server.go index ffa8c66b3..8089dae54 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -43,7 +43,7 @@ func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Long: `pilosa server runs Pilosa. It will load existing data from the configured -directory, and start listening client connections +directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr) diff --git a/diagnostics.go b/diagnostics.go index 3ad07ed42..5d3f6699b 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -18,9 +18,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" - "io/ioutil" - "log" "net/http" "strconv" "strings" @@ -52,7 +49,7 @@ type DiagnosticsCollector struct { client *http.Client - logOutput io.Writer + Logger Logger server *Server } @@ -66,7 +63,7 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector { start: time.Now(), client: &http.Client{Timeout: 10 * time.Second}, metrics: make(map[string]interface{}), - logOutput: ioutil.Discard, + Logger: NopLogger, } } @@ -119,7 +116,7 @@ func (d *DiagnosticsCollector) CheckVersion() error { d.lastVersion = rsp.Version if err := d.compareVersion(rsp.Version); err != nil { - d.logger().Printf("%s\n", err.Error()) + d.Logger.Printf("%s\n", err.Error()) } return nil @@ -160,20 +157,10 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) { d.metrics[name] = value } -// SetLogger Set the logger output type. -func (d *DiagnosticsCollector) SetLogger(logger io.Writer) { - d.logOutput = logger -} - -// logger returns a logger that writes to LogOutput. -func (d *DiagnosticsCollector) logger() *log.Logger { - return log.New(d.logOutput, "", log.LstdFlags) -} - // logErr logs the error and returns true if an error exists func (d *DiagnosticsCollector) logErr(err error) bool { if err != nil { - d.logOutput.Write([]byte(err.Error())) + d.Logger.Printf("%v", err) return true } return false diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index eb2498297..517dbed3d 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -16,7 +16,6 @@ package pilosa import ( "encoding/json" - "io/ioutil" "net/http" "net/http/httptest" "reflect" @@ -31,7 +30,6 @@ func TestDiagnosticsClient(t *testing.T) { // Create a new client. d := NewDiagnosticsCollector(server.URL) - d.SetLogger(ioutil.Discard) d.Set("gg", 10) d.Set("ss", "ss") @@ -146,7 +144,6 @@ func BenchmarkDiagnostics(b *testing.B) { // Create a new client. d := NewDiagnosticsCollector(server.URL) - d.SetLogger(ioutil.Discard) prev := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(prev) diff --git a/fragment.go b/fragment.go index ab397af47..674594081 100644 --- a/fragment.go +++ b/fragment.go @@ -26,7 +26,6 @@ import ( "hash" "io" "io/ioutil" - "log" "net/http" "os" "sort" @@ -103,8 +102,8 @@ type Fragment struct { // so that they can be mmapped and heap utilization can be kept low. MaxOpN int - // Writer used for out-of-band log entries. - LogOutput io.Writer + // Logger used for out-of-band log entries. + Logger Logger // Row attribute storage. // This is set by the parent frame unless overridden for testing. @@ -124,8 +123,8 @@ func NewFragment(path, index, frame, view string, slice uint64) *Fragment { CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, - LogOutput: ioutil.Discard, - MaxOpN: DefaultFragmentMaxOpN, + Logger: NopLogger, + MaxOpN: DefaultFragmentMaxOpN, stats: NopStatsClient, } @@ -273,7 +272,7 @@ func (f *Fragment) openCache() error { // Unmarshal cache data. var pb internal.Cache if err := proto.Unmarshal(buf, &pb); err != nil { - f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) + f.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) return nil } @@ -298,13 +297,13 @@ func (f *Fragment) Close() error { func (f *Fragment) close() error { // Flush cache if closing gracefully. if err := f.flushCache(); err != nil { - f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) + f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) return err } // Close underlying storage. if err := f.closeStorage(); err != nil { - f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) + f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) return err } @@ -343,9 +342,6 @@ func (f *Fragment) closeStorage() error { return nil } -// logger returns a logger instance for the fragment.nt. -func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) } - // Row returns a row by ID. func (f *Fragment) Row(rowID uint64) *Bitmap { f.mu.Lock() @@ -1385,18 +1381,17 @@ func (f *Fragment) Snapshot() error { defer f.mu.Unlock() return f.snapshot() } -func track(start time.Time, message string, stats StatsClient, logger *log.Logger) { +func track(start time.Time, message string, stats StatsClient, logger Logger) { elapsed := time.Since(start) logger.Printf("%s took %s", message, elapsed) stats.Histogram("snapshot", elapsed.Seconds(), 1.0) } func (f *Fragment) snapshot() error { - logger := f.logger() - logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) start := time.Now() - defer track(start, completeMessage, f.stats, logger) + defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt diff --git a/frame.go b/frame.go index 466d7ae96..831ac0a55 100644 --- a/frame.go +++ b/frame.go @@ -17,7 +17,6 @@ package pilosa import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -65,7 +64,7 @@ type Frame struct { rangeEnabled bool fields []*Field - LogOutput io.Writer + Logger Logger } // NewFrame returns a new instance of frame. @@ -95,7 +94,7 @@ func NewFrame(path, index, name string) (*Frame, error) { rangeEnabled: DefaultRangeEnabled, //fields - LogOutput: ioutil.Discard, + Logger: NopLogger, }, nil } @@ -624,7 +623,7 @@ func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { func (f *Frame) newView(path, name string) *View { view := NewView(path, f.index, f.name, name, f.cacheSize) view.cacheType = f.cacheType - view.LogOutput = f.LogOutput + view.Logger = f.Logger view.RowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster diff --git a/gossip/gossip.go b/gossip/gossip.go index 62c794560..4a1e0f5f1 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -16,10 +16,8 @@ package gossip import ( "fmt" - "io" "io/ioutil" "log" - "os" "strconv" "strings" "sync" @@ -51,8 +49,10 @@ type GossipMemberSet struct { statusHandler pilosa.StatusHandler config *gossipConfig - // The writer for any logging. - LogOutput io.Writer + Logger pilosa.Logger + + logger *log.Logger + transport *Transport } // Start implements the BroadcastReceiver interface and sets the BroadcastHandler. @@ -140,11 +140,6 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) { return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } -// logger returns a logger for the GossipMemberSet. -func (g *GossipMemberSet) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - //////////////////////////////////////////////////////////////// type gossipConfig struct { @@ -152,14 +147,58 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } -// NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport. -func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport *Transport, server *pilosa.Server) (*GossipMemberSet, error) { +type GossipMemberSetOption func(*GossipMemberSet) error + +func WithTransport(transport *Transport) func(*GossipMemberSet) error { + return func(g *GossipMemberSet) error { + g.transport = transport + return nil + } +} + +func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { + return func(g *GossipMemberSet) error { + g.logger = logger + return nil + } +} + +// NewGossipMemberSet returns a new instance of GossipMemberSet based on options. +func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { g := &GossipMemberSet{ - LogOutput: server.LogOutput, + Logger: server.Logger, } - port := transport.Net.GetAutoBindPort() + // options + for _, opt := range options { + if err := opt(g); err != nil { + return nil, err + } + } + + if g.transport == nil { + port, err := strconv.Atoi(cfg.Gossip.Port) + if err != nil { + return nil, fmt.Errorf("convert port: %s", err) + } + + bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) + if err != nil { + return nil, fmt.Errorf("getting uri from bind address: %s", err) + } + host := bindURI.Host() + + // Set up the transport. + transport, err := NewTransport(host, port, g.logger) + if err != nil { + return nil, fmt.Errorf("new tranport: %s", err) + } + + g.transport = transport + } + + port := g.transport.Net.GetAutoBindPort() bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) if err != nil { @@ -177,7 +216,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport // memberlist config conf := memberlist.DefaultWANConfig() - conf.Transport = transport.Net + conf.Transport = g.transport.Net conf.Name = name conf.BindAddr = host conf.BindPort = port @@ -196,6 +235,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport conf.Delegate = g conf.SecretKey = gossipKey conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) + conf.Logger = g.logger g.config = &gossipConfig{ memberlistConfig: conf, @@ -207,28 +247,6 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport return g, nil } -// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port. -func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) { - port, err := strconv.Atoi(cfg.Gossip.Port) - if err != nil { - return nil, fmt.Errorf("convert port: %s", err) - } - - bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) - if err != nil { - return nil, fmt.Errorf("getting uri from bind address: %s", err) - } - host := bindURI.Host() - - // Set up the transport. - transport, err := NewTransport(host, port) - if err != nil { - return nil, fmt.Errorf("new tranport: %s", err) - } - - return NewGossipMemberSetWithTransport(name, cfg, transport, server) -} - // SendSync implementation of the Broadcaster interface. func (g *GossipMemberSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) @@ -276,7 +294,7 @@ func (g *GossipMemberSet) SendAsync(pb proto.Message) error { func (g *GossipMemberSet) NodeMeta(limit int) []byte { buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) if err != nil { - g.logger().Printf("marshal message error: %s", err) + g.Logger.Printf("marshal message error: %s", err) return []byte{} } return buf @@ -287,11 +305,11 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte { func (g *GossipMemberSet) NotifyMsg(b []byte) { m, err := pilosa.UnmarshalMessage(b) if err != nil { - g.logger().Printf("unmarshal message error: %s", err) + g.Logger.Printf("unmarshal message error: %s", err) return } if err := g.handler.ReceiveMessage(m); err != nil { - g.logger().Printf("receive message error: %s", err) + g.Logger.Printf("receive message error: %s", err) return } } @@ -307,14 +325,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { func (g *GossipMemberSet) LocalState(join bool) []byte { pb, err := g.statusHandler.LocalStatus() if err != nil { - g.logger().Printf("error getting local state, err=%s", err) + g.Logger.Printf("error getting local state, err=%s", err) return []byte{} } // Marshal nodestate data to bytes. buf, err := proto.Marshal(pb) if err != nil { - g.logger().Printf("error marshalling nodestate data, err=%s", err) + g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} } return buf @@ -326,12 +344,12 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeStatus if err := proto.Unmarshal(buf, &pb); err != nil { - g.logger().Printf("error unmarshalling nodestate data, err=%s", err) + g.Logger.Printf("error unmarshalling nodestate data, err=%s", err) return } err := g.statusHandler.HandleRemoteStatus(&pb) if err != nil { - g.logger().Printf("merge state error: %s", err) + g.Logger.Printf("merge state error: %s", err) } } @@ -344,15 +362,14 @@ type GossipEventReceiver struct { ch chan memberlist.NodeEvent eventHandler pilosa.EventHandler - // The writer for any logging. - LogOutput io.Writer + Logger pilosa.Logger } // NewGossipEventReceiver returns a new instance of GossipEventReceiver. -func NewGossipEventReceiver(logOutput io.Writer) *GossipEventReceiver { +func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver { return &GossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - LogOutput: logOutput, + ch: make(chan memberlist.NodeEvent, 1), + Logger: logger, } } @@ -375,11 +392,6 @@ func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error { return nil } -// logger returns a logger for the GossipEventReceiver. -func (g *GossipEventReceiver) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - func (g *GossipEventReceiver) listen() { var nodeEventType pilosa.NodeEventType for { @@ -407,7 +419,7 @@ func (g *GossipEventReceiver) listen() { Node: node, } if err := g.eventHandler.ReceiveEvent(ne); err != nil { - g.logger().Printf("receive event error: %s", err) + g.Logger.Printf("receive event error: %s", err) } } } @@ -443,12 +455,13 @@ type Transport struct { // It will dynamically bind to a port if port is 0. // This is useful for test cases where specifiying a port is not reasonable. //func NewTransport(host string, port int) (*memberlist.NetTransport, error) { -func NewTransport(host string, port int) (*Transport, error) { +func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) { // memberlist config conf := memberlist.DefaultWANConfig() conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port + conf.Logger = logger net, err := newTransport(conf) if err != nil { @@ -469,24 +482,10 @@ func NewTransport(host string, port int) (*Transport, error) { // newTransport returns a NetTransport based on the memberlist configuration. // It will dynamically bind to a port if conf.BindPort is 0. func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { - if conf.LogOutput != nil && conf.Logger != nil { - return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") - } - - logDest := conf.LogOutput - if logDest == nil { - logDest = os.Stderr - } - - logger := conf.Logger - if logger == nil { - logger = log.New(logDest, "", log.LstdFlags) - } - nc := &memberlist.NetTransportConfig{ BindAddrs: []string{conf.BindAddr}, BindPort: conf.BindPort, - Logger: logger, + Logger: conf.Logger, } // See comment below for details about the retry in here. @@ -498,7 +497,7 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { return nt, nil } if strings.Contains(err.Error(), "address already in use") { - logger.Printf("[DEBUG] Got bind error: %v", err) + conf.Logger.Printf("[DEBUG] Got bind error: %v", err) continue } } diff --git a/handler.go b/handler.go index a23b2e5ac..ffd28da46 100644 --- a/handler.go +++ b/handler.go @@ -23,12 +23,10 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "net/url" // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" - "os" "runtime/debug" "strconv" "strings" @@ -67,8 +65,7 @@ type Handler struct { Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } - // The writer for any logging. - LogOutput io.Writer + Logger Logger // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec @@ -98,8 +95,7 @@ func NewHandler() *Handler { //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop //StatusHandler: NopStatusHandler, // TODO: implement the nop FileSystem: NopFileSystem, - - LogOutput: os.Stderr, + Logger: NopLogger, } BuildRouters(handler) handler.populateValidators() @@ -247,7 +243,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) stack := debug.Stack() msg := "PANIC: %s\n%s" - fmt.Fprintf(h.LogOutput, msg, err, stack) + h.Logger.Printf(msg, err, stack) fmt.Fprintf(w, msg, err, stack) } }() @@ -261,7 +257,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statsTags := make([]string, 0, 3) if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { - h.logger().Printf("%s %s %v", r.Method, r.URL.String(), dif) + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) statsTags = append(statsTags, "slow_query") } @@ -289,7 +285,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { filesystem, err := h.FileSystem.New() if err != nil { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") return } http.FileServer(filesystem).ServeHTTP(w, r) @@ -300,7 +296,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSchemaResponse{ Indexes: h.Holder.Schema(), }); err != nil { - h.logger().Printf("write schema response error: %s", err) + h.Logger.Printf("write schema response error: %s", err) } } @@ -308,7 +304,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { pb, err := h.StatusHandler.ClusterStatus() if err != nil { - h.logger().Printf("cluster status error: %s", err) + h.Logger.Printf("cluster status error: %s", err) return } @@ -317,7 +313,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { State: cs.State, Nodes: DecodeNodes(cs.Nodes), }); err != nil { - h.logger().Printf("write status response error: %s", err) + h.Logger.Printf("write status response error: %s", err) } } @@ -395,7 +391,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Write response back to client. if err := h.writeQueryResponse(w, r, resp); err != nil { - h.logger().Printf("write query response error: %s", err) + h.Logger.Printf("write query response error: %s", err) } } @@ -405,7 +401,7 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { Standard: h.Holder.MaxSlices(), Inverse: h.Holder.MaxInverseSlices(), }); err != nil { - h.logger().Printf("write slices-max response error: %s", err) + h.Logger.Printf("write slices-max response error: %s", err) } } @@ -431,7 +427,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getIndexResponse{ map[string]string{"name": index.Name()}, }); err != nil { - h.logger().Printf("write response error: %s", err) + h.Logger.Printf("write response error: %s", err) } } @@ -519,12 +515,12 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { Index: indexName, }) if err != nil { - h.logger().Printf("problem sending DeleteIndex message: %s", err) + h.Logger.Printf("problem sending DeleteIndex message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.Count("deleteIndex", 1, 1.0) @@ -564,14 +560,14 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { Meta: req.Options.Encode(), }) if err != nil { - h.logger().Printf("problem sending CreateIndex message: %s", err) + h.Logger.Printf("problem sending CreateIndex message: %s", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } // Encode response. if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.Count("createIndex", 1, 1.0) @@ -610,7 +606,7 @@ func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Req // Encode response. if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -665,7 +661,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -718,12 +714,12 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { Meta: req.Options.Encode(), }) if err != nil { - h.logger().Printf("problem sending CreateFrame message: %s", err) + h.Logger.Printf("problem sending CreateFrame message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -783,7 +779,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { index := h.Holder.Index(indexName) if index == nil { if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } return } @@ -801,12 +797,12 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { Frame: frameName, }) if err != nil { - h.logger().Printf("problem sending DeleteFrame message: %s", err) + h.Logger.Printf("problem sending DeleteFrame message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -848,7 +844,7 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req // Encode response. if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -899,12 +895,12 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { Field: encodeField(field), }) if err != nil { - h.logger().Printf("problem sending CreateField message: %s", err) + h.Logger.Printf("problem sending CreateField message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -943,12 +939,12 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) Field: fieldName, }) if err != nil { - h.logger().Printf("problem sending DeleteField message: %s", err) + h.Logger.Printf("problem sending DeleteField message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -979,7 +975,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { // Encode response. if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1012,7 +1008,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { // Encode response. if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1046,12 +1042,12 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { View: viewName, }) if err != nil { - h.logger().Printf("problem sending DeleteView message: %s", err) + h.Logger.Printf("problem sending DeleteView message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1107,7 +1103,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1276,10 +1272,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Find the Index. - h.logger().Println("importing:", req.Index, req.Frame, req.Slice) + h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) index := h.Holder.Index(req.Index) if index == nil { - h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) + h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1287,7 +1283,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Retrieve frame. f := index.Frame(req.Frame) if f == nil { - h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) + h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -1295,7 +1291,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Import into fragment. err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - h.logger().Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -1346,10 +1342,10 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) } // Find the Index. - h.logger().Println("importing:", req.Index, req.Frame, req.Slice) + h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) index := h.Holder.Index(req.Index) if index == nil { - h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) + h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1357,7 +1353,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) // Retrieve frame. f := index.Frame(req.Frame) if f == nil { - h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) + h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -1365,7 +1361,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) // Import into fragment. err = f.ImportValue(req.Field, req.ColumnIDs, req.Values) if err != nil { - h.logger().Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) return } @@ -1452,7 +1448,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.logger().Printf("json write error: %s", err) + h.Logger.Printf("json write error: %s", err) } } @@ -1475,7 +1471,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) // Stream fragment to response body. if _, err := f.WriteTo(w); err != nil { - h.logger().Printf("fragment backup error: %s", err) + h.Logger.Printf("fragment backup error: %s", err) } } @@ -1545,7 +1541,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Encode response. buf, err := proto.Marshal(&resp) if err != nil { - h.logger().Printf("merge block response encoding error: %s", err) + h.Logger.Printf("merge block response encoding error: %s", err) return } @@ -1579,7 +1575,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ Blocks: blocks, }); err != nil { - h.logger().Printf("block response encoding error: %s", err) + h.Logger.Printf("block response encoding error: %s", err) } } @@ -1680,7 +1676,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { - h.logger().Printf("write version response error: %s", err) + h.Logger.Printf("write version response error: %s", err) } } @@ -1696,7 +1692,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { }{ Version: version, }); err != nil { - h.logger().Printf("write version response error: %s", err) + h.Logger.Printf("write version response error: %s", err) } } @@ -1716,11 +1712,6 @@ func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "\n}\n") } -// logger returns a logger for the handler. -func (h *Handler) logger() *log.Logger { - return log.New(h.LogOutput, "", log.LstdFlags) -} - // QueryResult types. const ( QueryResultTypeNil uint32 = iota @@ -1908,11 +1899,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque Definition: def, }) if err != nil { - h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + h.Logger.Printf("problem sending CreateInputDefinition message: %s", err) } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1938,7 +1929,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques Frames: inputDef.frames, Fields: inputDef.fields, }); err != nil { - h.logger().Printf("write status response error: %s", err) + h.Logger.Printf("write status response error: %s", err) } } @@ -1967,11 +1958,11 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req Name: inputDefName, }) if err != nil { - h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) + h.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2013,7 +2004,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2061,7 +2052,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r Old: oldNode, New: newNode, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2101,7 +2092,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht if err := json.NewEncoder(w).Encode(removeNodeResponse{ Remove: removeNode, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2134,7 +2125,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ Info: msg, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2271,7 +2262,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 551c81d66..1e30f41ad 100644 --- a/handler_test.go +++ b/handler_test.go @@ -37,13 +37,13 @@ import ( func TestHandlerPanics(t *testing.T) { h := test.NewHandler() - buf := &bytes.Buffer{} - h.Handler.LogOutput = buf + bufLogger := test.NewBufferLogger() + h.Handler.Logger = bufLogger w := httptest.NewRecorder() // will panic since Handler has no Holder set up h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) - bufbytes, err := ioutil.ReadAll(buf) + bufbytes, err := bufLogger.ReadAll() if err != nil { t.Fatalf("reading all logoutput: %v", err) } diff --git a/holder.go b/holder.go index 63bbbbc86..6d1d4ef12 100644 --- a/holder.go +++ b/holder.go @@ -18,9 +18,7 @@ import ( "context" "errors" "fmt" - "io" "io/ioutil" - "log" "net/http" "os" "path" @@ -71,7 +69,7 @@ type Holder struct { // The interval at which the cached row ids are persisted to disk. CacheFlushInterval time.Duration - LogOutput io.Writer + Logger Logger } // NewHolder returns a new instance of Holder. @@ -89,7 +87,7 @@ func NewHolder() *Holder { CacheFlushInterval: DefaultCacheFlushInterval, - LogOutput: os.Stderr, + Logger: NopLogger, } } @@ -97,7 +95,7 @@ func NewHolder() *Holder { // without actually loading any data into memory. // HasData is returned, and h.hasData is set. func (h *Holder) Peek() bool { - h.logger().Printf("peek at holder path: %s", h.Path) + h.Logger.Printf("peek at holder path: %s", h.Path) h.hasData = false // Open path to read all index directories. @@ -127,7 +125,7 @@ func (h *Holder) Peek() bool { func (h *Holder) Open() error { h.setFileLimit() - h.logger().Printf("open holder path: %s", h.Path) + h.Logger.Printf("open holder path: %s", h.Path) if err := os.MkdirAll(h.Path, 0777); err != nil { return err } @@ -149,18 +147,18 @@ func (h *Holder) Open() error { continue } - h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) + h.Logger.Printf("opening index: %s", filepath.Base(fi.Name())) index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err) + h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { return err } if err := index.Open(); err != nil { if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err) + h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue } return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) @@ -169,7 +167,7 @@ func (h *Holder) Open() error { h.indexes[index.Name()] = index h.mu.Unlock() } - h.logger().Printf("open holder: complete") + h.Logger.Printf("open holder: complete") // Periodically flush cache. h.wg.Add(1) @@ -374,7 +372,7 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { if err != nil { return nil, err } - index.LogOutput = h.LogOutput + index.Logger = h.Logger index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.Broadcaster index.NewAttrStore = h.NewAttrStore @@ -464,7 +462,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) } } } @@ -488,7 +486,7 @@ func (h *Holder) setFileLimit() { newLimit := &syscall.Rlimit{} if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { - h.logger().Printf("ERROR checking open file limit: %s", err) + h.Logger.Printf("ERROR checking open file limit: %s", err) return } // If the soft limit is lower than the FileLimit constant, we will try to change it. @@ -512,32 +510,30 @@ func (h *Holder) setFileLimit() { } // Try setting again with lowered Max (hard limit) if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil { - h.logger().Printf("ERROR setting open file limit: %s", err) + h.Logger.Printf("ERROR setting open file limit: %s", err) } // If we weren't trying to change the hard limit, let the user know something is wrong. } else { - h.logger().Printf("ERROR setting open file limit: %s", err) + h.Logger.Printf("ERROR setting open file limit: %s", err) } } // Check the limit after setting it. OS may not obey Setrlimit call. if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { - h.logger().Printf("ERROR checking open file limit: %s", err) + h.Logger.Printf("ERROR checking open file limit: %s", err) } else { if oldLimit.Cur < FileLimit { - h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) + h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) } } } } -func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } - func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, "ID") nodeID := "" - h.logger().Printf("load NodeID: %s", idPath) + h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { return "", err } diff --git a/holder_test.go b/holder_test.go index 5a06a9272..41ede1adf 100644 --- a/holder_test.go +++ b/holder_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "bytes" "context" "os" "path/filepath" @@ -30,6 +31,10 @@ import ( func TestHolder_Open(t *testing.T) { t.Run("ErrIndexName", func(t *testing.T) { h := test.MustOpenHolder() + + bufLogger := test.NewBufferLogger() + h.Holder.Logger = bufLogger + defer h.Close() if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { @@ -39,8 +44,12 @@ func TestHolder_Open(t *testing.T) { } if err := h.Reopen(); err != nil { t.Fatal(err) - } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { - t.Fatalf("expected log error:\n%s", logOutput) + } + + if bufbytes, err := bufLogger.ReadAll(); err != nil { + t.Fatal(err) + } else if !bytes.Contains(bufbytes, []byte("ERROR opening index: !")) { + t.Fatalf("expected log error:\n%s", bufbytes) } }) diff --git a/index.go b/index.go index 707b87213..5ecad4f78 100644 --- a/index.go +++ b/index.go @@ -17,7 +17,6 @@ package pilosa import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -66,7 +65,7 @@ type Index struct { broadcaster Broadcaster Stats StatsClient - LogOutput io.Writer + Logger Logger } // NewIndex returns a new instance of Index. @@ -92,7 +91,7 @@ func NewIndex(path, name string) (*Index, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - LogOutput: ioutil.Discard, + Logger: NopLogger, }, nil } @@ -528,7 +527,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { if err != nil { return nil, err } - f.LogOutput = i.LogOutput + f.Logger = i.Logger f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) diff --git a/logger.go b/logger.go new file mode 100644 index 000000000..bf516d187 --- /dev/null +++ b/logger.go @@ -0,0 +1,69 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "log" +) + +// Ensure nopLogger implements interface. +var _ Logger = &nopLogger{} + +// Logger represents an interface for a shared logger. +type Logger interface { + Printf(format string, v ...interface{}) + Debugf(format string, v ...interface{}) +} + +func init() { + NopLogger = &nopLogger{ + logger: log.New(ioutil.Discard, "", log.LstdFlags), + } +} + +// NopLogger represents a Logger that doesn't do anything. +var NopLogger Logger + +type nopLogger struct { + logger *log.Logger +} + +// Printf is a no-op implementation of the Logger Printf method. +func (n *nopLogger) Printf(format string, v ...interface{}) {} + +// Debugf is a no-op implementation of the Logger Debugf method. +func (n *nopLogger) Debugf(format string, v ...interface{}) {} + +// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. +type StandardLogger struct { + logger *log.Logger +} + +func NewStandardLogger(logger *log.Logger) *StandardLogger { + return &StandardLogger{ + logger: logger, + } +} + +// Printf is a no-op implementation of the Logger Printf method. +func (s *StandardLogger) Printf(format string, v ...interface{}) { + s.logger.Printf(format, v...) +} + +// Debugf is a no-op implementation of the Logger Debugf method. +func (s *StandardLogger) Debugf(format string, v ...interface{}) { + // TODO: implement this +} diff --git a/server.go b/server.go index 7f6eeb6cb..11b2ee2a3 100644 --- a/server.go +++ b/server.go @@ -20,7 +20,7 @@ import ( "errors" "fmt" "io" - "log" + "io/ioutil" "net" "net/http" "os" @@ -88,7 +88,7 @@ type Server struct { MaxWritesPerRequest int LogOutput io.Writer - logger *log.Logger + Logger Logger defaultClient InternalClient } @@ -115,9 +115,9 @@ func NewServer() *Server { MetricInterval: 0, DiagnosticInterval: 0, - LogOutput: os.Stderr, + LogOutput: ioutil.Discard, + Logger: NopLogger, } - s.logger = log.New(s.LogOutput, "", log.LstdFlags) s.Handler.Holder = s.Holder s.diagnostics.server = s @@ -126,7 +126,7 @@ func NewServer() *Server { // Open opens and initializes the server. func (s *Server) Open() error { - s.Logger().Printf("open server") + s.Logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { if err := s.OpenListener(); err != nil { @@ -151,7 +151,6 @@ func (s *Server) Open() error { // Peek at the holder to determine if there is data on disk. // Don't actually load the data until after the Cluster // management starts. - s.Holder.LogOutput = s.LogOutput s.Holder.Peek() // Create default HTTP client @@ -175,7 +174,6 @@ func (s *Server) Open() error { s.Handler.Node = node s.Handler.Cluster = s.Cluster s.Handler.Executor = e - s.Handler.LogOutput = s.LogOutput s.Cluster.prefect = s.Handler @@ -186,7 +184,7 @@ func (s *Server) Open() error { go func() { err := http.Serve(s.ln, s.Handler) if err != nil { - s.Logger().Printf("HTTP handler terminated with error: %s\n", err) + s.Logger.Printf("HTTP handler terminated with error: %s\n", err) } }() @@ -226,7 +224,7 @@ func (s *Server) Open() error { // OpenListener opens a listener for the Server. func (s *Server) OpenListener() error { - s.Logger().Printf("open server listener: %s", s.URI) + s.Logger.Printf("open server listener: %s", s.URI) if s.ln != nil { return fmt.Errorf("a listener already exists for server: %s", s.URI) } @@ -288,7 +286,7 @@ func (s *Server) LoadNodeID() string { } nodeID, err := s.Holder.loadNodeID() if err != nil { - s.Logger().Printf("loading NodeID: %v", err) + s.Logger.Printf("loading NodeID: %v", err) return s.NodeID } return nodeID @@ -321,14 +319,11 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } -// Logger returns a logger that writes to LogOutput -func (s *Server) Logger() *log.Logger { return s.logger } - func (s *Server) monitorAntiEntropy() { ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() - s.Logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) for { // Wait for tick or a close. @@ -339,7 +334,7 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.Logger().Printf("holder sync beginning") + s.Logger.Printf("holder sync beginning") // Initialize syncer with local holder and remote client. var syncer HolderSyncer @@ -352,12 +347,12 @@ func (s *Server) monitorAntiEntropy() { // Sync holders. if err := syncer.SyncHolder(); err != nil { - s.Logger().Printf("holder sync error: err=%s", err) + s.Logger.Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.Logger().Printf("holder sync complete") + s.Logger.Printf("holder sync complete") dif := time.Since(t) s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } @@ -482,7 +477,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { - s.Logger().Printf("SendSync to: %s", node.URI) + s.Logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.URI == node.URI { continue @@ -504,7 +499,7 @@ func (s *Server) SendAsync(pb proto.Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, pb proto.Message) error { - s.Logger().Printf("SendTo: %s", to.URI) + s.Logger.Printf("SendTo: %s", to.URI) ctx := context.WithValue(context.Background(), "uri", &to.URI) return s.defaultClient.SendMessage(ctx, pb) } @@ -554,7 +549,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) if err != nil { - s.Logger().Printf("merge remote status: %s", err) + s.Logger.Printf("merge remote status: %s", err) } }() @@ -579,7 +574,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger().Printf("Local Index not found: %s", index) + s.Logger.Printf("Local Index not found: %s", index) continue } if newMax > oldmaxslices[index] { @@ -595,7 +590,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger().Printf("Local Index not found: %s", index) + s.Logger.Printf("Local Index not found: %s", index) continue } if newMaxInverse > oldMaxInverseSlices[index] { @@ -611,13 +606,13 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { func (s *Server) monitorDiagnostics() { // Do not send more than once a minute if s.DiagnosticInterval < time.Minute { - s.Logger().Printf("diagnostics disabled") + s.Logger.Printf("diagnostics disabled") return } else { - s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) + s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) } - s.diagnostics.SetLogger(s.LogOutput) + s.diagnostics.Logger = s.Logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) @@ -639,7 +634,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.CheckVersion() err = s.diagnostics.Flush() if err != nil { - s.Logger().Printf("Diagnostics error: %s", err) + s.Logger.Printf("Diagnostics error: %s", err) } } @@ -670,7 +665,7 @@ func (s *Server) monitorRuntime() { defer s.GCNotifier.Close() - s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) + s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval) for { // Wait for tick or a close. diff --git a/server/cluster_test.go b/server/cluster_test.go index fbb784691..6bbdea4b4 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -54,7 +54,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.Coordinator = m0.Server.NodeID m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} - m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput) + m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger) gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) if err != nil { t.Fatal(err) @@ -81,7 +81,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()} m1.Server.Cluster.Coordinator = m0.Server.NodeID - m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput) + m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger) gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) if err != nil { t.Fatal(err) diff --git a/server/server.go b/server/server.go index ce1eff629..d45be21e2 100644 --- a/server/server.go +++ b/server/server.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "log" "math/rand" "os" "path/filepath" @@ -71,6 +72,8 @@ type Command struct { Started chan struct{} // Done will be closed when Command.Close() is called Done chan struct{} + + logger *log.Logger } // NewCommand returns a new instance of Main. @@ -115,7 +118,7 @@ func (m *Command) Run(args ...string) (err error) { return fmt.Errorf("server.Open: %v", err) } - m.Server.Logger().Printf("Listening as %s\n", m.Server.URI) + m.Server.Logger.Printf("Listening as %s\n", m.Server.URI) return nil } @@ -126,6 +129,18 @@ func (m *Command) SetupServer() error { return err } + // Set up logger based on configuration. + lw, err := GetLogWriter(m.Config.LogPath, m.Stderr) + if err != nil { + return err + } + m.Server.LogOutput = lw + m.logger = log.New(lw, "", log.LstdFlags) + m.Server.Logger = pilosa.NewStandardLogger(m.logger) + m.Server.Handler.Logger = m.Server.Logger + m.Server.Holder.Logger = m.Server.Logger + m.Server.Holder.Stats.SetLogger(m.Server.Logger) + uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { @@ -136,15 +151,10 @@ func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN cluster.Holder = m.Server.Holder + cluster.Logger = m.Server.Logger m.Server.Cluster = cluster - // Setup logging output. - m.Server.LogOutput, err = GetLogWriter(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } - // Configure data directory (for Cluster .topology) m.Server.Cluster.Path = m.Config.DataDir @@ -152,7 +162,7 @@ func (m *Command) SetupServer() error { m.Server.Holder.NewAttrStore = boltdb.NewAttrStore // Configure holder. - m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) + m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) if m.Config.Metric.Diagnostics { @@ -165,8 +175,6 @@ func (m *Command) SetupServer() error { return err } - m.Server.Holder.Stats.SetLogger(m.Server.LogOutput) - // Copy configuration flags. m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest @@ -246,7 +254,7 @@ func (m *Command) SetupNetworking() error { if m.GossipTransport != nil { transport = m.GossipTransport } else { - transport, err = gossip.NewTransport(gossipHost, gossipPort) + transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger) if err != nil { return err } @@ -257,8 +265,8 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Coordinator = m.Server.NodeID } - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput) - gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger) + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) if err != nil { return err } diff --git a/stats.go b/stats.go index 998b84511..313708fbc 100644 --- a/stats.go +++ b/stats.go @@ -16,7 +16,6 @@ package pilosa import ( "expvar" - "io" "sort" "strings" "sync" @@ -57,7 +56,7 @@ type StatsClient interface { Timing(name string, value time.Duration, rate float64) // SetLogger Set the logger output type - SetLogger(logger io.Writer) + SetLogger(logger Logger) // Starts the service Open() @@ -79,7 +78,7 @@ func (c *nopStatsClient) Gauge(name string, value float64, rate float64) func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {} func (c *nopStatsClient) Set(name string, value string, rate float64) {} func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} -func (c *nopStatsClient) SetLogger(logger io.Writer) {} +func (c *nopStatsClient) SetLogger(logger Logger) {} func (c *nopStatsClient) Open() {} func (c *nopStatsClient) Close() error { return nil } @@ -154,7 +153,7 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 } // SetLogger has no logger. -func (c *ExpvarStatsClient) SetLogger(logger io.Writer) { +func (c *ExpvarStatsClient) SetLogger(logger Logger) { } // Open no-op. @@ -226,7 +225,7 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) } // SetLogger Sets the StatsD logger output type. -func (a MultiStatsClient) SetLogger(logger io.Writer) { +func (a MultiStatsClient) SetLogger(logger Logger) { for _, c := range a { c.SetLogger(logger) } diff --git a/stats_test.go b/stats_test.go index 93d7d9bb1..60deae993 100644 --- a/stats_test.go +++ b/stats_test.go @@ -16,8 +16,6 @@ package pilosa_test import ( "context" - "io" - "io/ioutil" "net/http" "strings" "testing" @@ -38,7 +36,6 @@ func TestMultiStatClient_Expvar(t *testing.T) { ms[0] = c hldr.Stats = ms - hldr.Stats.SetLogger(ioutil.Discard) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -357,6 +354,6 @@ func (c *MockStats) Gauge(name string, value float64, rate float64) {} func (c *MockStats) Histogram(name string, value float64, rate float64) {} func (c *MockStats) Set(name string, value string, rate float64) {} func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} -func (c *MockStats) SetLogger(logger io.Writer) {} +func (c *MockStats) SetLogger(logger pilosa.Logger) {} func (c *MockStats) Open() {} func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go index c7c072ae3..0cec63718 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,9 +15,6 @@ package statsd import ( - "io" - "io/ioutil" - "log" "time" "github.com/DataDog/datadog-go/statsd" @@ -40,9 +37,9 @@ var _ pilosa.StatsClient = &StatsClient{} // StatsClient represents a StatsD implementation of pilosa.StatsClient. type StatsClient struct { - client *statsd.Client - tags []string - logOutput io.Writer + client *statsd.Client + tags []string + logger pilosa.Logger } // NewStatsClient returns a new instance of StatsClient. @@ -53,8 +50,8 @@ func NewStatsClient(host string) (*StatsClient, error) { } return &StatsClient{ - client: c, - logOutput: ioutil.Discard, + client: c, + logger: pilosa.NopLogger, }, nil } @@ -74,16 +71,16 @@ func (c *StatsClient) Tags() []string { // WithTags returns a new client with additional tags appended. func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { return &StatsClient{ - client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), - logOutput: c.logOutput, + client: c.client, + tags: pilosa.UnionStringSlice(c.tags, tags), + logger: c.logger, } } // Count tracks the number of times something occurs per second. func (c *StatsClient) Count(name string, value int64, rate float64) { if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Count error: %s", err) + c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } @@ -91,44 +88,39 @@ func (c *StatsClient) Count(name string, value int64, rate float64) { func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { tags := append(c.tags, t...) if err := c.client.Count(Prefix+name, value, tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Count error: %s", err) + c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // Gauge sets the value of a metric. func (c *StatsClient) Gauge(name string, value float64, rate float64) { if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Gauge error: %s", err) + c.logger.Printf("statsd.StatsClient.Gauge error: %s", err) } } // Histogram tracks statistical distribution of a metric. func (c *StatsClient) Histogram(name string, value float64, rate float64) { if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Histogram error: %s", err) + c.logger.Printf("statsd.StatsClient.Histogram error: %s", err) } } // Set tracks number of unique elements. func (c *StatsClient) Set(name string, value string, rate float64) { if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Set error: %s", err) + c.logger.Printf("statsd.StatsClient.Set error: %s", err) } } // Timing tracks timing information for a metric. func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Timing error: %s", err) + c.logger.Printf("statsd.StatsClient.Timing error: %s", err) } } -// SetLogger has no logger -func (c *StatsClient) SetLogger(logger io.Writer) { - c.logOutput = logger -} - -// logger returns a logger that writes to LogOutput -func (c *StatsClient) logger() *log.Logger { - return log.New(c.logOutput, "", log.LstdFlags) +// SetLogger sets the logger for client. +func (c *StatsClient) SetLogger(logger pilosa.Logger) { + c.logger = logger } diff --git a/statsd/statsd_test.go b/statsd/statsd_test.go index 4d5c1cc98..38228349b 100644 --- a/statsd/statsd_test.go +++ b/statsd/statsd_test.go @@ -15,7 +15,6 @@ package statsd_test import ( - "io/ioutil" "reflect" "testing" "time" @@ -31,7 +30,6 @@ func TestStatsClient_WithTags(t *testing.T) { t.Fatal(err) } defer c.Close() - c.SetLogger(ioutil.Discard) // Create a new client with additional tags. c1 := c.WithTags("foo", "bar") diff --git a/test/handler.go b/test/handler.go index ea44bef9a..0f8c1cefd 100644 --- a/test/handler.go +++ b/test/handler.go @@ -41,7 +41,6 @@ func NewHandler() *Handler { Handler: pilosa.NewHandler(), } h.Handler.Executor = &h.Executor - h.Handler.LogOutput = ioutil.Discard // Handler test messages can no-op. h.Broadcaster = pilosa.NopBroadcaster diff --git a/test/holder.go b/test/holder.go index 59402cea0..97b399bea 100644 --- a/test/holder.go +++ b/test/holder.go @@ -15,7 +15,6 @@ package test import ( - "bytes" "io/ioutil" "os" @@ -26,7 +25,6 @@ import ( // Holder is a test wrapper for pilosa.Holder. type Holder struct { *pilosa.Holder - LogOutput bytes.Buffer } // NewHolder returns a new instance of Holder with a temporary path. @@ -38,7 +36,6 @@ func NewHolder() *Holder { h := &Holder{Holder: pilosa.NewHolder()} h.Path = path - h.Holder.LogOutput = &h.LogOutput h.Holder.NewAttrStore = boltdb.NewAttrStore return h @@ -62,10 +59,10 @@ func (h *Holder) Close() error { // Reopen instantiates and opens a new holder. // Note that the holder must be Closed first. func (h *Holder) Reopen() error { - path, logOutput := h.Path, h.Holder.LogOutput + path, logger := h.Path, h.Holder.Logger h.Holder = pilosa.NewHolder() h.Holder.Path = path - h.Holder.LogOutput = logOutput + h.Holder.Logger = logger h.Holder.NewAttrStore = boltdb.NewAttrStore if err := h.Holder.Open(); err != nil { return err diff --git a/test/logger.go b/test/logger.go new file mode 100644 index 000000000..b4a0079b1 --- /dev/null +++ b/test/logger.go @@ -0,0 +1,48 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "fmt" + "io/ioutil" +) + +// BufferLogger represents a test Logger that holds log messages +// in a buffer for review. +type BufferLogger struct { + buf *bytes.Buffer +} + +// NewBufferLogger returns a new instance of BufferLogger. +func NewBufferLogger() *BufferLogger { + return &BufferLogger{ + buf: &bytes.Buffer{}, + } +} + +func (b *BufferLogger) Printf(format string, v ...interface{}) { + s := fmt.Sprintf(format, v...) + _, err := b.buf.WriteString(s) + if err != nil { + panic(err) + } +} + +func (b *BufferLogger) Debugf(format string, v ...interface{}) {} + +func (b *BufferLogger) ReadAll() ([]byte, error) { + return ioutil.ReadAll(b.buf) +} diff --git a/test/pilosa.go b/test/pilosa.go index 86623c239..b00c9841c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -181,7 +181,7 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( } // Open gossip transport to use in SetupServer. - transport, err := gossip.NewTransport(host, bindPort) + transport, err := gossip.NewTransport(host, bindPort, nil) if err != nil { return seed, err } diff --git a/view.go b/view.go index dfedebbe3..f775f1b0e 100644 --- a/view.go +++ b/view.go @@ -16,9 +16,6 @@ package pilosa import ( "fmt" - "io" - "io/ioutil" - "log" "os" "path/filepath" "strconv" @@ -64,7 +61,7 @@ type View struct { stats StatsClient RowAttrStore AttrStore - LogOutput io.Writer + Logger Logger } // NewView returns a new instance of View. @@ -81,7 +78,7 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View { broadcaster: NopBroadcaster, stats: NopStatsClient, - LogOutput: ioutil.Discard, + Logger: NopLogger, } } @@ -126,9 +123,6 @@ func (v *View) Open() error { return nil } -// logger returns a logger instance for the view. -func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) } - // openFragments opens and initializes the fragments inside the view. func (v *View) openFragments() error { file, err := os.Open(filepath.Join(v.path, "fragments")) @@ -275,7 +269,7 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { frag := NewFragment(path, v.index, v.frame, v.name, slice) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize - frag.LogOutput = v.LogOutput + frag.Logger = v.Logger frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) return frag } @@ -288,7 +282,7 @@ func (v *View) DeleteFragment(slice uint64) error { return ErrFragmentNotFound } - v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) // Close data files before deletion. if err := fragment.Close(); err != nil { @@ -302,7 +296,7 @@ func (v *View) DeleteFragment(slice uint64) error { // Delete fragment cache file. if err := os.Remove(fragment.CachePath()); err != nil { - v.logger().Printf("no cache file to delete for slice %d", slice) + v.Logger.Printf("no cache file to delete for slice %d", slice) } delete(v.fragments, slice) From d113ebe147e33552e4ace9d9f1bcf366ac291b7a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 15:34:15 -0500 Subject: [PATCH 192/311] Support --verbose logging --- config.go | 2 ++ ctl/server.go | 1 + gossip/gossip.go | 3 +++ logger.go | 24 ++++++++++++++++++++---- server/server.go | 6 +++++- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/config.go b/config.go index f2dd41793..84f199dc3 100644 --- a/config.go +++ b/config.go @@ -133,6 +133,7 @@ type Config struct { MaxWritesPerRequest int `toml:"max-writes-per-request"` LogPath string `toml:"log-path"` + Verbose bool `toml:"verbose"` // TLS TLS TLSConfig @@ -178,6 +179,7 @@ func NewConfig() *Config { Bind: ":" + DefaultPort, MaxWritesPerRequest: DefaultMaxWritesPerRequest, // LogPath: "", + // Verbose: false, TLS: TLSConfig{}, } diff --git a/ctl/server.go b/ctl/server.go index 87d6a8e15..d21b8e889 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -28,6 +28,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") + flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Verbose logging") // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) diff --git a/gossip/gossip.go b/gossip/gossip.go index 4a1e0f5f1..a27da24c8 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -147,8 +147,10 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } +// GossipMemberSetOption describes a functional option for GossipMemberSet. type GossipMemberSetOption func(*GossipMemberSet) error +// WithTransport is a functional option for providing a transport to NewGossipMemberSet. func WithTransport(transport *Transport) func(*GossipMemberSet) error { return func(g *GossipMemberSet) error { g.transport = transport @@ -156,6 +158,7 @@ func WithTransport(transport *Transport) func(*GossipMemberSet) error { } } +// WithLogger is a functional option for providing a logger to NewGossipMemberSet. func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { return func(g *GossipMemberSet) error { g.logger = logger diff --git a/logger.go b/logger.go index bf516d187..ab26b0883 100644 --- a/logger.go +++ b/logger.go @@ -58,12 +58,28 @@ func NewStandardLogger(logger *log.Logger) *StandardLogger { } } -// Printf is a no-op implementation of the Logger Printf method. func (s *StandardLogger) Printf(format string, v ...interface{}) { s.logger.Printf(format, v...) } -// Debugf is a no-op implementation of the Logger Debugf method. -func (s *StandardLogger) Debugf(format string, v ...interface{}) { - // TODO: implement this +func (s *StandardLogger) Debugf(format string, v ...interface{}) {} + +// VerboseLogger is an implementation of pilosa.Logger which includes debug messages. +type VerboseLogger struct { + logger *log.Logger +} + +func NewVerboseLogger(logger *log.Logger) *VerboseLogger { + return &VerboseLogger{ + logger: logger, + } +} + +func (vb *VerboseLogger) Printf(format string, v ...interface{}) { + vb.logger.Printf(format, v...) +} + +func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { + vb.logger.Printf("VERBOSE...") + vb.logger.Printf(format, v...) } diff --git a/server/server.go b/server/server.go index d45be21e2..8fe508fbd 100644 --- a/server/server.go +++ b/server/server.go @@ -136,7 +136,11 @@ func (m *Command) SetupServer() error { } m.Server.LogOutput = lw m.logger = log.New(lw, "", log.LstdFlags) - m.Server.Logger = pilosa.NewStandardLogger(m.logger) + if m.Config.Verbose { + m.Server.Logger = pilosa.NewVerboseLogger(m.logger) + } else { + m.Server.Logger = pilosa.NewStandardLogger(m.logger) + } m.Server.Handler.Logger = m.Server.Logger m.Server.Holder.Logger = m.Server.Logger m.Server.Holder.Stats.SetLogger(m.Server.Logger) From 0dc8aa0a5cd9fe406caa13e5c3f3a11aca42cd2b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 15:40:28 -0500 Subject: [PATCH 193/311] add --verbose flag to docs --- ctl/server.go | 2 +- docs/configuration.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ctl/server.go b/ctl/server.go index d21b8e889..1c60a7c9c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -28,7 +28,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") - flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Verbose logging") + flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging") // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) diff --git a/docs/configuration.md b/docs/configuration.md index 633e83f3c..4d71dee8e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,6 +81,17 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h log-path = "/path/to/logfile" ``` +#### Verbose + +* Description: Enable verbose logging. +* Flag: `--verbose` +* Env: `PILOSA_VERBOSE` +* Config: + + ```toml + verbose = true + ``` + #### Max Writes Per Request * Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue. From 633d99b21791a1891bba7a2ad7f3f004fa3b1612 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 15:47:51 -0500 Subject: [PATCH 194/311] remove leftover `logger` from NopLogger implementation --- logger.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/logger.go b/logger.go index ab26b0883..2d3df732b 100644 --- a/logger.go +++ b/logger.go @@ -14,10 +14,7 @@ package pilosa -import ( - "io/ioutil" - "log" -) +import "log" // Ensure nopLogger implements interface. var _ Logger = &nopLogger{} @@ -29,17 +26,13 @@ type Logger interface { } func init() { - NopLogger = &nopLogger{ - logger: log.New(ioutil.Discard, "", log.LstdFlags), - } + NopLogger = &nopLogger{} } // NopLogger represents a Logger that doesn't do anything. var NopLogger Logger -type nopLogger struct { - logger *log.Logger -} +type nopLogger struct{} // Printf is a no-op implementation of the Logger Printf method. func (n *nopLogger) Printf(format string, v ...interface{}) {} From fd8040e69920ecdad5d47e88bbf3ccfaf5792b91 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 16:24:15 -0500 Subject: [PATCH 195/311] remove LogOutput and instead close any logger that implements io.Closer --- server.go | 8 ++------ server/server.go | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/server.go b/server.go index 11b2ee2a3..08923f13a 100644 --- a/server.go +++ b/server.go @@ -19,8 +19,6 @@ import ( "crypto/tls" "errors" "fmt" - "io" - "io/ioutil" "net" "net/http" "os" @@ -87,8 +85,7 @@ type Server struct { // Misc options. MaxWritesPerRequest int - LogOutput io.Writer - Logger Logger + Logger Logger defaultClient InternalClient } @@ -115,8 +112,7 @@ func NewServer() *Server { MetricInterval: 0, DiagnosticInterval: 0, - LogOutput: ioutil.Discard, - Logger: NopLogger, + Logger: NopLogger, } s.Handler.Holder = s.Holder diff --git a/server/server.go b/server/server.go index 8fe508fbd..2dcfd12f4 100644 --- a/server/server.go +++ b/server/server.go @@ -134,7 +134,6 @@ func (m *Command) SetupServer() error { if err != nil { return err } - m.Server.LogOutput = lw m.logger = log.New(lw, "", log.LstdFlags) if m.Config.Verbose { m.Server.Logger = pilosa.NewVerboseLogger(m.logger) @@ -299,8 +298,8 @@ func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) { func (m *Command) Close() error { var logErr error serveErr := m.Server.Close() - logOutput := m.Server.LogOutput - if closer, ok := logOutput.(io.Closer); ok { + logger := m.Server.Logger + if closer, ok := logger.(io.Closer); ok { logErr = closer.Close() } close(m.Done) From 36ce12da599a18209a23c8956d31564a2f7741e1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 23 Mar 2018 12:43:48 -0500 Subject: [PATCH 196/311] This commit adds a `Close()` method to the `pilosa.Logger` interface, and it moves the file handling (open/close) out of the main Command and into the interface implementation. The Logger implementations both have a `Logger()` method which returns their internal logger (`*log.Logger`). This is because the gossip setup (memberlist) needs a `*log.Logger` for its configuration. --- cmd/server.go | 9 +++--- logger.go | 77 +++++++++++++++++++++++++++++++++++++++++++----- server/server.go | 53 +++++++++++++++------------------ test/logger.go | 2 ++ 4 files changed, 98 insertions(+), 43 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 8089dae54..81633ea01 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -17,7 +17,6 @@ package cmd import ( "fmt" "io" - "log" "os" "os/signal" "runtime/pprof" @@ -46,11 +45,11 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr) - if err != nil { - return err + // Set up the logger. + if err := Server.SetupLogger(); err != nil { + return fmt.Errorf("error setting up the logger: %v", err) } - logger := log.New(logOutput, "", log.LstdFlags) + logger := Server.Server.Logger logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) // Start CPU profiling. diff --git a/logger.go b/logger.go index 2d3df732b..445bc5212 100644 --- a/logger.go +++ b/logger.go @@ -14,7 +14,11 @@ package pilosa -import "log" +import ( + "io" + "log" + "os" +) // Ensure nopLogger implements interface. var _ Logger = &nopLogger{} @@ -23,6 +27,7 @@ var _ Logger = &nopLogger{} type Logger interface { Printf(format string, v ...interface{}) Debugf(format string, v ...interface{}) + Close() error } func init() { @@ -40,15 +45,34 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {} // Debugf is a no-op implementation of the Logger Debugf method. func (n *nopLogger) Debugf(format string, v ...interface{}) {} +// Close is a no-op implementation of the Logger Close method. +func (n *nopLogger) Close() error { return nil } + // StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. type StandardLogger struct { logger *log.Logger + f *os.File } -func NewStandardLogger(logger *log.Logger) *StandardLogger { - return &StandardLogger{ - logger: logger, +func NewStandardLogger(path string, defaultWriter io.Writer) (*StandardLogger, error) { + var lw io.Writer + var err error + var f *os.File + + if path == "" { + lw = defaultWriter + } else { + f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + lw = f } + + return &StandardLogger{ + logger: log.New(lw, "", log.LstdFlags), + f: f, + }, nil } func (s *StandardLogger) Printf(format string, v ...interface{}) { @@ -57,15 +81,42 @@ func (s *StandardLogger) Printf(format string, v ...interface{}) { func (s *StandardLogger) Debugf(format string, v ...interface{}) {} +func (s *StandardLogger) Close() error { + if s.f == nil { + return nil + } + return s.f.Close() +} + +func (s *StandardLogger) Logger() *log.Logger { + return s.logger +} + // VerboseLogger is an implementation of pilosa.Logger which includes debug messages. type VerboseLogger struct { logger *log.Logger + f *os.File } -func NewVerboseLogger(logger *log.Logger) *VerboseLogger { - return &VerboseLogger{ - logger: logger, +func NewVerboseLogger(path string, defaultWriter io.Writer) (*VerboseLogger, error) { + var lw io.Writer + var err error + var f *os.File + + if path == "" { + lw = defaultWriter + } else { + f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + lw = f } + + return &VerboseLogger{ + logger: log.New(lw, "", log.LstdFlags), + f: f, + }, nil } func (vb *VerboseLogger) Printf(format string, v ...interface{}) { @@ -73,6 +124,16 @@ func (vb *VerboseLogger) Printf(format string, v ...interface{}) { } func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { - vb.logger.Printf("VERBOSE...") vb.logger.Printf(format, v...) } + +func (vb *VerboseLogger) Close() error { + if vb.f == nil { + return nil + } + return vb.f.Close() +} + +func (vb *VerboseLogger) Logger() *log.Logger { + return vb.logger +} diff --git a/server/server.go b/server/server.go index 2dcfd12f4..4141ae3aa 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,7 @@ type Command struct { // Done will be closed when Command.Close() is called Done chan struct{} + // Passed to the Gossip implementation. logger *log.Logger } @@ -122,6 +123,26 @@ func (m *Command) Run(args ...string) (err error) { return nil } +// SetupLogger sets up the logger based on the configuration. +func (m *Command) SetupLogger() error { + if m.Config.Verbose { + vbl, err := pilosa.NewVerboseLogger(m.Config.LogPath, m.Stderr) + if err != nil { + return err + } + m.logger = vbl.Logger() + m.Server.Logger = vbl + } else { + sl, err := pilosa.NewStandardLogger(m.Config.LogPath, m.Stderr) + if err != nil { + return err + } + m.logger = sl.Logger() + m.Server.Logger = sl + } + return nil +} + // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { err := m.Config.Validate() @@ -129,17 +150,6 @@ func (m *Command) SetupServer() error { return err } - // Set up logger based on configuration. - lw, err := GetLogWriter(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } - m.logger = log.New(lw, "", log.LstdFlags) - if m.Config.Verbose { - m.Server.Logger = pilosa.NewVerboseLogger(m.logger) - } else { - m.Server.Logger = pilosa.NewStandardLogger(m.logger) - } m.Server.Handler.Logger = m.Server.Logger m.Server.Holder.Logger = m.Server.Logger m.Server.Holder.Stats.SetLogger(m.Server.Logger) @@ -280,28 +290,11 @@ func (m *Command) SetupNetworking() error { return nil } -// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path. -func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) { - // This is split out so it can be used in NewServeCmd as well as SetupServer - if path == "" { - return defaultWriter, nil - } else { - logFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - return logFile, nil - } -} - // Close shuts down the server. func (m *Command) Close() error { - var logErr error serveErr := m.Server.Close() - logger := m.Server.Logger - if closer, ok := logger.(io.Closer); ok { - logErr = closer.Close() - } + logErr := m.Server.Logger.Close() + close(m.Done) if serveErr != nil && logErr != nil { return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr) diff --git a/test/logger.go b/test/logger.go index b4a0079b1..ae5984bb7 100644 --- a/test/logger.go +++ b/test/logger.go @@ -43,6 +43,8 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) { func (b *BufferLogger) Debugf(format string, v ...interface{}) {} +func (b *BufferLogger) Close() error { return nil } + func (b *BufferLogger) ReadAll() ([]byte, error) { return ioutil.ReadAll(b.buf) } From 5c52e48b1264de0ba776b8d874889e5954eb2c19 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 27 Mar 2018 10:23:08 -0500 Subject: [PATCH 197/311] change NewStandardLogger() to only take an io.Writer --- logger.go | 63 +++++------------------------------------------- server/server.go | 25 ++++++++++++------- test/logger.go | 2 -- 3 files changed, 22 insertions(+), 68 deletions(-) diff --git a/logger.go b/logger.go index 445bc5212..93b12c3f8 100644 --- a/logger.go +++ b/logger.go @@ -17,7 +17,6 @@ package pilosa import ( "io" "log" - "os" ) // Ensure nopLogger implements interface. @@ -27,7 +26,6 @@ var _ Logger = &nopLogger{} type Logger interface { Printf(format string, v ...interface{}) Debugf(format string, v ...interface{}) - Close() error } func init() { @@ -45,34 +43,15 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {} // Debugf is a no-op implementation of the Logger Debugf method. func (n *nopLogger) Debugf(format string, v ...interface{}) {} -// Close is a no-op implementation of the Logger Close method. -func (n *nopLogger) Close() error { return nil } - // StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. type StandardLogger struct { logger *log.Logger - f *os.File } -func NewStandardLogger(path string, defaultWriter io.Writer) (*StandardLogger, error) { - var lw io.Writer - var err error - var f *os.File - - if path == "" { - lw = defaultWriter - } else { - f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - lw = f - } - +func NewStandardLogger(w io.Writer) *StandardLogger { return &StandardLogger{ - logger: log.New(lw, "", log.LstdFlags), - f: f, - }, nil + logger: log.New(w, "", log.LstdFlags), + } } func (s *StandardLogger) Printf(format string, v ...interface{}) { @@ -81,13 +60,6 @@ func (s *StandardLogger) Printf(format string, v ...interface{}) { func (s *StandardLogger) Debugf(format string, v ...interface{}) {} -func (s *StandardLogger) Close() error { - if s.f == nil { - return nil - } - return s.f.Close() -} - func (s *StandardLogger) Logger() *log.Logger { return s.logger } @@ -95,28 +67,12 @@ func (s *StandardLogger) Logger() *log.Logger { // VerboseLogger is an implementation of pilosa.Logger which includes debug messages. type VerboseLogger struct { logger *log.Logger - f *os.File } -func NewVerboseLogger(path string, defaultWriter io.Writer) (*VerboseLogger, error) { - var lw io.Writer - var err error - var f *os.File - - if path == "" { - lw = defaultWriter - } else { - f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - lw = f - } - +func NewVerboseLogger(w io.Writer) *VerboseLogger { return &VerboseLogger{ - logger: log.New(lw, "", log.LstdFlags), - f: f, - }, nil + logger: log.New(w, "", log.LstdFlags), + } } func (vb *VerboseLogger) Printf(format string, v ...interface{}) { @@ -127,13 +83,6 @@ func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { vb.logger.Printf(format, v...) } -func (vb *VerboseLogger) Close() error { - if vb.f == nil { - return nil - } - return vb.f.Close() -} - func (vb *VerboseLogger) Logger() *log.Logger { return vb.logger } diff --git a/server/server.go b/server/server.go index 4141ae3aa..66725f4ee 100644 --- a/server/server.go +++ b/server/server.go @@ -74,7 +74,8 @@ type Command struct { Done chan struct{} // Passed to the Gossip implementation. - logger *log.Logger + logOutput io.Writer + logger *log.Logger } // NewCommand returns a new instance of Main. @@ -125,18 +126,22 @@ func (m *Command) Run(args ...string) (err error) { // SetupLogger sets up the logger based on the configuration. func (m *Command) SetupLogger() error { - if m.Config.Verbose { - vbl, err := pilosa.NewVerboseLogger(m.Config.LogPath, m.Stderr) + var err error + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { return err } + } + + if m.Config.Verbose { + vbl := pilosa.NewVerboseLogger(m.logOutput) m.logger = vbl.Logger() m.Server.Logger = vbl } else { - sl, err := pilosa.NewStandardLogger(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } + sl := pilosa.NewStandardLogger(m.logOutput) m.logger = sl.Logger() m.Server.Logger = sl } @@ -292,9 +297,11 @@ func (m *Command) SetupNetworking() error { // Close shuts down the server. func (m *Command) Close() error { + var logErr error serveErr := m.Server.Close() - logErr := m.Server.Logger.Close() - + if closer, ok := m.logOutput.(io.Closer); ok { + logErr = closer.Close() + } close(m.Done) if serveErr != nil && logErr != nil { return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr) diff --git a/test/logger.go b/test/logger.go index ae5984bb7..b4a0079b1 100644 --- a/test/logger.go +++ b/test/logger.go @@ -43,8 +43,6 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) { func (b *BufferLogger) Debugf(format string, v ...interface{}) {} -func (b *BufferLogger) Close() error { return nil } - func (b *BufferLogger) ReadAll() ([]byte, error) { return ioutil.ReadAll(b.buf) } From 96e2ef534ff4e88e929562b9ed33560af46128c0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Mar 2018 14:08:01 -0500 Subject: [PATCH 198/311] Clean up Makefile Turn off the automatic installation of build-time dependencies, which was confusing and inconsistent. Update PHONY list. Simplify git status check. Remove PKGS as Go 1.9+ automatically ignores `vendor` when running tests. Consolidate release/prerelease build code. Add comments. --- Makefile | 160 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 9eb2df7e6..ddd6cb1a5 100644 --- a/Makefile +++ b/Makefile @@ -1,113 +1,133 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test -DEP := $(shell command -v dep 2>/dev/null) -STATIK := $(shell command -v statik 2>/dev/null) -PROTOC := $(shell command -v protoc 2>/dev/null) -VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -STATUS := $(shell git status --porcelain) -IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa -PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) -BUILD_TIME=`date -u +%FT%T%z` +VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) +BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) +BUILD_TIME := $(shell date -u +%FT%T%z) LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" -DOCKER_GOLANG_IMAGE=golang:latest -GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH)) -BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH) +GO_VERSION=latest -default: test pilosa +# Run tests and compile Pilosa +default: test build +# Remove vendor and build directories clean: rm -rf vendor build -$(GOPATH)/bin: - mkdir $(GOPATH)/bin - -dep: $(GOPATH)/bin - go get -u github.com/golang/dep/cmd/dep - +# Set up vendor directory using `dep` vendor: Gopkg.toml -ifndef DEP - make dep -endif + $(MAKE) require-dep dep ensure touch vendor -Gopkg.lock: dep Gopkg.toml - dep ensure - +# Run test suite test: vendor - go test $(PKGS) $(TESTFLAGS) + go test ./... $(TESTFLAGS) +# Run test suite with coverage enabled cover: vendor - make test TESTFLAGS="-coverprofile=build/coverage.out" + $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" +# Run test suite with coverage enabled and view coverage results in browser cover-viz: cover go tool cover -html=build/coverage.out -pilosa: vendor - go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Compile Pilosa +build: vendor + go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Create a single release build under the build directory release-build: vendor -ifdef DOCKER_BUILD - make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -else - make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -endif - cp LICENSE README.md build/pilosa-$(IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ - @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" + $(MAKE) $(if DOCKER_BUILD,docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + cp LICENSE README.md build/pilosa-$(VERSION_ID) + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz -release: -ifeq ($(STATUS),"") - make release-build GOOS=darwin GOARCH=amd64 - make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 - make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 -else - @echo "Will not create release with unclean git status." -endif +# Error out if there are untracked changes in Git +check-clean: + $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) +# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. +release: check-clean + $(MAKE) release-build GOOS=darwin GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + $(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 + +# Create branch-tagged pre-release for client library CI jobs prerelease-build: vendor - make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa" - cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/ - @echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz" + $(MAKE) release-build VERSION_ID=$(BRANCH_ID) +# Create prerelease build for Linux/amd64 prerelease: - make prerelease-build GOOS=linux GOARCH=amd64 + $(MAKE) prerelease-build GOOS=linux GOARCH=amd64 +# Upload prerelease to S3 prerelease-upload: prerelease - aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read + aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read +# Install Pilosa install: vendor - go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -.protoc-gen-gofast: vendor -ifndef PROTOC - $(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases") -endif - go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast - cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast - -generate-protoc: .protoc-gen-gofast +# `go generate` protocol buffers +generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -generate-statik: statik +# `go generate` statik assets (WebUI) +generate-statik: require-statik go generate github.com/pilosa/pilosa/statik +# `go generate` all needed packages generate: generate-protoc generate-statik -statik: -ifndef STATIK - go get github.com/rakyll/statik -endif - +# Create Docker image from Dockerfile docker: docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . - @echo "Created image: pilosa:$(VERSION)" + @echo Created docker image: pilosa:$(VERSION) +# Compile Pilosa inside Docker container docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS) + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./... + +###################### +# Build dependencies # +###################### + +# Verifies that needed build dependency is installed. Errors out if not installed. +define require + $(if $(shell command -v $1 2>/dev/null), + $(info Verified build dependency "$1" is installed.), + $(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`)) +endef + +require-dep: + $(call require,dep) + +require-statik: + $(call require,statik) + +require-protoc-gen-gofast: + $(call require,protoc-gen-gofast) + +require-protoc: + $(call require,protoc) + +install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc + +install-dep: + go get -u github.com/golang/dep/cmd/dep + +install-statik: + go get -u github.com/rakyll/statik + +install-protoc-gen-gofast: + go get -u github.com/gogo/protobuf/protoc-gen-gofast + +install-protoc: + @echo This tool cannot automatically install protoc. Please download and install protoc from https://github.com/google/protobuf/releases From e601d694c0e4bd7f1508471d00581815a6472379 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Mar 2018 15:02:12 -0500 Subject: [PATCH 199/311] Document Makefile --- CONTRIBUTING.md | 66 ++++++++++++++++++++++++++++++++++++++++++++ docs/installation.md | 8 ++++++ 2 files changed, 74 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 402895f1d..0a29cb1b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,6 +66,72 @@ If you want to help but you aren't sure where to start, check out our [github la git remote add upstream git@github.com:pilosa/pilosa.git ``` +### Makefile + +Pilosa includes a Makefile that automates several tasks: + +- Install Pilosa: + + ```sh + make install + ``` + +- Create the vendor directory: + + ```sh + make vendor + ``` + +- Run the test suite: + + ```sh + make test + ``` + +- View the coverage report: + + ```sh + make cover-viz + ``` + +- Clear the `vendor/` and `build/` directories: + + ```sh + make clean + ``` + +- Create release tarballs: + + ```sh + make release + ``` + +- Generate static assets for the WebUI: + + ```sh + make generate-statik + ``` + +- Regenerate protocol buffer files in `internal/`: + + ```sh + make generate-protoc + ``` + +- Create tagged Docker image: + + ```sh + make docker + ``` + +- Run tests inside Docker container: + + ```sh + make docker-test + ``` + +Additional commands are available in the `Makefile`. + ### Submitting code changes - Before starting to work on a task, sync your branch with the upstream: diff --git a/docs/installation.md b/docs/installation.md index 9ca90ccf2..225e742ae 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -134,6 +134,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). @@ -288,6 +292,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). From 743726331fb893ffb7e23f68ba21195d91163253 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 28 Mar 2018 12:22:47 -0500 Subject: [PATCH 200/311] Add make instructions for travis-ci build process and use Makefile in Docker build --- .dockerignore | 1 - .travis.yml | 2 +- Dockerfile | 9 ++++----- Makefile | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9abb766ce..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 1a4691b8c..715f81a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" install: - - make vendor generate-statik + - make install-dep install-statik vendor generate-statik script: - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 diff --git a/Dockerfile b/Dockerfile index 50e88985c..145405bc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ FROM golang:1.10 as builder -ARG ldflags='' - -COPY . /go/src/github.com/pilosa/pilosa +COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && make vendor \ - && CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa + && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" + +RUN chmod +x /go/bin/pilosa FROM scratch diff --git a/Makefile b/Makefile index ddd6cb1a5..6094495e3 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ generate: generate-protoc generate-statik # Create Docker image from Dockerfile docker: - docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . + docker build -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) # Compile Pilosa inside Docker container From 9bf70705a3582e6b7c02af9c4dd3a77aa4c588b7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 28 Mar 2018 12:23:03 -0500 Subject: [PATCH 201/311] Add docs for "make install-build-deps" --- CONTRIBUTING.md | 6 ++++++ docs/installation.md | 2 ++ 2 files changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a29cb1b5..925f789b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,12 @@ Pilosa includes a Makefile that automates several tasks: make install ``` +- Install build dependencies (dep, statik, and protoc): + + ```sh + make install-build-deps + ``` + - Create the vendor directory: ```sh diff --git a/docs/installation.md b/docs/installation.md index 225e742ae..db0828b2f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -152,6 +152,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` @@ -310,6 +311,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` From d59d20d27bc86dff711e1a33134c005782aac9fb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 29 Mar 2018 10:34:34 -0500 Subject: [PATCH 202/311] Redundant chmod --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 145405bc4..bf076e526 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,6 @@ COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" -RUN chmod +x /go/bin/pilosa - FROM scratch LABEL maintainer "dev@pilosa.com" From 14dd54cc735c9d061d3cac2df008ac76b87b1787 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 29 Mar 2018 15:50:53 -0500 Subject: [PATCH 203/311] WebUI fixes to accommodate API changes --- webui/assets/main.js | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index 6a4415a5c..d38da454c 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -254,12 +254,12 @@ function populate_version() { xhr.onload = function() { var version = JSON.parse(xhr.responseText)['version'] - var version_major_minor = /(v\d+\.\d+)/.exec(version)[0] + var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1] var doc_link = document.getElementById('nav-documentation') doc_link.onclick = function() { - window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/') + window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/') } - node.innerHTML = version + node.innerHTML = "Pilosa v" + version } xhr.send(null) } @@ -295,12 +295,19 @@ function set_active_pane_by_name(name) { function update_cluster_status() { var xhr = new XMLHttpRequest(); xhr.open('GET', '/status') - status_node = document.getElementById('status') xhr.onload = function() { var status = JSON.parse(xhr.responseText) render_status(status) } xhr.send(null) + + var xhrSchema = new XMLHttpRequest(); + xhrSchema.open('GET', '/schema') + xhrSchema.onload = function() { + var schema = JSON.parse(xhrSchema.responseText) + render_schema(schema) + } + xhrSchema.send(null) } function render_status(status) { @@ -310,7 +317,7 @@ function render_status(status) { nodes_div.removeChild(nodes_div.firstChild); } - var nodes = status["status"]["Nodes"] + var nodes = status["nodes"] table = document.createElement("table") tbody = document.createElement("tbody") table.appendChild(tbody) @@ -320,31 +327,35 @@ function render_status(status) { var header = document.createElement('tr') markup = `Host - State` + ID + Coordinator` header.innerHTML = markup tbody.appendChild(header) for(var n=0; n${nodes[n]["Host"]} - ${nodes[n]["State"]}` + markup = `${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]} + ${nodes[n]["id"]} + ${nodes[n]["isCoordinator"]}` row.innerHTML = markup tbody.appendChild(row) } nodes_div.appendChild(table) +} +function render_schema(schema) { // render index tables var indexes_div = document.getElementById("status-indexes") while (indexes_div.firstChild) { indexes_div.removeChild(indexes_div.firstChild); } - var indexes = nodes[0]["Indexes"] // TODO currently comes from only node 0 + var indexes = schema["indexes"] // TODO currently comes from only node 0 for(var n=0; n${frames[m]["Name"]} - ${frames[m]["Meta"]["CacheType"]} - ${frames[m]["Meta"]["CacheSize"]}` + row.innerHTML = `${frames[m]["name"]} + ${frames[m]["options"]["cacheType"]} + ${frames[m]["options"]["cacheSize"]}` tbody.appendChild(row) } } @@ -583,4 +594,4 @@ function parse_options(option_str) { } } return options; -} \ No newline at end of file +} From e3cd03d902174524de1082cff72c609b9342b8f6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 2 Apr 2018 13:05:37 -0500 Subject: [PATCH 204/311] move rowLabel and columnLabel constants to executor.go --- executor.go | 65 ++++++++++++++++++++++++++++------------------------- frame.go | 1 - index.go | 1 - 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/executor.go b/executor.go index 0ed6dbaed..2154b60ff 100644 --- a/executor.go +++ b/executor.go @@ -33,6 +33,9 @@ const ( // MinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. MinThreshold = 1 + + columnLabel = "col" + rowLabel = "row" ) // Executor recursively executes calls in a PQL query across all slices. @@ -128,7 +131,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(DefaultRowLabel, DefaultColumnLabel) { + if call.IsInverse(rowLabel, columnLabel) { slices = inverseSlices } } @@ -262,7 +265,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { idx := e.Holder.Index(index) if idx != nil { - if columnID, ok, err := c.UintArg(DefaultColumnLabel); ok && err == nil { + if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err @@ -273,7 +276,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { frame, _ := c.Args["frame"].(string) if fr := idx.Frame(frame); fr != nil { - rowID, _, err := c.UintArg(DefaultRowLabel) + rowID, _, err := c.UintArg(rowLabel) if err != nil { return nil, err } @@ -529,15 +532,15 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. } // Return an error if both the row and column label are specified. - rowID, rowOK, rowErr := c.UintArg(DefaultRowLabel) - columnID, columnOK, columnErr := c.UintArg(DefaultColumnLabel) + rowID, rowOK, rowErr := c.UintArg(rowLabel) + columnID, columnOK, columnErr := c.UintArg(columnLabel) if rowErr != nil || columnErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) } if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, DefaultColumnLabel) + return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, DefaultColumnLabel) + return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel) } // Determine row or column orientation. @@ -604,11 +607,11 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Read row & column id. - columnID, columnOK, err := c.UintArg(DefaultColumnLabel) + columnID, columnOK, err := c.UintArg(columnLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) } - rowID, rowOK, err := c.UintArg(DefaultRowLabel) + rowID, rowOK, err := c.UintArg(rowLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } @@ -617,9 +620,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C var id uint64 var viewName string if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", DefaultColumnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel) } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", DefaultColumnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel) } else if columnOK { viewName, id = ViewInverse, columnID } else { @@ -893,18 +896,18 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal } // Read fields using labels. - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit() row field '%v' required", DefaultRowLabel) + return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) } - colID, ok, err := c.UintArg(DefaultColumnLabel) + colID, ok, err := c.UintArg(columnLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit col field '%v' required", DefaultColumnLabel) + return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } // Clear bits for each view. @@ -983,18 +986,18 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, } // Read fields using labels. - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { return false, fmt.Errorf("reading SetBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() row field '%v' required", DefaultRowLabel) + return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) } - colID, ok, err := c.UintArg(DefaultColumnLabel) + colID, ok, err := c.UintArg(columnLabel) if err != nil { return false, fmt.Errorf("reading SetBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() column field '%v' required", DefaultColumnLabel) + return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) } var timestamp *time.Time @@ -1080,11 +1083,11 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Parse labels. - columnID, ok, err := c.UintArg(DefaultColumnLabel) + columnID, ok, err := c.UintArg(columnLabel) if err != nil { return fmt.Errorf("reading SetFieldValue() column: %v", err) } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", DefaultColumnLabel) + return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) } // Copy args and remove reserved fields. @@ -1092,7 +1095,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq delete(args, "frame") // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. - delete(args, DefaultColumnLabel) + delete(args, columnLabel) // Set values. for name, value := range args { @@ -1146,17 +1149,17 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Parse labels. - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetRowAttrs() row field '%v' required", DefaultRowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required", rowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, DefaultRowLabel) + delete(attrs, rowLabel) // Set attributes. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { @@ -1205,17 +1208,17 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal return nil, ErrFrameNotFound } - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { - return nil, fmt.Errorf("reading SetRowAttrs() row: %v", DefaultRowLabel) + return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { - return nil, fmt.Errorf("SetRowAttrs row field '%v' required", DefaultRowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, DefaultRowLabel) + delete(attrs, rowLabel) // Create frame group, if not exists. frameMap := m[frame] @@ -1284,14 +1287,14 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return ErrIndexNotFound } - col, okCol, errCol := c.UintArg(DefaultColumnLabel) + col, okCol, errCol := c.UintArg(columnLabel) if errCol != nil || !okCol { return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, DefaultColumnLabel) + delete(attrs, columnLabel) delete(attrs, "frame") // Set attributes. diff --git a/frame.go b/frame.go index f786ac0b0..b5968a5d3 100644 --- a/frame.go +++ b/frame.go @@ -32,7 +32,6 @@ import ( // Default frame settings. const ( - DefaultRowLabel = "row" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false DefaultRangeEnabled = false diff --git a/index.go b/index.go index f17854e1e..0f0cad456 100644 --- a/index.go +++ b/index.go @@ -31,7 +31,6 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "col" InputDefinitionDir = ".input-definitions" ) From ba9c26e6895a3fe05a331dcbef47bc64a41a18ff Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 2 Apr 2018 16:39:14 -0500 Subject: [PATCH 205/311] Code review fixes: Create build directory if it doesn't exist. Use better URL for protoc installation instructions. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6094495e3..824d23cad 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ test: vendor # Run test suite with coverage enabled cover: vendor + mkdir -p build $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" # Run test suite with coverage enabled and view coverage results in browser @@ -130,4 +131,4 @@ install-protoc-gen-gofast: go get -u github.com/gogo/protobuf/protoc-gen-gofast install-protoc: - @echo This tool cannot automatically install protoc. Please download and install protoc from https://github.com/google/protobuf/releases + @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html From ce4fb3c4d347eb66bc0682ca27592e73c2708c87 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 3 Apr 2018 10:36:48 -0500 Subject: [PATCH 206/311] remove references to rowLabel and columnLabel --- docs/api-reference.md | 4 ++-- docs/query-language.md | 6 +----- handler_internal_test.go | 4 ++-- pilosa.go | 11 ----------- pilosa_test.go | 24 ------------------------ webui/assets/main.js | 4 ++-- webui/index.html | 4 ++-- 7 files changed, 9 insertions(+), 48 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 9092c92bc..446235cdc 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -215,7 +215,7 @@ The request payload is JSON, and it must contain the fields `frames` and `fields * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa. +The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa. * `name` (string): Maps the source data field to actions that process the field's corresponding value. * `actions` (array): List of actions that will process the field's value. @@ -310,7 +310,7 @@ Input definition is deprecated as of v0.9. Processes the JSON payload using the given input definition. -The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. +The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition. ``` request curl localhost:10101/index/user/input/stargazer-input \ diff --git a/docs/query-language.md b/docs/query-language.md index 0ccca3a80..6b0584320 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -21,8 +21,6 @@ This section will provide a detailed reference and examples for the Pilosa Query There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type. -The default row label is `rowID`, and the default column label is `columnID`. Changing these defaults is deprecated and this feature will be removed in a future release. - #### Conventions * Angle Brackets `<>` denote required arguments @@ -46,8 +44,6 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types * `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. -* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated. -* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated. * `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` @@ -223,7 +219,7 @@ Bitmap(, ( | =UINT)) **Description:** -`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. +`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column. **Result Type:** object with attrs and bits. diff --git a/handler_internal_test.go b/handler_internal_test.go index 395ec73b6..4603662e6 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -30,7 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"columnLabel": "test"}}`, err: "Unknown key: columnLabel:test"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, } for _, test := range tests { actual := &postIndexRequest{} @@ -65,7 +65,7 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"rowLabel": "test"}}`, err: "Unknown key: rowLabel:test"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, diff --git a/pilosa.go b/pilosa.go index a3e11b826..8f1ae287e 100644 --- a/pilosa.go +++ b/pilosa.go @@ -77,9 +77,6 @@ var ( // Regular expression to validate index and frame names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) -// Regular expression to validate row and column labels. -var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`) - // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { @@ -141,14 +138,6 @@ func ValidateName(name string) error { return nil } -// ValidateLabel ensures that the label is a valid format. -func ValidateLabel(label string) error { - if labelRegexp.Match([]byte(label)) == false { - return ErrLabel - } - return nil -} - // StringInSlice checks for substring a in the slice. func StringInSlice(a string, list []string) bool { for _, b := range list { diff --git a/pilosa_test.go b/pilosa_test.go index d82e27fa3..41b0d7098 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -46,30 +46,6 @@ func TestValidateNameInvalid(t *testing.T) { } } -func TestValidateLabel(t *testing.T) { - labels := []string{ - "a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, label := range labels { - if pilosa.ValidateLabel(label) != nil { - t.Fatalf("Should be valid label: %s", label) - } - } -} - -func TestValidateLabelInvalid(t *testing.T) { - labels := []string{ - "", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, label := range labels { - if pilosa.ValidateLabel(label) == nil { - t.Fatalf("Should be invalid label: %s", label) - } - } -} - func TestStringInSlice(t *testing.T) { list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} substr := "localhost:10101" diff --git a/webui/assets/main.js b/webui/assets/main.js index 6a4415a5c..c0a24bdf6 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -472,7 +472,7 @@ class Autocompleter { } init_dynamic_keywords() { - // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list + // hit /schema, parse indexes, frames, add to list } add_keyword() { @@ -583,4 +583,4 @@ function parse_options(option_str) { } } return options; -} \ No newline at end of file +} diff --git a/webui/index.html b/webui/index.html index dc06e310a..75eaa1d5a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -55,9 +55,9 @@
Special commands
- :create index test [columnLabel=column]
+ :create index test
:use test
- :create frame foo [rowLabel=row]
+ :create frame foo
:delete index test
:delete frame foo
From 4b0da099b58e8aaf03aa6f0a793a33171c9c14c0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 3 Apr 2018 11:09:49 -0500 Subject: [PATCH 207/311] Fix syntax on conditional check for docker-build config --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 824d23cad..9e2c6fa3f 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ build: vendor # Create a single release build under the build directory release-build: vendor - $(MAKE) $(if DOCKER_BUILD,docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" cp LICENSE README.md build/pilosa-$(VERSION_ID) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz From 0bf5f089a0e14677f4b3806f1d8f114c526ca76b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 3 Apr 2018 11:25:54 -0500 Subject: [PATCH 208/311] Run prerelease on Go 1.10 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 715f81a6e..15c511e78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ deploy: skip_cleanup: true on: all_branches: true - go: 1.9 + go: "1.10" matrix: allow_failures: - go: master From 6c3f3ac0f092340338604863d19388acf9bb8a98 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 1 Mar 2018 17:39:14 +0300 Subject: [PATCH 209/311] Added API struct; moved query logic to API --- api.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++ handler.go | 47 ++++--------------------- server.go | 5 +++ 3 files changed, 113 insertions(+), 40 deletions(-) create mode 100644 api.go diff --git a/api.go b/api.go new file mode 100644 index 000000000..5ab4a36aa --- /dev/null +++ b/api.go @@ -0,0 +1,101 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "context" + "strings" + + "github.com/pilosa/pilosa/pql" +) + +type QueryOptions struct { + Remote bool + ExcludeAttrs bool + ExcludeBits bool + ColumnAttrs bool +} + +type API struct { + holder *Holder + // The execution engine for running queries. + executor interface { + Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + } +} + +func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + resp := QueryResponse{} + + q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + if err != nil { + // TODO: Wrap + return resp, err + } + execOpts := &ExecOptions{ + Remote: req.Remote, + ExcludeAttrs: req.ExcludeAttrs, + ExcludeBits: req.ExcludeBits, + } + results, err := a.executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + if err != nil { + return resp, err + } + resp.Results = results + + // Fill column attributes if requested. + if req.ColumnAttrs && !req.ExcludeBits { + // Consolidate all column ids across all calls. + var columnIDs []uint64 + for _, result := range results { + bm, ok := result.(*Bitmap) + if !ok { + continue + } + columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) + } + + // Retrieve column attributes across all calls. + columnAttrSets, err := a.readColumnAttrSets(a.holder.Index(req.Index), columnIDs) + if err != nil { + return resp, err + } + resp.ColumnAttrSets = columnAttrSets + } + return resp, nil +} + +// readColumnAttrSets returns a list of column attribute objects by id. +func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { + return nil, nil + } + + ax := make([]*ColumnAttrSet, 0, len(ids)) + for _, id := range ids { + // Read attributes for column. Skip column if empty. + attrs, err := index.ColumnAttrStore().Attrs(id) + if err != nil { + return nil, err + } else if len(attrs) == 0 { + continue + } + + // Append column with attributes. + ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) + } + + return ax, nil +} diff --git a/handler.go b/handler.go index ffd28da46..c40148b43 100644 --- a/handler.go +++ b/handler.go @@ -69,6 +69,8 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec + + api *API } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -328,8 +330,6 @@ type getStatusResponse struct { // handlePostQuery handles /query requests. func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - // Parse incoming request. req, err := h.readQueryRequest(r) if err != nil { @@ -337,46 +337,13 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) return } + // TODO: Remove + req.Index = mux.Vars(r)["index"] - // Build execution options. - opt := &ExecOptions{ - Remote: req.Remote, - ExcludeAttrs: req.ExcludeAttrs, - ExcludeBits: req.ExcludeBits, - } - - // Parse query string. - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + resp, err := h.api.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - - // Execute the query. - results, err := h.Executor.Execute(r.Context(), indexName, q, req.Slices, opt) - resp := &QueryResponse{Results: results, Err: err} - - // Fill column attributes if requested. - if req.ColumnAttrs && !req.ExcludeBits { - // Consolidate all column ids across all calls. - var columnIDs []uint64 - for _, result := range results { - bm, ok := result.(*Bitmap) - if !ok { - continue - } - columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) - } - - // Retrieve column attributes across all calls. - columnAttrSets, err := h.readColumnAttrSets(h.Holder.Index(indexName), columnIDs) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - resp.ColumnAttrSets = columnAttrSets + h.writeQueryResponse(w, r, &resp) } // Set appropriate status code, if there is an error. @@ -390,7 +357,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Write response back to client. - if err := h.writeQueryResponse(w, r, resp); err != nil { + if err := h.writeQueryResponse(w, r, &resp); err != nil { h.Logger.Printf("write query response error: %s", err) } } diff --git a/server.go b/server.go index 08923f13a..ede265e19 100644 --- a/server.go +++ b/server.go @@ -117,6 +117,10 @@ func NewServer() *Server { s.Handler.Holder = s.Holder s.diagnostics.server = s + s.Handler.api = &API{ + holder: s.Holder, + } + return s } @@ -172,6 +176,7 @@ func (s *Server) Open() error { s.Handler.Executor = e s.Cluster.prefect = s.Handler + s.Handler.api.executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster From 2ebc1914be0865b074520cda3bfd0a21e897d8c7 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 2 Mar 2018 18:40:50 +0300 Subject: [PATCH 210/311] More API functions --- api.go | 133 ++++++++++++++++++++++++++++++++++++++++++++++++----- handler.go | 112 +++++++++++--------------------------------- server.go | 8 ++-- 3 files changed, 150 insertions(+), 103 deletions(-) diff --git a/api.go b/api.go index 5ab4a36aa..e004821fb 100644 --- a/api.go +++ b/api.go @@ -16,23 +16,31 @@ package pilosa import ( "context" + "fmt" + "io/ioutil" + "log" "strings" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) -type QueryOptions struct { - Remote bool - ExcludeAttrs bool - ExcludeBits bool - ColumnAttrs bool +type API struct { + Holder *Holder + // The execution engine for running queries. + Executor interface { + Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + } + Broadcaster Broadcaster + logger *log.Logger } -type API struct { - holder *Holder - // The execution engine for running queries. - executor interface { - Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) +func NewAPI(logger *log.Logger) *API { + if logger == nil { + logger = log.New(ioutil.Discard, "", 0) + } + return &API{ + logger: logger, } } @@ -49,7 +57,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons ExcludeAttrs: req.ExcludeAttrs, ExcludeBits: req.ExcludeBits, } - results, err := a.executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := a.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) if err != nil { return resp, err } @@ -68,7 +76,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons } // Retrieve column attributes across all calls. - columnAttrSets, err := a.readColumnAttrSets(a.holder.Index(req.Index), columnIDs) + columnAttrSets, err := a.readColumnAttrSets(a.Holder.Index(req.Index), columnIDs) if err != nil { return resp, err } @@ -99,3 +107,104 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet return ax, nil } + +func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + // Create index. + index, err := api.Holder.CreateIndex(indexName, options) + if err != nil { + return nil, err + } + // Send the create index message to all nodes. + err = api.Broadcaster.SendSync( + &internal.CreateIndexMessage{ + Index: indexName, + Meta: options.Encode(), + }) + if err != nil { + api.logger.Printf("problem sending CreateIndex message: %s", err) + return nil, err + } + api.Holder.Stats.Count("createIndex", 1, 1.0) + return index, nil +} + +func (api *API) ReadIndex(ctx context.Context, indexName string) (*Index, error) { + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + return index, nil +} + +func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + // Delete index from the holder. + err := api.Holder.DeleteIndex(indexName) + if err != nil { + return err + } + // Send the delete index message to all nodes. + err = api.Broadcaster.SendSync( + &internal.DeleteIndexMessage{ + Index: indexName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteIndex message: %s", err) + return err + } + api.Holder.Stats.Count("deleteIndex", 1, 1.0) + return nil +} + +func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Create frame. + frame, err := index.CreateFrame(frameName, options) + if err != nil { + return nil, err + } + + // Send the create frame message to all nodes. + err = api.Broadcaster.SendSync( + &internal.CreateFrameMessage{ + Index: indexName, + Frame: frameName, + Meta: options.Encode(), + }) + if err != nil { + api.logger.Printf("problem sending CreateFrame message: %s", err) + return nil, err + } + api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return frame, nil +} + +func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Delete frame from the index. + if err := index.DeleteFrame(frameName); err != nil { + return err + } + + // Send the delete frame message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteFrameMessage{ + Index: indexName, + Frame: frameName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteFrame message: %s", err) + return err + } + api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return nil +} diff --git a/handler.go b/handler.go index c40148b43..bd47524ed 100644 --- a/handler.go +++ b/handler.go @@ -70,7 +70,7 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec - api *API + API *API } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -340,7 +340,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // TODO: Remove req.Index = mux.Vars(r)["index"] - resp, err := h.api.ExecuteQuery(r.Context(), req) + resp, err := h.API.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) h.writeQueryResponse(w, r, &resp) @@ -385,9 +385,9 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + index, err := h.API.ReadIndex(r.Context(), indexName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) return } @@ -469,28 +469,17 @@ type postIndexResponse struct{} // handleDeleteIndex handles DELETE /index request. func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - - // Delete index from the holder. - if err := h.Holder.DeleteIndex(indexName); err != nil { + err := h.API.DeleteIndex(r.Context(), indexName) + if err != nil { + h.Logger.Printf("problem deleting index: %s", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete index message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteIndexMessage{ - Index: indexName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteIndex message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.Count("deleteIndex", 1, 1.0) } type deleteIndexResponse struct{} @@ -510,8 +499,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - // Create index. - _, err = h.Holder.CreateIndex(indexName, req.Options) + _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) if err == ErrIndexExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -520,24 +508,10 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - // Send the create index message to all nodes. - err = h.Broadcaster.SendSync( - &internal.CreateIndexMessage{ - Index: indexName, - Meta: req.Options.Encode(), - }) - if err != nil { - h.Logger.Printf("problem sending CreateIndex message: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - // Encode response. if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.Count("createIndex", 1, 1.0) } // handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. @@ -655,41 +629,22 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Create frame. - _, err = index.CreateFrame(frameName, req.Options) - if err == ErrFrameExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Send the create frame message to all nodes. - err = h.Broadcaster.SendSync( - &internal.CreateFrameMessage{ - Index: indexName, - Frame: frameName, - Meta: req.Options.Encode(), - }) + _, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options) if err != nil { - h.Logger.Printf("problem sending CreateFrame message: %s", err) + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrFrameExists: + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return } - // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) } type _postFrameRequest postFrameRequest @@ -742,37 +697,22 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) + err := h.API.DeleteFrame(r.Context(), indexName, frameName) + if err != nil { + if err == ErrIndexNotFound { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } + return } - return - } - - // Delete frame from the index. - if err := index.DeleteFrame(frameName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete frame message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteFrameMessage{ - Index: indexName, - Frame: frameName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteFrame message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) } type deleteFrameResponse struct{} diff --git a/server.go b/server.go index ede265e19..1973f746e 100644 --- a/server.go +++ b/server.go @@ -117,10 +117,7 @@ func NewServer() *Server { s.Handler.Holder = s.Holder s.diagnostics.server = s - s.Handler.api = &API{ - holder: s.Holder, - } - + s.Handler.API = NewAPI(s.logger) return s } @@ -169,6 +166,7 @@ func (s *Server) Open() error { // Initialize HTTP handler. s.Handler.Broadcaster = s.Broadcaster + s.Handler.API.Broadcaster = s.Broadcaster s.Handler.BroadcastHandler = s s.Handler.StatusHandler = s s.Handler.Node = node @@ -176,7 +174,7 @@ func (s *Server) Open() error { s.Handler.Executor = e s.Cluster.prefect = s.Handler - s.Handler.api.executor = e + s.Handler.API.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster From b702f70610962341116d9678ce7c5877b7e171ca Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 5 Mar 2018 18:28:24 +0300 Subject: [PATCH 211/311] Moved more of handler to API --- api.go | 464 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 462 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index e004821fb..e544d2d3f 100644 --- a/api.go +++ b/api.go @@ -16,11 +16,17 @@ package pilosa import ( "context" + "encoding/csv" "fmt" + "io" "io/ioutil" "log" + "net/http" + "reflect" + "strconv" "strings" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -31,8 +37,13 @@ type API struct { Executor interface { Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } - Broadcaster Broadcaster - logger *log.Logger + Broadcaster Broadcaster + BroadcastHandler BroadcastHandler + StatusHandler StatusHandler + Cluster *Cluster + URI *URI + RemoteClient *http.Client + logger *log.Logger } func NewAPI(logger *log.Logger) *API { @@ -208,3 +219,452 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } + +func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + // Find the fragment. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return ErrFragmentNotFound + } + + // Wrap writer with a CSV writer. + cw := csv.NewWriter(w) + + // Iterate over each bit. + if err := f.ForEachBit(func(rowID, columnID uint64) error { + return cw.Write([]string{ + strconv.FormatUint(rowID, 10), + strconv.FormatUint(columnID, 10), + }) + }); err != nil { + return err + } + + // Ensure data is flushed. + cw.Flush() + + return nil +} + +func (api *API) FragmentNodes(ctx context.Context, indexName string, slice uint64) []*Node { + return api.Cluster.FragmentNodes(indexName, slice) +} + +func (api *API) FragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (*Fragment, error) { + // Retrieve fragment from holder. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return nil, ErrFragmentNotFound + } + return f, nil +} + +func (api *API) WriteFragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Retrieve view. + view, err := f.CreateViewIfNotExists(viewName) + if err != nil { + return err + } + + // Retrieve fragment from frame. + frag, err := view.CreateFragmentIfNotExists(slice) + if err != nil { + return err + } + + // Read fragment in from request body. + if _, err := frag.ReadFrom(reader); err != nil { + return err + } + return nil +} + +func (api *API) FragmentBlockData(ctx context.Context, req internal.BlockDataRequest) (internal.BlockDataResponse, error) { + // Retrieve fragment from holder. + f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) + if f == nil { + return internal.BlockDataResponse{}, ErrFragmentNotFound + } + + // Read data + var resp internal.BlockDataResponse + resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + return resp, nil +} + +func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { + // Retrieve fragment from holder. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return nil, ErrFragmentNotFound + } + + // Retrieve blocks. + blocks := f.Blocks() + return blocks, nil +} + +func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { + // Create a client for the remote cluster. + client := NewInternalHTTPClientFromURI(host, api.RemoteClient) + + // Determine the maximum number of slices. + maxSlices, err := client.MaxSliceByIndex(ctx) + if err != nil { + return err + } + + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Retrieve list of all views. + views, err := client.FrameViews(ctx, indexName, frameName) + if err != nil { + return err + } + + // Loop over each slice and import it if this node owns it. + for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { + // Ignore this slice if we don't own it. + if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + continue + } + + // Loop over view names. + for _, view := range views { + // Create view. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + return err + } + + // Otherwise retrieve the local fragment. + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return err + } + + // Stream backup from remote node. + rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice) + if err != nil { + return err + } else if rd == nil { + continue // slice doesn't exist + } + + // Restore to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + return err + } + } + } + + return nil +} + +func (api *API) ClusterHosts(ctx context.Context) []*Node { + return api.Cluster.Nodes +} + +func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + if err := inputDef.Validate(); err != nil { + return err + } + + // Encode InputDefinition to its internal representation. + def := inputDef.Encode() + def.Name = inputDefName + + // Create InputDefinition. + if _, err := index.CreateInputDefinition(def); err != nil { + return err + } + + err := api.Broadcaster.SendSync( + &internal.CreateInputDefinitionMessage{ + Index: indexName, + Definition: def, + }) + if err != nil { + api.logger.Printf("problem sending CreateInputDefinition message: %s", err) + } + return nil +} + +func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + inputDef, err := index.InputDefinition(inputDefName) + if err != nil { + return nil, err + } + return inputDef, nil +} + +func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Delete input definition from the index. + if err := index.DeleteInputDefinition(inputDefName); err != nil { + return err + } + + err := api.Broadcaster.SendSync( + &internal.DeleteInputDefinitionMessage{ + Index: indexName, + Name: inputDefName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteInputDefinition message: %s", err) + } + return nil +} + +func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + for _, req := range reqs { + bits, err := api.inputJSONDataParser(req.(map[string]interface{}), index, inputDefName) + if err != nil { + return err + } + for fr, bs := range bits { + if err := index.InputBits(fr, bs); err != nil { + return err + } + } + } + + return nil +} + +func (api *API) RecalculateCaches(ctx context.Context) { + api.Holder.RecalculateCaches() +} + +func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error { + // Forward the error message. + if err := api.BroadcastHandler.ReceiveMessage(pb); err != nil { + return err + } + return nil +} + +func (api *API) LocalID(ctx context.Context) string { + return api.Holder.LocalID +} + +func (api *API) Schema(ctx context.Context) []*IndexInfo { + return api.Holder.Schema() +} + +func (api *API) Status(ctx context.Context) (proto.Message, error) { + return api.StatusHandler.ClusterStatus() +} + +func (api *API) CreateFrameField(ctx context.Context, indexName string, frameName string, field *Field) error { + // Retrieve frame by name. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Create new field. + if err := f.CreateField(field); err != nil { + return err + } + + // Send the create field message to all nodes. + err := api.Broadcaster.SendSync( + &internal.CreateFieldMessage{ + Index: indexName, + Frame: frameName, + Field: encodeField(field), + }) + if err != nil { + api.logger.Printf("problem sending CreateField message: %s", err) + } + return err +} + +func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameName string, fieldName string) error { + // Retrieve frame by name. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Delete field. + if err := f.DeleteField(fieldName); err != nil { + return err + } + + // Send the delete field message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteFieldMessage{ + Index: indexName, + Frame: frameName, + Field: fieldName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteField message: %s", err) + } + return err +} + +func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) (*FrameSchema, error) { + index := api.Holder.index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + frame := index.frame(frameName) + if frame == nil { + return nil, ErrFrameNotFound + } + + return frame.GetFields() +} + +func (api *API) FrameViews(ctx context.Context, indexName string, frameName string) ([]*View, error) { + // Retrieve views. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return nil, ErrFrameNotFound + } + + // Fetch views. + views := f.Views() + return views, nil +} + +func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Delete the view. + if err := f.DeleteView(viewName); err != nil { + // Ingore this error becuase views do not exist on all nodes due to slice distribution. + if err != ErrInvalidView { + return err + } + } + + // Send the delete view message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteViewMessage{ + Index: indexName, + Frame: frameName, + View: viewName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteView message: %s", err) + } + + return err +} + +// InputJSONDataParser validates input json file and executes SetBit. +func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { + inputDef, err := index.InputDefinition(name) + if err != nil { + return nil, err + } + // If field in input data is not in defined definition, return error. + var colValue uint64 + validFields := make(map[string]bool) + timestampFrame := make(map[string]int64) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true + if field.PrimaryKey { + value, ok := req[field.Name] + if !ok { + return nil, fmt.Errorf("primary key does not exist") + } + rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float + if !ok { + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + } + colValue = uint64(rawValue) + } + // Find frame that need to add timestamp. + for _, action := range field.Actions { + if action.ValueDestination == InputSetTimestamp { + timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) + if err != nil { + return nil, err + } + } + } + } + + for key := range req { + _, ok := validFields[key] + if !ok { + return nil, fmt.Errorf("field not found: %s", key) + } + } + + setBits := make(map[string][]*Bit) + + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + if _, ok := req[field.Name]; !ok { + continue + } + + // Looking into timestampFrame map and set timestamp to the whole frame + for _, action := range field.Actions { + frame := action.Frame + timestamp := timestampFrame[action.Frame] + // Skip input data field values that are set to null + if req[field.Name] == nil { + continue + } + bit, err := HandleAction(action, req[field.Name], colValue, timestamp) + if err != nil { + return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) + } + if bit != nil { + setBits[frame] = append(setBits[frame], bit) + } + } + } + return setBits, nil +} From 72526f753ce0d05033df6e1dbade86d291212fca Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 5 Mar 2018 18:28:24 +0300 Subject: [PATCH 212/311] Moved more of handler to API --- api.go | 2 +- handler.go | 478 ++++++++++++++++------------------------------------- 2 files changed, 141 insertions(+), 339 deletions(-) diff --git a/api.go b/api.go index e544d2d3f..84c2f5146 100644 --- a/api.go +++ b/api.go @@ -484,7 +484,7 @@ func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error } func (api *API) LocalID(ctx context.Context) string { - return api.Holder.LocalID + return api.Cluster.Node.ID } func (api *API) Schema(ctx context.Context) []*IndexInfo { diff --git a/handler.go b/handler.go index bd47524ed..f9fc73c52 100644 --- a/handler.go +++ b/handler.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "encoding/csv" "encoding/json" "errors" "expvar" @@ -44,21 +43,7 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Holder *Holder - Broadcaster Broadcaster - BroadcastHandler BroadcastHandler - StatusHandler StatusHandler - - FileSystem FileSystem - - // Local hostname & cluster configuration. - Node *Node - Cluster *Cluster - RemoteClient *http.Client - - Router *mux.Router - NormalRouter *mux.Router - RestrictedRouter *mux.Router + Router *mux.Router // The execution engine for running queries. Executor interface { @@ -295,8 +280,9 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { // handleGetSchema handles GET /schema requests. func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + schema := h.API.Schema(r.Context()) if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: h.Holder.Schema(), + Indexes: schema, }); err != nil { h.Logger.Printf("write schema response error: %s", err) } @@ -304,7 +290,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - pb, err := h.StatusHandler.ClusterStatus() + status, err := h.API.Status(r.Context()) if err != nil { h.Logger.Printf("cluster status error: %s", err) return @@ -774,13 +760,6 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { return } - // Retrieve frame by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - field := &Field{ Name: fieldName, Type: req.Type, @@ -788,23 +767,15 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { Max: req.Max, } - // Create new field. - if err := f.CreateField(field); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err := h.API.CreateFrameField(r.Context(), indexName, frameName, field); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Send the create field message to all nodes. - err := h.Broadcaster.SendSync( - &internal.CreateFieldMessage{ - Index: indexName, - Frame: frameName, - Field: encodeField(field), - }) - if err != nil { - h.Logger.Printf("problem sending CreateField message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) @@ -825,30 +796,15 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) frameName := mux.Vars(r)["frame"] fieldName := mux.Vars(r)["field"] - // Retrieve frame by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + if err := h.API.DeleteFrameField(r.Context(), indexName, frameName, fieldName); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Delete field. - if err := f.DeleteField(fieldName); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Send the delete field message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteFieldMessage{ - Index: indexName, - Frame: frameName, - Field: fieldName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteField message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) @@ -859,24 +815,18 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - index := h.Holder.index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - frame := index.frame(frameName) - if frame == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - fields, err := frame.GetFields() - if err == ErrFrameFieldsNotAllowed { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + schema, err := h.API.FrameFields(r.Context(), indexName, frameName) + if err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrFrameFieldsNotAllowed: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -899,15 +849,16 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Retrieve views. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + views, err := h.API.FrameViews(r.Context(), indexName, frameName) + if err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Fetch views. - views := f.Views() names := make([]string, len(views)) for i := range views { names[i] = views[i].Name() @@ -925,31 +876,13 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { frameName := mux.Vars(r)["frame"] viewName := mux.Vars(r)["view"] - // Retrieve frame. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Delete the view. - if err := f.DeleteView(viewName); err != nil { - // Ingore this error because views do not exist on all nodes due to slice distribution. - if err != ErrInvalidView { + if err := h.API.DeleteView(r.Context(), indexName, frameName, viewName); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { http.Error(w, err.Error(), http.StatusBadRequest) - return } - } - - // Send the delete view message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteViewMessage{ - Index: indexName, - Frame: frameName, - View: viewName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteView message: %s", err) + return } // Encode response. @@ -1314,28 +1247,13 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - // Find the fragment. - f := h.Holder.Fragment(index, frame, view, slice) - if f == nil { - return + if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } - - // Wrap writer with a CSV writer. - cw := csv.NewWriter(w) - - // Iterate over each bit. - if err := f.ForEachBit(func(rowID, columnID uint64) error { - return cw.Write([]string{ - strconv.FormatUint(rowID, 10), - strconv.FormatUint(columnID, 10), - }) - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Ensure data is flushed. - cw.Flush() } // handleGetFragmentNodes handles /fragment/nodes requests. @@ -1351,7 +1269,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.Cluster.FragmentNodes(index, slice) + nodes := h.API.FragmentNodes(r.Context(), index, slice) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1370,9 +1288,9 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if f == nil { - http.Error(w, "fragment not found", http.StatusNotFound) + f, err := h.API.FragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) return } @@ -1392,31 +1310,12 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } - // Retrieve frame. - f := h.Holder.Frame(q.Get("index"), q.Get("frame")) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve view. - view, err := f.CreateViewIfNotExists(q.Get("view")) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Retrieve fragment from frame. - frag, err := view.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read fragment in from request body. - if _, err := frag.ReadFrom(r.Body); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + if err = h.API.WriteFragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { + if err == ErrFrameNotFound { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } } @@ -1432,19 +1331,16 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ return } - // Retrieve fragment from holder. - f := h.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) - if f == nil { - http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) + resp, err := h.API.FragmentBlockData(r.Context(), req) + if err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Read data - var resp internal.BlockDataResponse - if f != nil { - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) - } - // Encode response. buf, err := proto.Marshal(&resp) if err != nil { @@ -1468,16 +1364,16 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if f == nil { - http.Error(w, "fragment not found", http.StatusNotFound) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + if err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Retrieve blocks. - blocks := f.Blocks() - // Encode response. if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ Blocks: blocks, @@ -1509,80 +1405,23 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) http.Error(w, err.Error(), http.StatusBadRequest) } - // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, h.RemoteClient) - - // Determine the maximum number of slices. - maxSlices, err := client.MaxSliceByIndex(r.Context()) - if err != nil { - http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError) - return - } - - // Retrieve frame. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve list of all views. - views, err := client.FrameViews(r.Context(), indexName, frameName) - if err != nil { - http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError) - return - } - - // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { - // Ignore this slice if we don't own it. - if !h.Cluster.OwnsFragment(h.Node.ID, indexName, slice) { - continue - } - - // Loop over view names. - for _, view := range views { - // Create view. - v, err := f.CreateViewIfNotExists(view) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Otherwise retrieve the local fragment. - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(r.Context(), indexName, frameName, view, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return err - } - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } + err = h.API.RestoreFrame(r.Context(), indexName, frameName, host) + switch err { + case nil: + break + case ErrFrameNotFound: + fallthrough + case ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) } } // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { + hosts := h.API.ClusterHosts(r.Context()) + if err := json.NewEncoder(w).Encode(hosts); err != nil { h.Logger.Printf("write version response error: %s", err) } } @@ -1766,13 +1605,6 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - // Decode request. var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) @@ -1781,36 +1613,26 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - if err := req.Validate(); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Encode InputDefinition to its internal representation. - def := req.Encode() - def.Name = inputDefName - - // Create InputDefinition. - _, err = index.CreateInputDefinition(def) - if err == ErrInputDefinitionExists { + err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req) + switch err { + case nil: + break + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrInputDefinitionExists: http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { + case ErrInputDefinitionAttrsRequired: + fallthrough + case ErrInputDefinitionNameRequired: + fallthrough + case ErrInputDefinitionActionRequired: + fallthrough + case ErrInputDefinitionHasPrimaryKey: + fallthrough + case ErrInputDefinitionDupePrimaryKey: + http.Error(w, err.Error(), http.StatusBadRequest) + default: http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = h.Broadcaster.SendSync( - &internal.CreateInputDefinitionMessage{ - Index: indexName, - Definition: def, - }) - if err != nil { - h.Logger.Printf("problem sending CreateInputDefinition message: %s", err) - } - - if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) } } @@ -1819,16 +1641,18 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - inputDef, err := index.InputDefinition(inputDefName) + inputDef, err := h.API.InputDefinition(r.Context(), indexName, inputDefName) if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1838,7 +1662,6 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques }); err != nil { h.Logger.Printf("write status response error: %s", err) } - } // handleDeleteInputDefinition handles DELETE /input-definition request. @@ -1846,28 +1669,20 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + if err := h.API.DeleteInputDefinition(r.Context(), indexName, inputDefName); err != nil { + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusNotFound) + } return } - // Delete input definition from the index. - if err := index.DeleteInputDefinition(inputDefName); err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - err := h.Broadcaster.SendSync( - &internal.DeleteInputDefinitionMessage{ - Index: indexName, - Name: inputDefName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) - } - if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } @@ -1879,13 +1694,6 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - // Decode request. var reqs []interface{} err := json.NewDecoder(r.Body).Decode(&reqs) @@ -1893,28 +1701,27 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - for _, req := range reqs { - bits, err := h.InputJSONDataParser(req.(map[string]interface{}), index, inputDefName) - if err == ErrInputDefinitionNotFound { + + if err = h.API.WriteInput(r.Context(), indexName, inputDefName, reqs); err != nil { + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: http.Error(w, err.Error(), http.StatusNotFound) - return - } else if err != nil { + default: http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for fr, bs := range bits { - err := index.InputBits(fr, bs) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } } + return } + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } } +// <<<<<<< b702f70610962341116d9678ce7c5877b7e171ca // handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request. func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -2109,14 +1916,11 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return setBits, nil } +// ======= +// >>>>>>> Moved more of handler to API + func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.Broadcaster.SendSync(&internal.RecalculateCaches{}) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - h.Holder.RecalculateCaches() + h.API.RecalculateCaches(r.Context()) w.WriteHeader(http.StatusNoContent) } @@ -2161,9 +1965,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - // Forward the error message. - err = h.BroadcastHandler.ReceiveMessage(pb) - if err != nil { + if err := h.API.PostClusterMessage(r.Context(), pb); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -2174,7 +1976,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(h.Cluster.Node.ID)) + _, err := w.Write([]byte(h.API.LocalID(r.Context()))) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } From 8336784d57d48827cf60787ce5d431f1c2a56d93 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 6 Mar 2018 17:28:33 +0300 Subject: [PATCH 213/311] More API updates; removed Cluster, Holder, etc from Handler --- api.go | 168 ++++++++++++++++++++++++++++++- handler.go | 255 ++++++++++++++++------------------------------- pilosa.go | 4 + server.go | 12 +-- server/server.go | 3 +- 5 files changed, 262 insertions(+), 180 deletions(-) diff --git a/api.go b/api.go index 84c2f5146..f369ba10a 100644 --- a/api.go +++ b/api.go @@ -25,6 +25,7 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -221,6 +222,12 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str } func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + // Validate that this handler owns the slice. + if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + return ErrClusterDoesNotOwnSlice + } + // Find the fragment. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -600,7 +607,166 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri return err } -// InputJSONDataParser validates input json file and executes SetBit. +func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + // Retrieve index from holder. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Retrieve local blocks. + localBlocks, err := index.ColumnAttrStore().Blocks() + if err != nil { + return nil, err + } + + // Read all attributes from all mismatched blocks. + attrs := make(map[uint64]map[string]interface{}) + for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + // Retrieve block data. + m, err := index.ColumnAttrStore().BlockData(blockID) + if err != nil { + return nil, err + } + + // Copy to index-wide struct. + for k, v := range m { + attrs[k] = v + } + } + return attrs, nil +} + +func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + // Retrieve index from holder. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return nil, ErrFrameNotFound + } + + // Retrieve local blocks. + localBlocks, err := f.RowAttrStore().Blocks() + if err != nil { + return nil, err + } + + // Read all attributes from all mismatched blocks. + attrs := make(map[uint64]map[string]interface{}) + for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + // Retrieve block data. + m, err := f.RowAttrStore().BlockData(blockID) + if err != nil { + return nil, err + } + + // Copy to index-wide struct. + for k, v := range m { + attrs[k] = v + } + } + return attrs, nil +} + +func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + if err != nil { + return err + } + + // Convert timestamps to time.Time. + timestamps := make([]*time.Time, len(req.Timestamps)) + for i, ts := range req.Timestamps { + if ts == 0 { + continue + } + t := time.Unix(0, ts) + timestamps[i] = &t + } + + // Import into fragment. + err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) + if err != nil { + api.logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + } + return err +} + +func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + if err != nil { + return err + } + + // Import into fragment. + err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) + if err != nil { + api.logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + } + return err +} + +func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { + // Retrieve index by name. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Set default time quantum on index. + return index.SetTimeQuantum(timeQuantum) +} + +func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { + // Retrieve index by name. + frame := api.Holder.Frame(indexName, frameName) + if frame == nil { + return ErrFrameNotFound + } + + // Set default time quantum on index. + return frame.SetTimeQuantum(timeQuantum) +} + +func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { + if inverse { + return api.Holder.MaxInverseSlices() + } + return api.Holder.MaxSlices() +} + +func (api *API) StatsWithTags(tags []string) StatsClient { + return api.Holder.Stats.WithTags(tags...) +} + +func (api *API) ClusterLongQueryTime() time.Duration { + return api.Cluster.LongQueryTime +} + +func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { + // Validate that this handler owns the slice. + if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + return nil, nil, ErrClusterDoesNotOwnSlice + } + + // Find the Index. + api.logger.Println("importing:", indexName, frameName, slice) + index := api.Holder.Index(indexName) + if index == nil { + api.logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + return nil, nil, ErrIndexNotFound + } + + // Retrieve frame. + frame := index.Frame(frameName) + if frame == nil { + api.logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) + return nil, nil, ErrFrameNotFound + } + return index, frame, nil +} + +// inputJSONDataParser validates input json file and executes SetBit. func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef, err := index.InputDefinition(name) if err != nil { diff --git a/handler.go b/handler.go index f9fc73c52..0f04028e0 100644 --- a/handler.go +++ b/handler.go @@ -240,27 +240,26 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { dif := time.Since(t) // Calculate per request StatsD metrics when the handler is fully configured. - if h.Holder != nil && h.Cluster != nil { - statsTags := make([]string, 0, 3) + statsTags := make([]string, 0, 3) - if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } - - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") - - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } - - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - - stats := h.Holder.Stats.WithTags(statsTags...) - stats.Histogram("http."+endpointName, float64(dif), 0.1) + longQueryTime := h.API.ClusterLongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + + stats := h.API.StatsWithTags(statsTags) + stats.Histogram("http."+endpointName, float64(dif), 0.1) } func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { @@ -348,13 +347,23 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.Holder.MaxSlices(), - Inverse: h.Holder.MaxInverseSlices(), - }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) +func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { + inverse, err := strconv.ParseBool(r.URL.Query().Get("inverse")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + ms := h.API.SliceMax(r.Context(), inverse) + if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { + pb := &internal.MaxSlicesResponse{ + MaxSlices: ms, + } + if buf, err := proto.Marshal(pb); err != nil { + h.Logger.Printf("protobuf marshal error: %s", err) + } else if _, err := w.Write(buf); err != nil { + h.Logger.Printf("stream write error: %s", err) + } + return } } @@ -518,16 +527,12 @@ func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Req return } - // Retrieve index by name. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Set default time quantum on index. - if err := index.SetTimeQuantum(tq); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err = h.API.ModifyIndexTimeQuantum(r.Context(), indexName, tq); err != nil { + if err == ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -554,34 +559,14 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve index from holder. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve local blocks. - blks, err := index.ColumnAttrStore().Blocks() + attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { - // Retrieve block data. - m, err := index.ColumnAttrStore().BlockData(blockID) - if err != nil { + if err == ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v } + return } // Encode response. @@ -722,16 +707,12 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req return } - // Retrieve index by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Set default time quantum on index. - if err := f.SetTimeQuantum(tq); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err := h.API.ModifyFrameTimeQuantum(r.Context(), indexName, frameName, tq); err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -909,34 +890,15 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve index from holder. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve local blocks. - blks, err := f.RowAttrStore().Blocks() + attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { - // Retrieve block data. - m, err := f.RowAttrStore().BlockData(blockID) - if err != nil { + switch err { + case ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v } + return } // Encode response. @@ -1094,44 +1056,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Convert timestamps to time.Time. - timestamps := make([]*time.Time, len(req.Timestamps)) - for i, ts := range req.Timestamps { - if ts == 0 { - continue + if err := h.API.Import(r.Context(), req); err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) } - t := time.Unix(0, ts) - timestamps[i] = &t - } - - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice) - http.Error(w, msg, http.StatusPreconditionFailed) - return - } - - // Find the Index. - h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) - index := h.Holder.Index(req.Index) - if index == nil { - h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve frame. - f := index.Frame(req.Frame) - if f == nil { - h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Import into fragment. - err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) - if err != nil { - h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -1174,34 +1109,17 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) return } - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice) - http.Error(w, msg, http.StatusPreconditionFailed) - return - } - - // Find the Index. - h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) - index := h.Holder.Index(req.Index) - if index == nil { - h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve frame. - f := index.Frame(req.Frame) - if f == nil { - h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Import into fragment. - err = f.ImportValue(req.Field, req.ColumnIDs, req.Values) - if err != nil { - h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + if err = h.API.ImportValue(r.Context(), req); err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1240,19 +1158,16 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Node.ID, index, slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, index, slice) - http.Error(w, msg, http.StatusPreconditionFailed) - return - } - if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { - if err == ErrFragmentNotFound { + switch err { + case ErrFragmentNotFound: http.Error(w, err.Error(), http.StatusNotFound) - } else { + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: http.Error(w, err.Error(), http.StatusInternalServerError) } + return } } diff --git a/pilosa.go b/pilosa.go index 98c67203f..04c76a133 100644 --- a/pilosa.go +++ b/pilosa.go @@ -74,6 +74,10 @@ var ( ErrTooManyWrites = errors.New("too many write commands") ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed") + ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") + ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") + + ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") ) // Regular expression to validate index and frame names. diff --git a/server.go b/server.go index 1973f746e..338183cf7 100644 --- a/server.go +++ b/server.go @@ -115,9 +115,8 @@ func NewServer() *Server { Logger: NopLogger, } - s.Handler.Holder = s.Holder - s.diagnostics.server = s s.Handler.API = NewAPI(s.logger) + s.Handler.API.Holder = s.Holder return s } @@ -165,12 +164,11 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest // Initialize HTTP handler. - s.Handler.Broadcaster = s.Broadcaster s.Handler.API.Broadcaster = s.Broadcaster - s.Handler.BroadcastHandler = s - s.Handler.StatusHandler = s - s.Handler.Node = node - s.Handler.Cluster = s.Cluster + s.Handler.API.BroadcastHandler = s + s.Handler.API.StatusHandler = s + s.Handler.API.URI = s.URI + s.Handler.API.Cluster = s.Cluster s.Handler.Executor = e s.Cluster.prefect = s.Handler diff --git a/server/server.go b/server/server.go index 66725f4ee..a50ad3e71 100644 --- a/server/server.go +++ b/server/server.go @@ -218,8 +218,7 @@ func (m *Command) SetupServer() error { } c := pilosa.GetHTTPClient(TLSConfig) m.Server.RemoteClient = c - m.Server.Handler.RemoteClient = c - m.Server.Cluster.RemoteClient = c + m.Server.Handler.API.RemoteClient = c // Statik file system. m.Server.Handler.FileSystem = &statik.FileSystem{} From c36dd395e664be375d37fe70f9988d5104a20d57 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 7 Mar 2018 15:39:44 +0300 Subject: [PATCH 214/311] All tests pass --- api.go | 7 +- client_test.go | 67 ++++++++++--------- ctl/backup_test.go | 10 ++- ctl/export_test.go | 10 ++- ctl/import_test.go | 21 +++--- ctl/restore_test.go | 10 ++- executor_test.go | 10 +-- handler.go | 67 +++++++++++-------- handler_test.go | 158 ++++++++++++++++++++++---------------------- holder_test.go | 2 +- stats_test.go | 16 ++--- test/handler.go | 33 ++++++--- 12 files changed, 218 insertions(+), 193 deletions(-) diff --git a/api.go b/api.go index f369ba10a..97d7828e5 100644 --- a/api.go +++ b/api.go @@ -61,7 +61,6 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { - // TODO: Wrap return resp, err } execOpts := &ExecOptions{ @@ -735,10 +734,16 @@ func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { } func (api *API) StatsWithTags(tags []string) StatsClient { + if api.Holder == nil || api.Cluster == nil { + return nil + } return api.Holder.Stats.WithTags(tags...) } func (api *API) ClusterLongQueryTime() time.Duration { + if api.Cluster == nil { + return 0 + } return api.Cluster.LongQueryTime } diff --git a/client_test.go b/client_test.go index f2bd18e50..36b9dc38d 100644 --- a/client_test.go +++ b/client_test.go @@ -36,10 +36,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { for i := 0; i < numNodes; i++ { hldr[i] = test.MustOpenHolder() server[i] = test.NewServer() - server[i].Handler.Cluster = c - server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI() - server[i].Handler.Holder = hldr[i].Holder - server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i] + server[i].Handler.API.URI = server[i].HostURI() + server[i].Handler.API.Cluster = c + server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() + server[i].Handler.API.Holder = hldr[i].Holder } return server, hldr } @@ -86,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI()) + owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].Host()) ownsNum := false for _, ownNum := range owns { if ownNum == num { @@ -217,10 +217,10 @@ func TestClient_Import(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -268,10 +268,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -317,10 +317,10 @@ func TestClient_ImportValue(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -355,10 +355,10 @@ func TestClient_BackupRestore(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -420,10 +420,11 @@ func TestClient_BackupInverseView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -457,10 +458,10 @@ func TestClient_BackupInvalidView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -486,10 +487,10 @@ func TestClient_FragmentBlocks(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) diff --git a/ctl/backup_test.go b/ctl/backup_test.go index d23b1a917..7a4feb3d9 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,12 +50,10 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewBackupCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") diff --git a/ctl/export_test.go b/ctl/export_test.go index 5d1d5a4d7..2bcccd6ba 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,12 +63,10 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) diff --git a/ctl/import_test.go b/ctl/import_test.go index 9022efae9..119a56071 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,12 +69,10 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() cm.Index = "i" @@ -111,16 +109,15 @@ func TestImportCommand_RunValue(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"rangeEnabled": true, "fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) cm.Index = "i" cm.Frame = "f" diff --git a/ctl/restore_test.go b/ctl/restore_test.go index bb8eb3b5f..f1edb7fe9 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,12 +52,10 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewRestoreCommand(stdin, stdout, stderr) cm.Path = file.Name() diff --git a/executor_test.go b/executor_test.go index c98239137..690bfc5ce 100644 --- a/executor_test.go +++ b/executor_test.go @@ -927,7 +927,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) @@ -961,7 +961,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) @@ -1004,7 +1004,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1056,7 +1056,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1130,7 +1130,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 2 & 4. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) diff --git a/handler.go b/handler.go index 0f04028e0..f6a385ccb 100644 --- a/handler.go +++ b/handler.go @@ -257,9 +257,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // useragent tag identifies internal/external endpoints statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - stats.Histogram("http."+endpointName, float64(dif), 0.1) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } } func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { @@ -328,7 +329,8 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.API.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &resp) + h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + return } // Set appropriate status code, if there is an error. @@ -348,10 +350,15 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - inverse, err := strconv.ParseBool(r.URL.Query().Get("inverse")) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + var err error + inverse := false + inverseStr := r.URL.Query().Get("inverse") + if inverseStr != "" { + inverse, err = strconv.ParseBool(r.URL.Query().Get("inverse")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } } ms := h.API.SliceMax(r.Context(), inverse) if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { @@ -1161,7 +1168,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { switch err { case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) + break case ErrClusterDoesNotOwnSlice: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: @@ -1528,26 +1535,30 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req) - switch err { - case nil: - break - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrInputDefinitionExists: - http.Error(w, err.Error(), http.StatusConflict) - case ErrInputDefinitionAttrsRequired: - fallthrough - case ErrInputDefinitionNameRequired: - fallthrough - case ErrInputDefinitionActionRequired: - fallthrough - case ErrInputDefinitionHasPrimaryKey: - fallthrough - case ErrInputDefinitionDupePrimaryKey: - http.Error(w, err.Error(), http.StatusBadRequest) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) + if err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req); err != nil { + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrInputDefinitionExists: + http.Error(w, err.Error(), http.StatusConflict) + case ErrInputDefinitionAttrsRequired: + fallthrough + case ErrInputDefinitionNameRequired: + fallthrough + case ErrInputDefinitionActionRequired: + fallthrough + case ErrInputDefinitionHasPrimaryKey: + fallthrough + case ErrInputDefinitionDupePrimaryKey: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 1e30f41ad..39bcc7fff 100644 --- a/handler_test.go +++ b/handler_test.go @@ -65,8 +65,8 @@ func TestHandler_NotFound(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -100,8 +100,8 @@ func TestHandler_Schema(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { @@ -139,9 +139,9 @@ func TestHandler_Status(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) - h.StatusHandler = s + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.StatusHandler = s s.Handler = h w := httptest.NewRecorder() @@ -186,8 +186,8 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { @@ -227,8 +227,8 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { @@ -244,8 +244,8 @@ func TestHandler_Query_Args_URL(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -272,8 +272,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -312,8 +312,8 @@ func TestHandler_Query_Args_Err(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { @@ -339,8 +339,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -360,8 +360,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -390,8 +390,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -423,8 +423,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -446,8 +446,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -494,8 +494,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -555,8 +555,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -579,8 +579,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -612,15 +612,15 @@ func TestHandler_Query_Err_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -633,8 +633,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -643,7 +643,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -661,8 +661,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) if w.Code != http.StatusMethodNotAllowed { @@ -676,8 +676,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -693,7 +693,7 @@ func TestHandler_Index_Delete(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -733,8 +733,8 @@ func TestHandler_DeleteFrame(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -753,8 +753,8 @@ func TestHandler_SetIndexTimeQuantum(t *testing.T) { hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -777,8 +777,8 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -796,7 +796,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -845,7 +845,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -895,7 +895,7 @@ func TestHandler_Frame_AddField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -999,7 +999,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -1064,7 +1064,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -1132,7 +1132,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set bits in the index. @@ -1181,8 +1181,8 @@ func TestHandler_Version(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/version", nil) @@ -1204,9 +1204,9 @@ func TestHandler_Fragment_Nodes(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(3) - h.Cluster.ReplicaN = 2 + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(3) + h.API.Cluster.ReplicaN = 2 w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) @@ -1233,8 +1233,8 @@ func TestHandler_Expvars(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) @@ -1279,8 +1279,8 @@ func TestHandler_CreateInputDefinition(t *testing.T) { ] }`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1314,8 +1314,8 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) //Ensure throwing error if there's duplicated primaryKey field invalidPrimaryKey := []byte(` @@ -1419,8 +1419,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Test index not found. w := httptest.NewRecorder() @@ -1467,8 +1467,8 @@ func TestHandler_GetInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} @@ -1634,8 +1634,8 @@ func TestHandler_CreateInput(t *testing.T) { "null_value": null }]`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Return error if index does not exist. w := httptest.NewRecorder() @@ -1749,8 +1749,8 @@ func TestInput_JSON(t *testing.T) { err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) for _, req := range tests { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json)))) @@ -1810,8 +1810,8 @@ func TestHandler_DeleteView(t *testing.T) { hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD") h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -1836,8 +1836,8 @@ func TestHandler_RecalculateCaches(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) diff --git a/holder_test.go b/holder_test.go index 41ede1adf..4c63e6e76 100644 --- a/holder_test.go +++ b/holder_test.go @@ -393,7 +393,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer hldr1.Close() s := test.NewServer() defer s.Close() - s.Handler.Holder = hldr1.Holder + s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder diff --git a/stats_test.go b/stats_test.go index 60deae993..591236268 100644 --- a/stats_test.go +++ b/stats_test.go @@ -215,10 +215,10 @@ func TestStatsCount_CreateIndex(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "createIndex" { t.Errorf("Expected createIndex, Results %s", name) @@ -239,7 +239,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -247,7 +247,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "deleteIndex" { t.Errorf("Expected deleteIndex, Results %s", name) @@ -268,7 +268,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -276,7 +276,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "createFrame" { t.Errorf("Expected createFrame, Results %s", name) @@ -300,7 +300,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false // Create index. @@ -308,7 +308,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "deleteFrame" { t.Errorf("Expected deleteFrame, Results %s", name) diff --git a/test/handler.go b/test/handler.go index 0f8c1cefd..ae7605603 100644 --- a/test/handler.go +++ b/test/handler.go @@ -27,6 +27,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) // Handler represents a test wrapper for pilosa.Handler. @@ -40,10 +41,12 @@ func NewHandler() *Handler { h := &Handler{ Handler: pilosa.NewHandler(), } - h.Handler.Executor = &h.Executor + h.API = pilosa.NewAPI(nil) + h.Handler.API = pilosa.NewAPI(nil) + h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. - h.Broadcaster = pilosa.NopBroadcaster + h.API.Broadcaster = pilosa.NopBroadcaster h.SetNormal() @@ -80,21 +83,35 @@ func NewServer() *Server { if err != nil { panic(err) } + s.Handler.API.URI = uri // Handler test messages can no-op. - s.Handler.Broadcaster = pilosa.NopBroadcaster + s.Handler.API.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler - s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.Cluster = NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() return s } // LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - return nil, nil + if s.Handler.API.Holder == nil { + return nil, errors.New("Server.Holder is nil") + } + + ns := internal.NodeStatus{ + Host: s.Handler.Handler.API.URI.HostPort(), + State: pilosa.NodeStateUp, + Indexes: pilosa.EncodeIndexes(s.Handler.API.Holder.Indexes()), + } + + // Append Slice list per this Node's indexes + for _, index := range ns.Indexes { + index.Slices = s.Handler.API.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.API.URI.HostPort()) + } + + return &ns, nil } // ClusterStatus exists so that test.Server implements StatusHandler. From be4965d962dcbf216dfb01a9334ca76b878ae66b Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 3 Apr 2018 16:09:11 -0700 Subject: [PATCH 215/311] fixup logger, handler.FileSystem --- api.go | 46 ++++++++++++++++++++++------------------------ handler.go | 5 ++--- server.go | 3 ++- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/api.go b/api.go index 97d7828e5..54dd65244 100644 --- a/api.go +++ b/api.go @@ -19,8 +19,6 @@ import ( "encoding/csv" "fmt" "io" - "io/ioutil" - "log" "net/http" "reflect" "strconv" @@ -44,15 +42,15 @@ type API struct { Cluster *Cluster URI *URI RemoteClient *http.Client - logger *log.Logger + Logger Logger } -func NewAPI(logger *log.Logger) *API { - if logger == nil { - logger = log.New(ioutil.Discard, "", 0) - } +func NewAPI() *API { return &API{ - logger: logger, + Broadcaster: NopBroadcaster, + //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop + //StatusHandler: NopStatusHandler, // TODO: implement the nop + Logger: NopLogger, } } @@ -132,7 +130,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index Meta: options.Encode(), }) if err != nil { - api.logger.Printf("problem sending CreateIndex message: %s", err) + api.Logger.Printf("problem sending CreateIndex message: %s", err) return nil, err } api.Holder.Stats.Count("createIndex", 1, 1.0) @@ -159,7 +157,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { Index: indexName, }) if err != nil { - api.logger.Printf("problem sending DeleteIndex message: %s", err) + api.Logger.Printf("problem sending DeleteIndex message: %s", err) return err } api.Holder.Stats.Count("deleteIndex", 1, 1.0) @@ -187,7 +185,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str Meta: options.Encode(), }) if err != nil { - api.logger.Printf("problem sending CreateFrame message: %s", err) + api.Logger.Printf("problem sending CreateFrame message: %s", err) return nil, err } api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -213,7 +211,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str Frame: frameName, }) if err != nil { - api.logger.Printf("problem sending DeleteFrame message: %s", err) + api.Logger.Printf("problem sending DeleteFrame message: %s", err) return err } api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -223,7 +221,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { // Validate that this handler owns the slice. if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { - api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return ErrClusterDoesNotOwnSlice } @@ -413,7 +411,7 @@ func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inp Definition: def, }) if err != nil { - api.logger.Printf("problem sending CreateInputDefinition message: %s", err) + api.Logger.Printf("problem sending CreateInputDefinition message: %s", err) } return nil } @@ -450,7 +448,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp Name: inputDefName, }) if err != nil { - api.logger.Printf("problem sending DeleteInputDefinition message: %s", err) + api.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) } return nil } @@ -521,7 +519,7 @@ func (api *API) CreateFrameField(ctx context.Context, indexName string, frameNam Field: encodeField(field), }) if err != nil { - api.logger.Printf("problem sending CreateField message: %s", err) + api.Logger.Printf("problem sending CreateField message: %s", err) } return err } @@ -546,7 +544,7 @@ func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameNam Field: fieldName, }) if err != nil { - api.logger.Printf("problem sending DeleteField message: %s", err) + api.Logger.Printf("problem sending DeleteField message: %s", err) } return err } @@ -600,7 +598,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri View: viewName, }) if err != nil { - api.logger.Printf("problem sending DeleteView message: %s", err) + api.Logger.Printf("problem sending DeleteView message: %s", err) } return err @@ -685,7 +683,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) } return err } @@ -699,7 +697,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // Import into fragment. err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) if err != nil { - api.logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) } return err } @@ -750,22 +748,22 @@ func (api *API) ClusterLongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { - api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return nil, nil, ErrClusterDoesNotOwnSlice } // Find the Index. - api.logger.Println("importing:", indexName, frameName, slice) + api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice) index := api.Holder.Index(indexName) if index == nil { - api.logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) return nil, nil, ErrIndexNotFound } // Retrieve frame. frame := index.Frame(frameName) if frame == nil { - api.logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) + api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) return nil, nil, ErrFrameNotFound } return index, frame, nil diff --git a/handler.go b/handler.go index f6a385ccb..6e87bb4be 100644 --- a/handler.go +++ b/handler.go @@ -45,6 +45,8 @@ import ( type Handler struct { Router *mux.Router + FileSystem FileSystem + // The execution engine for running queries. Executor interface { Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) @@ -78,9 +80,6 @@ type errorResponse struct { // NewHandler returns a new instance of Handler with a default logger. func NewHandler() *Handler { handler := &Handler{ - Broadcaster: NopBroadcaster, - //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop - //StatusHandler: NopStatusHandler, // TODO: implement the nop FileSystem: NopFileSystem, Logger: NopLogger, } diff --git a/server.go b/server.go index 338183cf7..54f68a2f2 100644 --- a/server.go +++ b/server.go @@ -115,13 +115,14 @@ func NewServer() *Server { Logger: NopLogger, } - s.Handler.API = NewAPI(s.logger) + s.Handler.API = NewAPI() s.Handler.API.Holder = s.Holder return s } // Open opens and initializes the server. func (s *Server) Open() error { + s.Handler.API.Logger = s.Logger // TODO do this in NewServer with functional options s.Logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { From 17468e7d90bcaa42cd90c6968e1c5a40e805b896 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 5 Apr 2018 19:43:40 +0300 Subject: [PATCH 216/311] Updated tutorials --- docs/tutorials.md | 90 +++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 2401cdfe4..995e3d41d 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -70,6 +70,8 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. +One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. + Create `node1.config.toml` in the project directory and paste the following in it: ```toml @@ -79,7 +81,7 @@ data-dir = "node1_data" bind = "https://01.pilosa.local:10501" [cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] +coordinator = true [tls] certificate = "pilosa.local.crt" @@ -87,7 +89,7 @@ key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 15000 key = "pilosa.local.gossip32" ``` @@ -100,16 +102,13 @@ Create `node2.config.toml` in the project directory and paste the following in i data-dir = "node2_data" bind = "https://02.pilosa.local:10502" -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - [tls] certificate = "pilosa.local.crt" key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 16000 key = "pilosa.local.gossip32" ``` @@ -122,16 +121,13 @@ Create `node3.config.toml` in the project directory and paste the following in i data-dir = "node3_data" bind = "https://03.pilosa.local:10503" -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - [tls] certificate = "pilosa.local.crt" key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 17000 key = "pilosa.local.gossip32" ``` @@ -140,9 +136,9 @@ Here is some explanation of the configuration items: * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. +* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. -* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. +* `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. #### Final Touches Before Running the Cluster @@ -189,9 +185,9 @@ curl -k --ipv4 https://01.pilosa.local:10501/status The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. -All nodes should be in the `UP` state: +All nodes should be in the `NORMAL` state: ``` response -{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} +{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]} ``` #### Running Queries @@ -216,7 +212,7 @@ We just created frame `sample-frame` with default options. Let's run a `SetBit` query: ``` request -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", row=1, col=100)' ``` ``` response {"results":[true]} @@ -224,7 +220,7 @@ curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit Confirm that the bit was indeed set: ``` request -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)' ``` ``` response {"results":[{"attrs":{},"bits":[100]}]} @@ -232,7 +228,7 @@ curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap The same response should be returned when querying other nodes in the cluster: ``` request -curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)' ``` ``` response {"results":[{"attrs":{},"bits":[100]}]} @@ -283,7 +279,7 @@ This query sets the age, weight, and t-cell count for the patient with ID `1` in ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' + -d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)' ``` ``` response {"results":[null]} @@ -373,11 +369,11 @@ Now, let's add some books to our index. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) - SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972) - SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000) - SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) - SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' + -d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960) + SetColumnAttrs(col=2, name="No Name in the Street", year=1972) + SetColumnAttrs(col=3, name="The Tipping Point", year=2000) + SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003) + SetColumnAttrs(col=5, name="The Forever War", year=2008)' ``` ``` response {"results":[null,null,null,null,null]} @@ -387,11 +383,11 @@ And add some members. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") - SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins") - SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks") - SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") - SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' + -d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith") + SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins") + SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks") + SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez") + SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")' ``` ``` response {"results":[null,null,null,null,null]} @@ -401,7 +397,7 @@ At this point we can query one of the `member` records by querying that row. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' + -d 'Bitmap(frame="members", row=10002)' ``` ``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} @@ -411,23 +407,19 @@ Now let's add some data to the matrix such that each pair represents a member wh ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetBit(frame="members", rowID=10001, columnID=3) - SetBit(frame="members", rowID=10001, columnID=5) - - SetBit(frame="members", rowID=10002, columnID=1) - SetBit(frame="members", rowID=10002, columnID=2) - SetBit(frame="members", rowID=10002, columnID=4) - - SetBit(frame="members", rowID=10003, columnID=3) - - SetBit(frame="members", rowID=10004, columnID=4) - SetBit(frame="members", rowID=10004, columnID=5) - - SetBit(frame="members", rowID=10005, columnID=1) - SetBit(frame="members", rowID=10005, columnID=2) - SetBit(frame="members", rowID=10005, columnID=3) - SetBit(frame="members", rowID=10005, columnID=4) - SetBit(frame="members", rowID=10005, columnID=5)' + -d 'SetBit(frame="members", row=10001, col=3) + SetBit(frame="members", row=10001, col=5) + SetBit(frame="members", row=10002, col=1) + SetBit(frame="members", row=10002, col=2) + SetBit(frame="members", row=10002, col=4) + SetBit(frame="members", row=10003, col=3) + SetBit(frame="members", row=10004, col=4) + SetBit(frame="members", row=10004, col=5) + SetBit(frame="members", row=10005, col=1) + SetBit(frame="members", row=10005, col=2) + SetBit(frame="members", row=10005, col=3) + SetBit(frame="members", row=10005, col=4) + SetBit(frame="members", row=10005, col=5)' ``` ``` response {"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} @@ -437,7 +429,7 @@ Now pull the record for `Sue Perkins` again. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' + -d 'Bitmap(frame="members", row=10002)' ``` ``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} @@ -448,7 +440,7 @@ In order to retrieve the attribute information that we stored for each book, we ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' + -d 'Bitmap(frame="members", row=10002)' ``` ``` response { @@ -466,7 +458,7 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`, ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' + -d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))' ``` ``` response { From 81cac4775f34ce51ab2d5b2e30e58cd8d59812a9 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 5 Apr 2018 19:53:05 +0300 Subject: [PATCH 217/311] fix cluster config link --- docs/tutorials.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 995e3d41d..4d1a06c42 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -136,7 +136,7 @@ Here is some explanation of the configuration items: * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. +* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings. * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. * `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. From cc521e5c91ddee93efb47b56ba11cf8578f9c39a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 5 Apr 2018 22:29:03 +0300 Subject: [PATCH 218/311] index param is required for /fragment/nodes --- handler.go | 2 +- handler_test.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/handler.go b/handler.go index ffd28da46..02d20564c 100644 --- a/handler.go +++ b/handler.go @@ -122,7 +122,7 @@ func BuildRouters(handler *Handler) { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index") + h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice", "index") h.validators["GetSliceMax"] = QueryValidationSpecRequired().Optional("inverse") h.validators["PostQuery"] = QueryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") h.validators["GetExport"] = QueryValidationSpecRequired("index", "frame", "view", "slice") diff --git a/handler_test.go b/handler_test.go index ba9480544..12c145e87 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1225,6 +1225,13 @@ func TestHandler_Fragment_Nodes(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } + // index is required + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } } // Ensure the handler can return expvars without panicking. From 6d3f079b7aa73c8c8c57a49e8384e9fa3effb26e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 6 Apr 2018 17:34:46 +0300 Subject: [PATCH 219/311] updated row/col labels in getting started --- docs/getting-started.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index dc1e07ceb..c9cdb8aba 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -129,7 +129,7 @@ Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Bitmap(frame="stargazer", rowID=14)' + -d 'Bitmap(frame="stargazer", row=14)' ``` ``` response { @@ -167,8 +167,8 @@ Which repositories were starred by user 14 and 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", rowID=14), - Bitmap(frame="stargazer", rowID=19) + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19) )' ``` ``` response @@ -187,8 +187,8 @@ Which repositories were starred by user 14 or 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Union( - Bitmap(frame="stargazer", rowID=14), - Bitmap(frame="stargazer", rowID=19) + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19) )' ``` ``` response @@ -207,9 +207,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", rowID=14), - Bitmap(frame="stargazer", rowID=19), - Bitmap(frame="language", rowID=1) + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19), + Bitmap(frame="language", row=1) )' ``` ``` response @@ -227,7 +227,7 @@ Set user 99999 as a stargazer for repository 77777: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)' + -d 'SetBit(frame="stargazer", column=77777, row=99999)' ``` ``` response {"results":[true]} From f8cb579187ab6730a6b31b2c588ce993b7431565 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 9 Apr 2018 14:29:43 -0500 Subject: [PATCH 220/311] get tests passing --- api.go | 79 ++++++++++-- client_test.go | 2 +- cluster.go | 6 +- ctl/backup_test.go | 2 +- ctl/export_test.go | 2 +- ctl/import_test.go | 4 +- ctl/restore_test.go | 2 +- handler.go | 272 ++++++++++++++++++----------------------- handler_test.go | 11 +- pilosa.go | 4 + server/cluster_test.go | 14 +-- server/server.go | 1 + server/server_test.go | 6 +- test/handler.go | 24 +--- 14 files changed, 222 insertions(+), 207 deletions(-) diff --git a/api.go b/api.go index 54dd65244..82ab3a4ce 100644 --- a/api.go +++ b/api.go @@ -28,6 +28,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) type API struct { @@ -40,7 +41,7 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster - URI *URI + URI URI RemoteClient *http.Client Logger Logger } @@ -220,7 +221,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return ErrClusterDoesNotOwnSlice } @@ -339,7 +340,7 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st // Loop over each slice and import it if this node owns it. for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { continue } @@ -475,8 +476,13 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s return nil } -func (api *API) RecalculateCaches(ctx context.Context) { +func (api *API) RecalculateCaches(ctx context.Context) error { + err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) + if err != nil { + return errors.Wrap(err, "broacasting message") + } api.Holder.RecalculateCaches() + return nil } func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error { @@ -487,7 +493,7 @@ func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error return nil } -func (api *API) LocalID(ctx context.Context) string { +func (api *API) LocalID() string { return api.Cluster.Node.ID } @@ -549,7 +555,7 @@ func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameNam return err } -func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) (*FrameSchema, error) { +func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -724,13 +730,14 @@ func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, fr return frame.SetTimeQuantum(timeQuantum) } -func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { - if inverse { - return api.Holder.MaxInverseSlices() - } +func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() } +func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { + return api.Holder.MaxInverseSlices() +} + func (api *API) StatsWithTags(tags []string) StatsClient { if api.Holder == nil || api.Cluster == nil { return nil @@ -747,7 +754,7 @@ func (api *API) ClusterLongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return nil, nil, ErrClusterDoesNotOwnSlice } @@ -837,3 +844,53 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na } return setBits, nil } + +func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.nodeByID(id) + if newNode == nil { + return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") + } + + // If the new coordinator is this node, do the SetCoordinator directly. + if newNode.ID == api.LocalID() { + return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + } + + // Send the set-coordinator message to new node. + err = api.Broadcaster.SendTo( + newNode, + &internal.SetCoordinatorMessage{ + New: EncodeNode(newNode), + }) + if err != nil { + return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) + } + return oldNode, newNode, nil +} + +func (api *API) RemoveNode(id string) (*Node, error) { + removeNode := api.Cluster.nodeByID(id) + if removeNode == nil { + return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") + } + + // Start the resize process (similar to NodeJoin) + err := api.Cluster.NodeLeave(removeNode) + if err != nil { + return removeNode, errors.Wrap(err, "calling node leave") + } + return removeNode, nil +} + +func (api *API) ResizeAbort() error { + if !api.Cluster.IsCoordinator() { + return ErrNodeNotCoordinator + } + err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + return errors.Wrap(err, "complete current job") +} + +func (api *API) State() string { + return api.Cluster.State() +} diff --git a/client_test.go b/client_test.go index 36b9dc38d..cdbaba788 100644 --- a/client_test.go +++ b/client_test.go @@ -86,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].Host()) + owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false for _, ownNum := range owns { if ownNum == num { diff --git a/cluster.go b/cluster.go index 7e53628ac..c490c58cb 100644 --- a/cluster.go +++ b/cluster.go @@ -1198,7 +1198,7 @@ func (c *Cluster) CompleteCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() if c.currentJob == nil { - return fmt.Errorf("no resize job currently running") + return ErrResizeNotRunning } c.currentJob.SetState(state) c.currentJob = nil @@ -1747,7 +1747,7 @@ func (c *Cluster) nodeJoin(node *Node) error { func (c *Cluster) NodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. if !c.IsCoordinator() { - return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.CoordinatorNode().ID) + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) } if c.State() != ClusterStateNormal { @@ -1761,7 +1761,7 @@ func (c *Cluster) NodeLeave(node *Node) error { // Prevent removing the coordinator node (this node). if node.ID == c.Node.ID { - return fmt.Errorf("The coordinator node cannot be removed. First, make a different node the new coordinator.") + return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.") } // See if resize job can be generated diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 7a4feb3d9..d80b72475 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,7 +50,7 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/export_test.go b/ctl/export_test.go index 2bcccd6ba..b414f711f 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,7 +63,7 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/import_test.go b/ctl/import_test.go index 119a56071..bd259e342 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,7 +69,7 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -110,7 +110,7 @@ func TestImportCommand_RunValue(t *testing.T) { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/restore_test.go b/ctl/restore_test.go index f1edb7fe9..04cb30415 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,7 +52,7 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/handler.go b/handler.go index 6e87bb4be..3ac7f6e94 100644 --- a/handler.go +++ b/handler.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "encoding/json" - "errors" "expvar" "fmt" "io" @@ -26,26 +25,27 @@ import ( "net/url" // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" + "reflect" "runtime/debug" "strconv" "strings" "time" - - "reflect" + "unicode" "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - - "unicode" + "github.com/pkg/errors" ) // Handler represents an HTTP handler. type Handler struct { Router *mux.Router - FileSystem FileSystem + FileSystem FileSystem + NormalRouter *mux.Router + RestrictedRouter *mux.Router // The execution engine for running queries. Executor interface { @@ -215,7 +215,7 @@ func loadNormal(router *mux.Router, handler *Handler) { } func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.Cluster.State()), http.StatusMethodNotAllowed) + http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.API.State()), http.StatusMethodNotAllowed) } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { @@ -295,7 +295,10 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { return } - cs := pb.(*internal.ClusterStatus) + cs, ok := status.(*internal.ClusterStatus) + if !ok { + panic("status is not a status") + } if err := json.NewEncoder(w).Encode(getStatusResponse{ State: cs.State, Nodes: DecodeNodes(cs.Nodes), @@ -348,28 +351,13 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - var err error - inverse := false - inverseStr := r.URL.Query().Get("inverse") - if inverseStr != "" { - inverse, err = strconv.ParseBool(r.URL.Query().Get("inverse")) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - ms := h.API.SliceMax(r.Context(), inverse) - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - pb := &internal.MaxSlicesResponse{ - MaxSlices: ms, - } - if buf, err := proto.Marshal(pb); err != nil { - h.Logger.Printf("protobuf marshal error: %s", err) - } else if _, err := w.Write(buf); err != nil { - h.Logger.Printf("stream write error: %s", err) - } - return +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.API.MaxSlices(r.Context()), + Inverse: h.API.MaxInverseSlices(r.Context()), + }); err != nil { + h.Logger.Printf("write slices-max response error: %s", err) } } @@ -802,7 +790,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - schema, err := h.API.FrameFields(r.Context(), indexName, frameName) + fields, err := h.API.FrameFields(r.Context(), indexName, frameName) if err != nil { switch err { case ErrIndexNotFound: @@ -1557,7 +1545,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1646,46 +1634,24 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// <<<<<<< b702f70610962341116d9678ce7c5877b7e171ca -// handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request. func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { // Decode request. var req setCoordinatorRequest err := json.NewDecoder(r.Body).Decode(&req) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) return } - oldNode := h.Cluster.nodeByID(h.Cluster.Coordinator) - newNode := h.Cluster.nodeByID(req.ID) - if newNode == nil { - http.Error(w, "Node with provided ID does not exist", http.StatusBadRequest) - return - } - - if err := func() error { - // If the new coordinator is this node, do the SetCoordinator directly. - if newNode.ID == h.Node.ID { - return h.Cluster.SetCoordinator(newNode) + oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) } - - // Send the set-coordinator message to new node. - err := h.Broadcaster.SendTo( - newNode, - &internal.SetCoordinatorMessage{ - New: EncodeNode(newNode), - }) - if err != nil { - return fmt.Errorf("problem sending SetCoordinator message: %s", err) - } - - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Encode response. if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ Old: oldNode, @@ -1714,16 +1680,13 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht return } - removeNode := h.Cluster.nodeByID(req.ID) - if removeNode == nil { - http.Error(w, fmt.Sprintf("Node is not a member of the cluster: %s", req.ID), http.StatusBadRequest) - return - } - - // Start the resize process (similar to NodeJoin) - err = h.Cluster.NodeLeave(removeNode) + removeNode, err := h.API.RemoveNode(req.ID) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if errors.Cause(err) == ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) + } return } @@ -1745,21 +1708,20 @@ type removeNodeResponse struct { // handlePostClusterResizeAbort handles POST /cluster/resize/abort request. func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + err := h.API.ResizeAbort() var msg string - - if err := func() error { - if !h.Cluster.IsCoordinator() { - return fmt.Errorf("abort requests must be made on the coordinator node") + if err != nil { + switch errors.Cause(err) { + case ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return } - err := h.Cluster.CompleteCurrentJob(ResizeJobStateAborted) - if err != nil { - return err - } - return nil - }(); err != nil { - msg = err.Error() } - // Encode response. if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ Info: msg, @@ -1772,80 +1734,82 @@ type clusterResizeAbortResponse struct { Info string `json:"info"` } -// InputJSONDataParser validates input json file and executes SetBit. -func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { - inputDef, err := index.InputDefinition(name) - if err != nil { - return nil, err - } - // If field in input data is not in defined definition, return error. - var colValue uint64 - validFields := make(map[string]bool) - timestampFrame := make(map[string]int64) - for _, field := range inputDef.Fields() { - validFields[field.Name] = true - if field.PrimaryKey { - value, ok := req[field.Name] - if !ok { - return nil, fmt.Errorf("primary key does not exist") - } - rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float - if !ok { - return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) - } - colValue = uint64(rawValue) - } - // Find frame that need to add timestamp. - for _, action := range field.Actions { - if action.ValueDestination == InputSetTimestamp { - timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) - if err != nil { - return nil, err - } - } - } - } +// // InputJSONDataParser validates input json file and executes SetBit. +// func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { +// inputDef, err := index.InputDefinition(name) +// if err != nil { +// return nil, err +// } +// // If field in input data is not in defined definition, return error. +// var colValue uint64 +// validFields := make(map[string]bool) +// timestampFrame := make(map[string]int64) +// for _, field := range inputDef.Fields() { +// validFields[field.Name] = true +// if field.PrimaryKey { +// value, ok := req[field.Name] +// if !ok { +// return nil, fmt.Errorf("primary key does not exist") +// } +// rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float +// if !ok { +// return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) +// } +// colValue = uint64(rawValue) +// } +// // Find frame that need to add timestamp. +// for _, action := range field.Actions { +// if action.ValueDestination == InputSetTimestamp { +// timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) +// if err != nil { +// return nil, err +// } +// } +// } +// } - for key := range req { - _, ok := validFields[key] - if !ok { - return nil, fmt.Errorf("field not found: %s", key) - } - } +// for key := range req { +// _, ok := validFields[key] +// if !ok { +// return nil, fmt.Errorf("field not found: %s", key) +// } +// } - setBits := make(map[string][]*Bit) +// setBits := make(map[string][]*Bit) - for _, field := range inputDef.Fields() { - // skip field that defined in definition but not in input data - if _, ok := req[field.Name]; !ok { - continue - } +// for _, field := range inputDef.Fields() { +// // skip field that defined in definition but not in input data +// if _, ok := req[field.Name]; !ok { +// continue +// } - // Looking into timestampFrame map and set timestamp to the whole frame - for _, action := range field.Actions { - frame := action.Frame - timestamp := timestampFrame[action.Frame] - // Skip input data field values that are set to null - if req[field.Name] == nil { - continue - } - bit, err := HandleAction(action, req[field.Name], colValue, timestamp) - if err != nil { - return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) - } - if bit != nil { - setBits[frame] = append(setBits[frame], bit) - } - } - } - return setBits, nil -} - -// ======= -// >>>>>>> Moved more of handler to API +// // Looking into timestampFrame map and set timestamp to the whole frame +// for _, action := range field.Actions { +// frame := action.Frame +// timestamp := timestampFrame[action.Frame] +// // Skip input data field values that are set to null +// if req[field.Name] == nil { +// continue +// } +// bit, err := HandleAction(action, req[field.Name], colValue, timestamp) +// if err != nil { +// return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) +// } +// if bit != nil { +// setBits[frame] = append(setBits[frame], bit) +// } +// } +// } +// return setBits, nil +// } func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - h.API.RecalculateCaches(r.Context()) + err := h.API.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) } @@ -1901,7 +1865,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(h.API.LocalID(r.Context()))) + _, err := w.Write([]byte(h.API.LocalID())) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -1936,12 +1900,12 @@ func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { func (s queryValidationSpec) validate(query url.Values) error { for _, req := range s.required { if query.Get(req) == "" { - return errors.New(fmt.Sprintf("%s is required", req)) + return errors.Errorf("%s is required", req) } } - for k, _ := range query { + for k := range query { if _, ok := s.args[k]; !ok { - return errors.New(fmt.Sprintf("%s is not a valid argument", k)) + return errors.Errorf("%s is not a valid argument", k) } } return nil diff --git a/handler_test.go b/handler_test.go index 39bcc7fff..c9a3ef988 100644 --- a/handler_test.go +++ b/handler_test.go @@ -158,14 +158,15 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { h := test.NewHandler() - h.Cluster = test.NewCluster(1) + h.API.Cluster = test.NewCluster(1) h.SetRestricted() w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"info":"no resize job currently running"}`+"\n" { + bod, err := ioutil.ReadAll(w.Body) + t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) + } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { t.Fatalf("unexpected body: %s", body) } }) @@ -1852,8 +1853,8 @@ func TestHandler_WebUI(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.FileSystem = &statik.FileSystem{} w := httptest.NewRecorder() diff --git a/pilosa.go b/pilosa.go index 04c76a133..0c7e790a3 100644 --- a/pilosa.go +++ b/pilosa.go @@ -78,6 +78,10 @@ var ( ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") + + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") + ErrNodeNotCoordinator = errors.New("node is not the coordinator") + ErrResizeNotRunning = errors.New("no resize job currently running") ) // Regular expression to validate index and frame names. diff --git a/server/cluster_test.go b/server/cluster_test.go index 6bbdea4b4..99cd62225 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -62,7 +62,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.MemberSet = gossipMemberSet0 m0.Server.Broadcaster = m0.Server m0.Server.Gossiper = gossipMemberSet0 - m0.Server.Handler.Broadcaster = m0.Server.Broadcaster + m0.Server.Handler.API.Broadcaster = m0.Server.Broadcaster m0.Server.Holder.Broadcaster = m0.Server.Broadcaster m0.Server.BroadcastReceiver = gossipMemberSet0 @@ -89,7 +89,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Server.Cluster.MemberSet = gossipMemberSet1 m1.Server.Broadcaster = m1.Server m1.Server.Gossiper = gossipMemberSet1 - m1.Server.Handler.Broadcaster = m1.Server.Broadcaster + m1.Server.Handler.API.Broadcaster = m1.Server.Broadcaster m1.Server.Holder.Broadcaster = m1.Server.Broadcaster m1.Server.BroadcastReceiver = gossipMemberSet1 @@ -507,9 +507,9 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Run("ErrorRemoveInvalidNode", func(t *testing.T) { resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`) - expBody := "Node is not a member of the cluster: invalid-node-id" - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("expected StatusCode %d but got %d", http.StatusBadRequest, resp.StatusCode) + expBody := "removing node: finding node to remove: node with provided ID does not exist" + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected StatusCode %d but got %d", http.StatusNotFound, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) } @@ -521,7 +521,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) - expBody := "The coordinator node cannot be removed. First, make a different node the new coordinator." + expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator." if resp.StatusCode != http.StatusInternalServerError { t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { @@ -538,7 +538,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) - expBody := fmt.Sprintf("Node removal requests are only valid on the Coordinator node: %s", coordinatorNodeID) + expBody := fmt.Sprintf("removing node: calling node leave: node removal requests are only valid on the coordinator node: %s", coordinatorNodeID) if resp.StatusCode != http.StatusInternalServerError { t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { diff --git a/server/server.go b/server/server.go index a50ad3e71..62c7a70a5 100644 --- a/server/server.go +++ b/server/server.go @@ -219,6 +219,7 @@ func (m *Command) SetupServer() error { c := pilosa.GetHTTPClient(TLSConfig) m.Server.RemoteClient = c m.Server.Handler.API.RemoteClient = c + m.Server.Cluster.RemoteClient = c // Statik file system. m.Server.Handler.FileSystem = &statik.FileSystem{} diff --git a/server/server_test.go b/server/server_test.go index 3852845a6..9f6cdfa8a 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -409,7 +409,11 @@ func TestMain_RecalculateHashes(t *testing.T) { } // Calculate caches on the first node - cluster[0].RecalculateCaches() + err := cluster[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}` // Run a TopN query on all nodes. The result should be the same as the target. diff --git a/test/handler.go b/test/handler.go index ae7605603..167187e0b 100644 --- a/test/handler.go +++ b/test/handler.go @@ -27,7 +27,6 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pkg/errors" ) // Handler represents a test wrapper for pilosa.Handler. @@ -41,8 +40,8 @@ func NewHandler() *Handler { h := &Handler{ Handler: pilosa.NewHandler(), } - h.API = pilosa.NewAPI(nil) - h.Handler.API = pilosa.NewAPI(nil) + h.API = pilosa.NewAPI() + h.Handler.API = pilosa.NewAPI() h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. @@ -83,7 +82,7 @@ func NewServer() *Server { if err != nil { panic(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri // Handler test messages can no-op. s.Handler.API.Broadcaster = pilosa.NopBroadcaster @@ -96,22 +95,7 @@ func NewServer() *Server { // LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - if s.Handler.API.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Host: s.Handler.Handler.API.URI.HostPort(), - State: pilosa.NodeStateUp, - Indexes: pilosa.EncodeIndexes(s.Handler.API.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Handler.API.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.API.URI.HostPort()) - } - - return &ns, nil + return nil, nil } // ClusterStatus exists so that test.Server implements StatusHandler. From 96e953e6e07f5c894ece6d4b8b93920552a1efe2 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 9 Apr 2018 16:42:50 -0500 Subject: [PATCH 221/311] rename var to be more descriptive --- handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handler.go b/handler.go index 3ac7f6e94..bb9fbc441 100644 --- a/handler.go +++ b/handler.go @@ -289,13 +289,13 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status, err := h.API.Status(r.Context()) + pb, err := h.API.Status(r.Context()) if err != nil { h.Logger.Printf("cluster status error: %s", err) return } - cs, ok := status.(*internal.ClusterStatus) + cs, ok := pb.(*internal.ClusterStatus) if !ok { panic("status is not a status") } From 401c9682b8bfd0e6983c8d59b4d9e6cf7b17dcf9 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 10 Apr 2018 09:40:28 -0500 Subject: [PATCH 222/311] remove commented code and fix double-NewAPI --- handler.go | 69 ------------------------------------------------- test/handler.go | 2 +- 2 files changed, 1 insertion(+), 70 deletions(-) diff --git a/handler.go b/handler.go index bb9fbc441..d7536f738 100644 --- a/handler.go +++ b/handler.go @@ -1734,75 +1734,6 @@ type clusterResizeAbortResponse struct { Info string `json:"info"` } -// // InputJSONDataParser validates input json file and executes SetBit. -// func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { -// inputDef, err := index.InputDefinition(name) -// if err != nil { -// return nil, err -// } -// // If field in input data is not in defined definition, return error. -// var colValue uint64 -// validFields := make(map[string]bool) -// timestampFrame := make(map[string]int64) -// for _, field := range inputDef.Fields() { -// validFields[field.Name] = true -// if field.PrimaryKey { -// value, ok := req[field.Name] -// if !ok { -// return nil, fmt.Errorf("primary key does not exist") -// } -// rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float -// if !ok { -// return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) -// } -// colValue = uint64(rawValue) -// } -// // Find frame that need to add timestamp. -// for _, action := range field.Actions { -// if action.ValueDestination == InputSetTimestamp { -// timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) -// if err != nil { -// return nil, err -// } -// } -// } -// } - -// for key := range req { -// _, ok := validFields[key] -// if !ok { -// return nil, fmt.Errorf("field not found: %s", key) -// } -// } - -// setBits := make(map[string][]*Bit) - -// for _, field := range inputDef.Fields() { -// // skip field that defined in definition but not in input data -// if _, ok := req[field.Name]; !ok { -// continue -// } - -// // Looking into timestampFrame map and set timestamp to the whole frame -// for _, action := range field.Actions { -// frame := action.Frame -// timestamp := timestampFrame[action.Frame] -// // Skip input data field values that are set to null -// if req[field.Name] == nil { -// continue -// } -// bit, err := HandleAction(action, req[field.Name], colValue, timestamp) -// if err != nil { -// return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) -// } -// if bit != nil { -// setBits[frame] = append(setBits[frame], bit) -// } -// } -// } -// return setBits, nil -// } - func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { err := h.API.RecalculateCaches(r.Context()) if err != nil { diff --git a/test/handler.go b/test/handler.go index 167187e0b..cb90cd383 100644 --- a/test/handler.go +++ b/test/handler.go @@ -41,7 +41,7 @@ func NewHandler() *Handler { Handler: pilosa.NewHandler(), } h.API = pilosa.NewAPI() - h.Handler.API = pilosa.NewAPI() + h.Handler.API = h.API h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. From 3d595e6966f8a8643b36120e52d58b65dac08d22 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 10 Apr 2018 15:36:18 -0500 Subject: [PATCH 223/311] make sure that slices is treated as inverseSlices on inverse calls --- executor.go | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/executor.go b/executor.go index 2154b60ff..e74cd63ba 100644 --- a/executor.go +++ b/executor.go @@ -84,29 +84,31 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // to send queries to different slices based on orientation. var inverseSlices []uint64 - // If slices aren't specified, then include all of them. - if len(slices) == 0 { - // Determine slices and inverseSlices for use in e.executeCall(). - if needsSlices { - // Round up the number of slices. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } - maxSlice := idx.MaxSlice() - maxInverseSlice := idx.MaxInverseSlice() + // If slices are specified, then use that value for slices or + // inverseSlices. If slices aren't specified, then include all of them. + if len(slices) > 0 { + // For inverse queries, the values of `slices` provided to the Execute() method + // on the remote node actually represents inverseSlices. + inverseSlices = slices + } else if needsSlices { + // Round up the number of slices. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + maxSlice := idx.MaxSlice() + maxInverseSlice := idx.MaxInverseSlice() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) - } + // Generate a slices of all slices. + slices = make([]uint64, maxSlice+1) + for i := range slices { + slices[i] = uint64(i) + } - // Generate a slices of all inverse slices. - inverseSlices = make([]uint64, maxInverseSlice+1) - for i := range inverseSlices { - inverseSlices[i] = uint64(i) - } + // Generate a slices of all inverse slices. + inverseSlices = make([]uint64, maxInverseSlice+1) + for i := range inverseSlices { + inverseSlices[i] = uint64(i) } } @@ -118,7 +120,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - if call.SupportsInverse() && needsSlices { // Fetch frame & row label based on argument. frame, _ := call.Args["frame"].(string) @@ -147,7 +148,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // executeCall executes a call. func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { - if err := e.validateCallArgs(c); err != nil { return nil, err } From 07ca6d57ad19f87412211efe760d81e1ed66f21a Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 09:42:05 -0500 Subject: [PATCH 224/311] variety of cleanup in api and handler don't export QueryValidationSpecRequired add docs to some methods simplify /debug/vars handling using expvar.Handler() implement Version in API and simplify --- api.go | 22 +++++++++++++++++++--- handler.go | 52 ++++++++++++++++------------------------------------ 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/api.go b/api.go index 82ab3a4ce..35d478d30 100644 --- a/api.go +++ b/api.go @@ -31,6 +31,8 @@ import ( "github.com/pkg/errors" ) +// API provides the top level programmatic interface to Pilosa. It is usually +// wrapped by a handler which provides an external interface (e.g. HTTP). type API struct { Holder *Holder // The execution engine for running queries. @@ -46,6 +48,7 @@ type API struct { Logger Logger } +// NewAPI returns a new API instance. func NewAPI() *API { return &API{ Broadcaster: NopBroadcaster, @@ -55,7 +58,8 @@ func NewAPI() *API { } } -func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { +// ExecuteQuery parses a PQL query out of the request and executes it. +func (api *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -67,7 +71,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons ExcludeAttrs: req.ExcludeAttrs, ExcludeBits: req.ExcludeBits, } - results, err := a.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) if err != nil { return resp, err } @@ -86,7 +90,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons } // Retrieve column attributes across all calls. - columnAttrSets, err := a.readColumnAttrSets(a.Holder.Index(req.Index), columnIDs) + columnAttrSets, err := api.readColumnAttrSets(api.Holder.Index(req.Index), columnIDs) if err != nil { return resp, err } @@ -118,6 +122,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet return ax, nil } +// CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { // Create index. index, err := api.Holder.CreateIndex(indexName, options) @@ -845,6 +850,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na return setBits, nil } +// SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) newNode = api.Cluster.nodeByID(id) @@ -869,6 +875,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return oldNode, newNode, nil } +// RemoveNode puts the cluster into the "RESIZING" state and begins the job of +// removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { removeNode := api.Cluster.nodeByID(id) if removeNode == nil { @@ -891,6 +899,14 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "complete current job") } +// State returns the cluster state which is usually "NORMAL", but could be +// "STARTING", "RESIZING", or potentially others. See cluster.go for more +// details. func (api *API) State() string { return api.Cluster.State() } + +// Version returns the Pilosa version. +func (api *API) Version() string { + return strings.TrimPrefix(Version, "v") +} diff --git a/handler.go b/handler.go index d7536f738..4cedeb866 100644 --- a/handler.go +++ b/handler.go @@ -108,14 +108,14 @@ func BuildRouters(handler *Handler) { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index") - h.validators["GetSliceMax"] = QueryValidationSpecRequired().Optional("inverse") - h.validators["PostQuery"] = QueryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") - h.validators["GetExport"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["GetFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["PostFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["GetFragmentBlocks"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["PostFrameRestore"] = QueryValidationSpecRequired("host") + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice").Optional("index") + h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") + h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFrameRestore"] = queryValidationSpecRequired("host") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -154,7 +154,7 @@ func loadCommon(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/id", handler.handleGetID).Methods("GET") @@ -174,7 +174,7 @@ func loadRestricted(router *mux.Router, handler *Handler) { func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") @@ -270,7 +270,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { } filesystem, err := h.FileSystem.New() if err != nil { - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + _ = h.writeQueryResponse(w, r, &QueryResponse{Err: err}) h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") return } @@ -1337,36 +1337,16 @@ func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - version := Version - if strings.HasPrefix(version, "v") { - // make the version string semver-compatible - version = version[1:] - } - if err := json.NewEncoder(w).Encode(struct { + err := json.NewEncoder(w).Encode(struct { Version string `json:"version"` }{ - Version: version, - }); err != nil { + Version: h.API.Version(), + }) + if err != nil { h.Logger.Printf("write version response error: %s", err) } } -// handleExpvar handles /debug/vars requests. -func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) { - // Copied from $GOROOT/src/expvar/expvar.go - w.Header().Set("Content-Type", "application/json; charset=utf-8") - fmt.Fprintf(w, "{\n") - first := true - expvar.Do(func(kv expvar.KeyValue) { - if !first { - fmt.Fprintf(w, ",\n") - } - first = false - fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) - }) - fmt.Fprintf(w, "\n}\n") -} - // QueryResult types. const ( QueryResultTypeNil uint32 = iota @@ -1809,7 +1789,7 @@ type queryValidationSpec struct { args map[string]struct{} } -func QueryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { args := map[string]struct{}{} for _, arg := range requiredArgs { args[arg] = struct{}{} From 0d27139c04b4561ea9871d7ab4e38ade3ea4468e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 11 Apr 2018 12:51:26 -0500 Subject: [PATCH 225/311] test the inverseSlice fix --- server/server_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/server/server_test.go b/server/server_test.go index da62273a4..497ffd7da 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" "testing/quick" + "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" @@ -234,6 +235,49 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } +// Ensure inverse slices get handled correctly in a multi-node query. +func TestMain_InverseSlices(t *testing.T) { + mains := test.MustRunMainWithCluster(t, 2) + + m0 := mains[0] + m1 := mains[1] + + // Make sure to use node0 in the cluster. + var m *test.Main + if m0.Server.NodeID < m1.Server.NodeID { + m = m0 + } else { + m = m1 + } + + // Create frames. + client := m.Client() + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal("create index:", err) + } + if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal("create frame:", err) + } + + // Write data on cluster. + if _, err := m.Query("i", "", fmt.Sprintf(` + SetBit(col=1, frame="f", row=1000) + SetBit(col=1, frame="f", row=2000) + SetBit(col=1, frame="f", row=%d) + `, 1*pilosa.SliceWidth)); err != nil { + t.Fatal("setting bits:", err) + } + + time.Sleep(1 * time.Second) + + // Query the cluster. + if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil { + t.Fatal("another bitmap query:", err) + } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { + t.Fatalf("unexpected result: %s", res) + } +} + // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.MustRunMainWithCluster(t, 2) From feec5f07e1487b74a76a928f2ef45ae50dca9dd6 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 13:30:35 -0500 Subject: [PATCH 226/311] add more doc comments to api --- api.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 35d478d30..be20e5143 100644 --- a/api.go +++ b/api.go @@ -151,6 +151,8 @@ func (api *API) ReadIndex(ctx context.Context, indexName string) (*Index, error) return index, nil } +// DeleteIndex removes the named index. If the index is not found it does +// nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // Delete index from the holder. err := api.Holder.DeleteIndex(indexName) @@ -170,6 +172,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return nil } +// CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { // Find index. index := api.Holder.Index(indexName) @@ -198,6 +201,9 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str return frame, nil } +// DeleteFrame removes the named frame from the named index. If the index is not +// found, an error is returned. If the frame is not found, it is ignored and no +// action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { // Find index. index := api.Holder.Index(indexName) @@ -387,6 +393,8 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st return nil } +// ClusterHosts returns a list of the hosts in the cluster including their ID, +// URL, and which is the coordinator. func (api *API) ClusterHosts(ctx context.Context) []*Node { return api.Cluster.Nodes } @@ -481,6 +489,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s return nil } +// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) if err != nil { @@ -498,10 +507,13 @@ func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error return nil } +// LocalID returns the current node's ID. func (api *API) LocalID() string { return api.Cluster.Node.ID } +// Schema returns information about each index in Pilosa including which frames +// and views they contain. func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } @@ -595,7 +607,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ingore this error becuase views do not exist on all nodes due to slice distribution. + // Ignore this error becuase views do not exist on all nodes due to slice distribution. if err != ErrInvalidView { return err } @@ -675,6 +687,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s return attrs, nil } +// Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { @@ -699,6 +712,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { return err } +// ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { @@ -750,6 +764,8 @@ func (api *API) StatsWithTags(tags []string) StatsClient { return api.Holder.Stats.WithTags(tags...) } +// ClusterLongQueryTime returns the configured threshold for logging/statting +// long running queries. func (api *API) ClusterLongQueryTime() time.Duration { if api.Cluster == nil { return 0 From 9daef2180cd84bebdf827c37440b5b4df0814f7e Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 13:49:51 -0500 Subject: [PATCH 227/311] rename a number of API methods --- api.go | 22 +++++++++++----------- handler.go | 16 ++++++++-------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/api.go b/api.go index be20e5143..7c570b59d 100644 --- a/api.go +++ b/api.go @@ -58,8 +58,8 @@ func NewAPI() *API { } } -// ExecuteQuery parses a PQL query out of the request and executes it. -func (api *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { +// Query parses a PQL query out of the request and executes it. +func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -143,7 +143,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return index, nil } -func (api *API) ReadIndex(ctx context.Context, indexName string) (*Index, error) { +func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { index := api.Holder.Index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -393,9 +393,9 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st return nil } -// ClusterHosts returns a list of the hosts in the cluster including their ID, +// Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. -func (api *API) ClusterHosts(ctx context.Context) []*Node { +func (api *API) Hosts(ctx context.Context) []*Node { return api.Cluster.Nodes } @@ -522,7 +522,7 @@ func (api *API) Status(ctx context.Context) (proto.Message, error) { return api.StatusHandler.ClusterStatus() } -func (api *API) CreateFrameField(ctx context.Context, indexName string, frameName string, field *Field) error { +func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -547,7 +547,7 @@ func (api *API) CreateFrameField(ctx context.Context, indexName string, frameNam return err } -func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameName string, fieldName string) error { +func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -572,7 +572,7 @@ func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameNam return err } -func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { +func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -586,7 +586,7 @@ func (api *API) FrameFields(ctx context.Context, indexName string, frameName str return frame.GetFields() } -func (api *API) FrameViews(ctx context.Context, indexName string, frameName string) ([]*View, error) { +func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { // Retrieve views. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -764,9 +764,9 @@ func (api *API) StatsWithTags(tags []string) StatsClient { return api.Holder.Stats.WithTags(tags...) } -// ClusterLongQueryTime returns the configured threshold for logging/statting +// LongQueryTime returns the configured threshold for logging/statting // long running queries. -func (api *API) ClusterLongQueryTime() time.Duration { +func (api *API) LongQueryTime() time.Duration { if api.Cluster == nil { return 0 } diff --git a/handler.go b/handler.go index 4cedeb866..208c5922f 100644 --- a/handler.go +++ b/handler.go @@ -241,7 +241,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Calculate per request StatsD metrics when the handler is fully configured. statsTags := make([]string, 0, 3) - longQueryTime := h.API.ClusterLongQueryTime() + longQueryTime := h.API.LongQueryTime() if longQueryTime > 0 && dif > longQueryTime { h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) statsTags = append(statsTags, "slow_query") @@ -328,7 +328,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // TODO: Remove req.Index = mux.Vars(r)["index"] - resp, err := h.API.ExecuteQuery(r.Context(), req) + resp, err := h.API.Query(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) @@ -374,7 +374,7 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - index, err := h.API.ReadIndex(r.Context(), indexName) + index, err := h.API.Index(r.Context(), indexName) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return @@ -742,7 +742,7 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { Max: req.Max, } - if err := h.API.CreateFrameField(r.Context(), indexName, frameName, field); err != nil { + if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil { if err == ErrFrameNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { @@ -771,7 +771,7 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) frameName := mux.Vars(r)["frame"] fieldName := mux.Vars(r)["field"] - if err := h.API.DeleteFrameField(r.Context(), indexName, frameName, fieldName); err != nil { + if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil { if err == ErrFrameNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { @@ -790,7 +790,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - fields, err := h.API.FrameFields(r.Context(), indexName, frameName) + fields, err := h.API.Fields(r.Context(), indexName, frameName) if err != nil { switch err { case ErrIndexNotFound: @@ -824,7 +824,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - views, err := h.API.FrameViews(r.Context(), indexName, frameName) + views, err := h.API.Views(r.Context(), indexName, frameName) if err != nil { if err == ErrFrameNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -1329,7 +1329,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { - hosts := h.API.ClusterHosts(r.Context()) + hosts := h.API.Hosts(r.Context()) if err := json.NewEncoder(w).Encode(hosts); err != nil { h.Logger.Printf("write version response error: %s", err) } From 9257029da5fb00736d1ac43d5ec89940d0b3e8eb Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 14:25:02 -0500 Subject: [PATCH 228/311] simplify Status endpoint had to update test which was relying on a fake ClusterStatus implementation. Now the status endpoint uses information directly from Cluster.Nodes and Cluster.state - which is what Server (the usual ClusterStatus impl) uses, so it should make no difference for real clusters. --- api.go | 4 ---- handler.go | 17 ++++------------- handler_test.go | 3 ++- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/api.go b/api.go index 7c570b59d..27827fcec 100644 --- a/api.go +++ b/api.go @@ -518,10 +518,6 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } -func (api *API) Status(ctx context.Context) (proto.Message, error) { - return api.StatusHandler.ClusterStatus() -} - func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) diff --git a/handler.go b/handler.go index 208c5922f..4ce8560b5 100644 --- a/handler.go +++ b/handler.go @@ -289,20 +289,11 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - pb, err := h.API.Status(r.Context()) - if err != nil { - h.Logger.Printf("cluster status error: %s", err) - return + status := getStatusResponse{ + State: h.API.State(), + Nodes: h.API.Hosts(r.Context()), } - - cs, ok := pb.(*internal.ClusterStatus) - if !ok { - panic("status is not a status") - } - if err := json.NewEncoder(w).Encode(getStatusResponse{ - State: cs.State, - Nodes: DecodeNodes(cs.Nodes), - }); err != nil { + if err := json.NewEncoder(w).Encode(status); err != nil { h.Logger.Printf("write status response error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index b605a3650..c7d2a80b7 100644 --- a/handler_test.go +++ b/handler_test.go @@ -141,6 +141,7 @@ func TestHandler_Status(t *testing.T) { h := test.NewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) + h.API.Cluster.SetState(pilosa.ClusterStateNormal) h.API.StatusHandler = s s.Handler = h @@ -148,7 +149,7 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101},"isCoordinator":false}]}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } From 16539bbab12ccb1743714cd59f0e27e63406c228 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 10 Apr 2018 11:16:27 -0500 Subject: [PATCH 229/311] WIP: implement Min/Max BSI queries --- client_test.go | 36 ++++++++- executor.go | 196 ++++++++++++++++++++++++++++++++++++++++++++++- fragment.go | 64 ++++++++++++++++ fragment_test.go | 73 ++++++++++++++++++ frame.go | 40 ++++++++++ view.go | 43 +++++++++++ 6 files changed, 445 insertions(+), 7 deletions(-) diff --git a/client_test.go b/client_test.go index cdbaba788..4db9156c5 100644 --- a/client_test.go +++ b/client_test.go @@ -332,14 +332,44 @@ func TestClient_ImportValue(t *testing.T) { t.Fatal(err) } + // Verify Sum. sum, cnt, err := frame.FieldSum(nil, fld.Name) if err != nil { t.Fatal(err) } - - // Verify data. if sum != 50 || cnt != 3 { - t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt) + t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) + } + + // Verify Min. + min, cnt, err := frame.FieldMin(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if min != -10 || cnt != 1 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt) + } + + // Verify Min with Filter. + filter, err := frame.FieldRange(fld.Name, pql.GT, 40) + if err != nil { + t.Fatal(err) + } + min, cnt, err = frame.FieldMin(filter, fld.Name) // TODO: change this to use the client + if err != nil { + t.Fatal(err) + } + if min != -100 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + } + + // Verify Max. + max, cnt, err := frame.FieldMax(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if max != 40 || cnt != 1 { + t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt) } } diff --git a/executor.go b/executor.go index e74cd63ba..022e48dda 100644 --- a/executor.go +++ b/executor.go @@ -157,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeSum(ctx, index, c, slices, opt) + case "Min": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMin(ctx, index, c, slices, opt) + case "Max": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMax(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -233,6 +239,76 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other, nil } +// executeFieldMin executes a Min() call. +func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MinCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return MinCount{}, errors.New("Min(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return MinCount{}, errors.New("Min(): field required") + } + + if len(c.Children) > 1 { + return MinCount{}, errors.New("Min() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMinSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(MinCount) + return other.Smaller(v.(MinCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return MinCount{}, err + } + other, _ := result.(MinCount) + + if other.Count == 0 { + return MinCount{}, nil + } + return other, nil +} + +// executeFieldMax executes a Max() call. +func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MaxCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return MaxCount{}, errors.New("Max(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return MaxCount{}, errors.New("Max(): field required") + } + + if len(c.Children) > 1 { + return MaxCount{}, errors.New("Max() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMaxSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(MaxCount) + return other.Larger(v.(MaxCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return MaxCount{}, err + } + other, _ := result.(MaxCount) + + if other.Count == 0 { + return MaxCount{}, nil + } + return other, nil +} + // executeBitmapCall executes a call that returns a bitmap. func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) { // Execute calls in bulk on each remote node and merge. @@ -318,7 +394,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } } -// executeSumCountSlice executes calculates the sum & count for fields on a slice. +// executeSumCountSlice calculates the sum and count for fields on a slice. func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { var filter *Bitmap if len(c.Children) == 1 { @@ -342,12 +418,12 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return SumCount{}, nil } - view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) - if view == nil { + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { return SumCount{}, nil } - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { return SumCount{}, err } @@ -357,6 +433,84 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq }, nil } +// executeFieldMinSlice calculates the min for fields on a slice. +func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MinCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return MinCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return MinCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return MinCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return MinCount{}, nil + } + + fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + if err != nil { + return MinCount{}, err + } + return MinCount{ + Min: int64(fmin) + field.Min, + Count: int64(fcount), + }, nil +} + +// executeFieldMaxSlice calculates the max for fields on a slice. +func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MaxCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return MaxCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return MaxCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return MaxCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return MaxCount{}, nil + } + + fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + if err != nil { + return MaxCount{}, err + } + return MaxCount{ + Max: int64(fmax) + field.Min, + Count: int64(fcount), + }, nil +} + // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. @@ -1619,3 +1773,37 @@ func decodeSumCount(pb *internal.SumCount) SumCount { Count: pb.Count, } } + +// MinCount represents a grouping of min and count for Min() calls. +type MinCount struct { + Min int64 `json:"min"` + Count int64 `json:"count"` +} + +// Smaller returns the smaller of the two MinCounts. +func (mc *MinCount) Smaller(other MinCount) MinCount { + if mc.Count == 0 || other.Count < mc.Count { + return other + } + return MinCount{ + Min: mc.Min, + Count: mc.Count, + } +} + +// MaxCount represents a grouping of max and count for Max() calls. +type MaxCount struct { + Max int64 `json:"max"` + Count int64 `json:"count"` +} + +// Larger returns the larger of the two MaxCounts. +func (mc *MaxCount) Larger(other MaxCount) MaxCount { + if mc.Count == 0 || other.Count < mc.Count { + return other + } + return MaxCount{ + Max: mc.Max, + Count: mc.Count, + } +} diff --git a/fragment.go b/fragment.go index 438a80ae7..f40c0c8d9 100644 --- a/fragment.go +++ b/fragment.go @@ -614,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e return sum, count, nil } +// FieldMin returns the min of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := consider.Difference(row) + count = x.Count() + if count > 0 { + consider = x + } else { + min += (1 << ii) + if ii == 0 { + count = consider.Count() + } + } + } + + return min, count, nil +} + +// FieldMax returns the max of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := row.Intersect(consider) + count = x.Count() + if count > 0 { + max += (1 << ii) + consider = x + } else if ii == 0 { + count = consider.Count() + } + } + + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { switch op { diff --git a/fragment_test.go b/fragment_test.go index 6a4a2f377..6b49b5d75 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) { }) } +// Ensure a fragment can find the max of field values. +func TestFragment_FieldMinMax(t *testing.T) { + const bitDepth = 16 + + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 0, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if min != test.exp { + t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 2818, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if max != test.exp { + t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) +} + // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { const bitDepth = 16 diff --git a/frame.go b/frame.go index def41d1b1..a8213e264 100644 --- a/frame.go +++ b/frame.go @@ -763,6 +763,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } +// FieldMin returns the min for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmin) + field.Min, int64(vcount), nil +} + +// FieldMax returns the max for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmax) + field.Min, int64(vcount), nil +} + func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) { // Retrieve and validate field. field := f.Field(name) diff --git a/view.go b/view.go index f775f1b0e..dbcc73b8d 100644 --- a/view.go +++ b/view.go @@ -357,6 +357,49 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e return sum, count, nil } +// FieldMin returns the min and count of a field. +func (v *View) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + var minHasValue bool + for _, f := range v.Fragments() { + fmin, fcount, err := f.FieldMin(filter, bitDepth) + if err != nil { + return min, count, err + } + // Don't consider a min based on zero columns. + if fcount == 0 { + continue + } + + if !minHasValue { + min = fmin + minHasValue = true + count += fcount + continue + } + + if fmin < min { + min = fmin + count += fcount + } + } + return min, count, nil +} + +// FieldMax returns the max and count of a field. +func (v *View) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + for _, f := range v.Fragments() { + fmax, fcount, err := f.FieldMax(filter, bitDepth) + if err != nil { + return max, count, err + } + if fcount > 0 && fmax > max { + max = fmax + count += fcount + } + } + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { bm := NewBitmap() From ee4dbbf328735bdff1a38317c985fcb2e8759898 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 10 Apr 2018 18:01:05 -0500 Subject: [PATCH 230/311] min/max documentation --- docs/data-model.md | 2 +- docs/glossary.md | 6 ++++- docs/query-language.md | 50 ++++++++++++++++++++++++++++++++++++++++++ docs/tutorials.md | 46 ++++++++++++++++++++++++++++++++++++-- fragment_test.go | 2 +- 5 files changed, 101 insertions(+), 5 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 6a57a96a1..655d1633e 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -112,7 +112,7 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") #### BSI Range-Encoding Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded -bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. +bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. diff --git a/docs/glossary.md b/docs/glossary.md index 612c65dd4..6157eba57 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -14,7 +14,7 @@ nav = [] [Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. -[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries. +[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries. Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. @@ -30,8 +30,12 @@ nav = [] [Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. +[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). + MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. +[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). + Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data. diff --git a/docs/query-language.md b/docs/query-language.md index 6b0584320..efb143996 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -523,6 +523,56 @@ Range(frame="stats", commitactivity >< [100, 200]) This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. +#### Min + +**Spec:** + +``` +Min([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered. + +**Result Type:** object with the min and count of columns containing the min value. + +**Examples:** + +Query the size of all repositories. +``` +Min(frame="stats", field="diskusage") +``` + +Return `{"min":4,"count":2}` + +* Result is the smallest repository in kilobytes, plus the number of repositories of that size. + +#### Max + +**Spec:** + +``` +Max([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. + +**Result Type:** object with the max and count of columns containing the max value. + +**Examples:** + +Query the size of all repositories. +``` +Max(frame="stats", field="diskusage") +``` + +Return `{"max":88,"count":13}` + +* Result is the largest repository in kilobytes, plus the number of repositories of that size. + #### Sum **Spec:** diff --git a/docs/tutorials.md b/docs/tutorials.md index 2401cdfe4..6715e1cd1 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -247,7 +247,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: ``` request @@ -333,7 +333,7 @@ curl localhost:10101/index/patients/query \ ``` The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. +You can also provide a filter to the `Sum()` function to find the average age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ @@ -344,6 +344,48 @@ curl localhost:10101/index/patients/query \ ``` Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. +To find the minimum age of all patients, run a `Min` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(frame="measurements", field="age")' +``` +``` response +{"results":[{"min":19,"count":1}]} +``` +The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"min":57,"count":1}]} +``` + +To find the maximum age of all patients, run a `Max` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(frame="measurements", field="age")' +``` +``` response +{"results":[{"max":71,"count":1}]} +``` +The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"max":34,"count":1}]} +``` + ### Storing Row and Column Attributes #### Introduction diff --git a/fragment_test.go b/fragment_test.go index 6b49b5d75..d7c97a5cf 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -256,7 +256,7 @@ func TestFragment_FieldSum(t *testing.T) { }) } -// Ensure a fragment can find the max of field values. +// Ensure a fragment can find the min and max of field values. func TestFragment_FieldMinMax(t *testing.T) { const bitDepth = 16 From 04f5d5875c9ab49c48321be08bf0c5d2c1e13195 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 12 Apr 2018 11:28:35 -0500 Subject: [PATCH 231/311] api docs, rename funcs, refactor usage of internal All exported funcs in api.go are now documented Several poorly named methods of API and Cluster were renamed. Particularly, the word Fragment was often changed to Slice in cases where it was really a slice being specified and not a fragment. several methods which received or returned internal data structures have been refactored to be more opaque. Deprecation logging was added to input definition methods. --- api.go | 105 +++++++++++++++++++++++++++++++++++++++++------- cluster.go | 12 +++--- executor.go | 6 +-- fragment.go | 4 +- handler.go | 46 +++++---------------- holder.go | 2 +- pilosa.go | 7 ++++ test/cluster.go | 4 +- 8 files changed, 120 insertions(+), 66 deletions(-) diff --git a/api.go b/api.go index 27827fcec..f57d159f2 100644 --- a/api.go +++ b/api.go @@ -19,6 +19,7 @@ import ( "encoding/csv" "fmt" "io" + "io/ioutil" "net/http" "reflect" "strconv" @@ -143,6 +144,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return index, nil } +// Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { index := api.Holder.Index(indexName) if index == nil { @@ -230,9 +232,11 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str return nil } +// ExportCSV encodes the fragment designated by the index,frame,view,slice as +// CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return ErrClusterDoesNotOwnSlice } @@ -262,11 +266,20 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin return nil } -func (api *API) FragmentNodes(ctx context.Context, indexName string, slice uint64) []*Node { - return api.Cluster.FragmentNodes(indexName, slice) +// SliceNodes returns the node and all replicas which should contain a slice's data. +func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) []*Node { + return api.Cluster.SliceNodes(indexName, slice) } -func (api *API) FragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (*Fragment, error) { +// WriterTo is an interface for any Object which knows how to serialize itself to an io.Writer +type WriterTo interface { + WriteTo(w io.Writer) (n int64, err error) +} + +// MarshalFragment returns an object which can write the specified fragment's data +// to an io.Writer. The serialized data can be read back into a fragment with +// the UnmarshalFragment API call. +func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (WriterTo, error) { // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -275,7 +288,10 @@ func (api *API) FragmentData(ctx context.Context, indexName string, frameName st return f, nil } -func (api *API) WriteFragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { +// UnmarshalFragment creates a new fragment (if necessary) and reads data from a +// Reader which was previously written by MarshalFragment to populate the +// fragment's data. +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -301,19 +317,38 @@ func (api *API) WriteFragmentData(ctx context.Context, indexName string, frameNa return nil } -func (api *API) FragmentBlockData(ctx context.Context, req internal.BlockDataRequest) (internal.BlockDataResponse, error) { +// FragmentBlockData is an endpoint for internal usage. It is not guaranteed to +// return anything useful. Currently it returns protobuf encoded row and column +// ids from a "block" which is a subdivision of a fragment. +func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + reqBytes, err := ioutil.ReadAll(body) + if err != nil { + return nil, BadRequestError{errors.Wrap(err, "read body error")} + } + var req internal.BlockDataRequest + if err := proto.Unmarshal(reqBytes, &req); err != nil { + return nil, BadRequestError{errors.Wrap(err, "unmarshal body error")} + } + // Retrieve fragment from holder. f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) if f == nil { - return internal.BlockDataResponse{}, ErrFragmentNotFound + return nil, ErrFragmentNotFound } - // Read data - var resp internal.BlockDataResponse + var resp = internal.BlockDataResponse{} resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) - return resp, nil + + // Encode response. + buf, err := proto.Marshal(&resp) + if err != nil { + return nil, errors.Wrap(err, "merge block response encoding error: %s") + } + + return buf, nil } +// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) @@ -326,6 +361,8 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName return blocks, nil } +// RestoreFrame reads all the data that this host should have for a given frame +// from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { // Create a client for the remote cluster. client := NewInternalHTTPClientFromURI(host, api.RemoteClient) @@ -351,7 +388,7 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st // Loop over each slice and import it if this node owns it. for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { continue } @@ -399,7 +436,10 @@ func (api *API) Hosts(ctx context.Context) []*Node { return api.Cluster.Nodes } +// CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. +Please open an issue if you need to continue using it.`) // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -430,7 +470,9 @@ func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inp return nil } +// InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -444,7 +486,9 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN return inputDef, nil } +// DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -467,7 +511,9 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp return nil } +// WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + api.Logger.Printf("WriteInput is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -499,10 +545,24 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return nil } -func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error { +// PostClusterMessage is for internal use. It decodes a protobuf message out of +// the body and forwards it to the BroadcastHandler. +func (api *API) PostClusterMessage(ctx context.Context, reqBody io.Reader) error { + // Read entire body. + body, err := ioutil.ReadAll(reqBody) + if err != nil { + return errors.Wrap(err, "reading body") + } + + // Marshal into request object. + pb, err := UnmarshalMessage(body) + if err != nil { + return errors.Wrap(err, "unmarshaling message") + } + // Forward the error message. if err := api.BroadcastHandler.ReceiveMessage(pb); err != nil { - return err + return errors.Wrap(err, "receiving message") } return nil } @@ -518,6 +578,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } +// CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) @@ -543,6 +604,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str return err } +// DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) @@ -568,6 +630,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str return err } +// Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { index := api.Holder.index(indexName) if index == nil { @@ -582,6 +645,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) return frame.GetFields() } +// Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { // Retrieve views. f := api.Holder.Frame(indexName, frameName) @@ -594,6 +658,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( return views, nil } +// DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { // Retrieve frame. f := api.Holder.Frame(indexName, frameName) @@ -623,6 +688,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri return err } +// IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { // Retrieve index from holder. index := api.Holder.Index(indexName) @@ -723,6 +789,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return err } +// ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { // Retrieve index by name. index := api.Holder.Index(indexName) @@ -734,6 +801,8 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti return index.SetTimeQuantum(timeQuantum) } +// ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: +// what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { // Retrieve index by name. frame := api.Holder.Frame(indexName, frameName) @@ -745,14 +814,19 @@ func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, fr return frame.SetTimeQuantum(timeQuantum) } +// MaxSlices returns the maximum slice number for each index in a map. func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() } +// MaxInverseSlices returns the maximum inverse slice number for each index in a +// map. func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxInverseSlices() } +// StatsWithTags returns an instance of whatever implementation of StatsClient +// pilosa is using with the given tags. func (api *API) StatsWithTags(tags []string) StatsClient { if api.Holder == nil || api.Cluster == nil { return nil @@ -771,7 +845,7 @@ func (api *API) LongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return nil, nil, ErrClusterDoesNotOwnSlice } @@ -793,7 +867,7 @@ func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*I return index, frame, nil } -// inputJSONDataParser validates input json file and executes SetBit. +// inputJSONDataParser validates input json file and executes SetBit. Deprecated - remove with input definition stuff. func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef, err := index.InputDefinition(name) if err != nil { @@ -903,6 +977,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { return removeNode, nil } +// ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if !api.Cluster.IsCoordinator() { return ErrNodeNotCoordinator diff --git a/cluster.go b/cluster.go index c490c58cb..abf753892 100644 --- a/cluster.go +++ b/cluster.go @@ -654,7 +654,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { - nodes := c.FragmentNodes(idx, i) + nodes := c.SliceNodes(idx, i) for _, n := range nodes { // for each frame/view combination: for frame, views := range frameViews { @@ -807,14 +807,14 @@ func (c *Cluster) Partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.PartitionN)) } -// FragmentNodes returns a list of nodes that own a fragment. -func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node { +// SliceNodes returns a list of nodes that own a fragment. +func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { return c.PartitionNodes(c.Partition(index, slice)) } -// OwnsFragment returns true if a host owns a fragment. -func (c *Cluster) OwnsFragment(nodeID string, index string, slice uint64) bool { - return Nodes(c.FragmentNodes(index, slice)).ContainsID(nodeID) +// OwnsSlice returns true if a host owns a fragment. +func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { + return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) } // PartitionNodes returns a list of nodes that own a partition. diff --git a/executor.go b/executor.go index 2154b60ff..c2cd860c4 100644 --- a/executor.go +++ b/executor.go @@ -941,7 +941,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) @@ -1042,7 +1042,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) @@ -1385,7 +1385,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m loop: for _, slice := range slices { - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop diff --git a/fragment.go b/fragment.go index 438a80ae7..c44a07380 100644 --- a/fragment.go +++ b/fragment.go @@ -1702,7 +1702,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) SyncFragment() error { // Determine replica set. - nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) if len(nodes) == 1 { return nil } @@ -1784,7 +1784,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet var clients []InternalClient - for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { + for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { if s.Node.ID == node.ID { continue } diff --git a/handler.go b/handler.go index 4ce8560b5..33d2e5846 100644 --- a/handler.go +++ b/handler.go @@ -1169,7 +1169,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.API.FragmentNodes(r.Context(), index, slice) + nodes := h.API.SliceNodes(r.Context(), index, slice) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1188,7 +1188,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from holder. - f, err := h.API.FragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1210,7 +1210,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } - if err = h.API.WriteFragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { + if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { if err == ErrFrameNotFound { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) } else { @@ -1221,19 +1221,11 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) // handleGetFragmentBlockData handles GET /fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - // Read request object. - var req internal.BlockDataRequest - if body, err := ioutil.ReadAll(r.Body); err != nil { - http.Error(w, "ready body error", http.StatusBadRequest) - return - } else if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, "unmarshal body error", http.StatusBadRequest) - return - } - - resp, err := h.API.FragmentBlockData(r.Context(), req) + buf, err := h.API.FragmentBlockData(r.Context(), r.Body) if err != nil { - if err == ErrFragmentNotFound { + if _, ok := err.(BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if err == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1241,13 +1233,6 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ return } - // Encode response. - buf, err := proto.Marshal(&resp) - if err != nil { - h.Logger.Printf("merge block response encoding error: %s", err) - return - } - // Write response. w.Header().Set("Content-Type", "application/protobuf") w.Header().Set("Content-Length", strconv.Itoa(len(buf))) @@ -1742,23 +1727,10 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - // Read entire body. - body, err := ioutil.ReadAll(r.Body) + err := h.API.PostClusterMessage(r.Context(), r.Body) if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - pb, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.PostClusterMessage(r.Context(), pb); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return } if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { diff --git a/holder.go b/holder.go index 22c53c7a4..570433ce4 100644 --- a/holder.go +++ b/holder.go @@ -614,7 +614,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.Node.ID, di.Name, slice) { + if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { continue } diff --git a/pilosa.go b/pilosa.go index 4b28daccb..ffb836ae8 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,6 +82,13 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) +// BadRequestError wraps an error value to signify that a request could not be +// read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest +// would be returned. +type BadRequestError struct { + error +} + // Regular expression to validate index and frame names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) diff --git a/test/cluster.go b/test/cluster.go index 8362ca964..308a7e69c 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -129,7 +129,7 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / pilosa.SliceWidth - nodes := c0.FragmentNodes(index, slice) + nodes := c0.SliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) @@ -153,7 +153,7 @@ func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name s // Determine which node should receive the SetFieldValue. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := columnID / pilosa.SliceWidth - nodes := c0.FragmentNodes(index, slice) + nodes := c0.SliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) From 47a5ed84bd70e78d480b6d308ba2589c044b3c81 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 12 Apr 2018 14:46:44 -0500 Subject: [PATCH 232/311] add executor min/max tests. fix related bugs. --- client_test.go | 2 +- executor.go | 4 +-- executor_test.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/client_test.go b/client_test.go index 4db9156c5..2fde7a7c7 100644 --- a/client_test.go +++ b/client_test.go @@ -355,7 +355,7 @@ func TestClient_ImportValue(t *testing.T) { if err != nil { t.Fatal(err) } - min, cnt, err = frame.FieldMin(filter, fld.Name) // TODO: change this to use the client + min, cnt, err = frame.FieldMin(filter, fld.Name) if err != nil { t.Fatal(err) } diff --git a/executor.go b/executor.go index 022e48dda..83f35e7d3 100644 --- a/executor.go +++ b/executor.go @@ -1782,7 +1782,7 @@ type MinCount struct { // Smaller returns the smaller of the two MinCounts. func (mc *MinCount) Smaller(other MinCount) MinCount { - if mc.Count == 0 || other.Count < mc.Count { + if mc.Count == 0 || (other.Min < mc.Min && other.Count > 0) { return other } return MinCount{ @@ -1799,7 +1799,7 @@ type MaxCount struct { // Larger returns the larger of the two MaxCounts. func (mc *MaxCount) Larger(other MaxCount) MaxCount { - if mc.Count == 0 || other.Count < mc.Count { + if mc.Count == 0 || (other.Max > mc.Max && other.Count > 0) { return other } return MaxCount{ diff --git a/executor_test.go b/executor_test.go index f2e0d210d..d5fa6e5d6 100644 --- a/executor_test.go +++ b/executor_test.go @@ -599,6 +599,98 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { } } +// Ensure Min() and Max() queries can be executed. +func TestExecutor_Execute_MinMax(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=3) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=1, col=1) + SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + + SetFieldValue(frame=f, foo=20, col=0) + SetFieldValue(frame=f, foo=-5, col=1) + SetFieldValue(frame=f, foo=-5, col=2) + SetFieldValue(frame=f, foo=10, col=3) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.MinCount{Min: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.MaxCount{Max: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) +} + // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() From 8ec801a7322ce64fce3bc34d16c646b67e837cc1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 12 Apr 2018 16:37:49 -0500 Subject: [PATCH 233/311] use io.WriterTo instead of custom --- api.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api.go b/api.go index f57d159f2..c447ef9f2 100644 --- a/api.go +++ b/api.go @@ -271,15 +271,10 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) return api.Cluster.SliceNodes(indexName, slice) } -// WriterTo is an interface for any Object which knows how to serialize itself to an io.Writer -type WriterTo interface { - WriteTo(w io.Writer) (n int64, err error) -} - // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { From a04ae99fb30d44e9ebc3f3a59b849f125f751084 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 13 Apr 2018 03:41:59 +0300 Subject: [PATCH 234/311] updated cluster tutorial with feedback --- docs/tutorials.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 4d1a06c42..5e0d5376d 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -70,7 +70,7 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. -One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. +One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated. Create `node1.config.toml` in the project directory and paste the following in it: From 3f303d1098e6173d9f0af0bc031772f99525d67d Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 12 Apr 2018 19:52:48 -0500 Subject: [PATCH 235/311] remove global defaults from config.go these were occaisionally referenced elsewhere in the codebase - in all but one case, there were workarounds that are actually better I think. In the one case there wasn't I created a single top level DefaultConfig object which is instantiated with all the default values and can be referred to if necessary. There was a bug in fragment.go with the way MaxWritesPerRequest was treated if it was 0. Elsewhere, 0 meant no limit, but here, it would have caused a division by 0. Changed the default metrics provider from "nop" to "none", although "nop" will still work. Previously, any value other than "statsd" or "expvar" was treated as "nop", but I've changed this behavior to return an error if an invalid string is provided. I think this is better behavior, because in the case that someone bothered to change the default, they were probably interested in actually getting stats, and might be annoyed when it silently failed. --- cluster.go | 9 +- config.go | 215 ++++++++++++++++-------------------- config_test.go | 2 +- ctl/generate_config_test.go | 3 +- ctl/import_test.go | 2 +- docs/configuration.md | 2 +- fragment.go | 10 +- server.go | 5 +- server/server.go | 14 +-- 9 files changed, 118 insertions(+), 144 deletions(-) diff --git a/cluster.go b/cluster.go index abf753892..c7945d938 100644 --- a/cluster.go +++ b/cluster.go @@ -275,11 +275,10 @@ type Cluster struct { // NewCluster returns a new instance of Cluster with defaults. func NewCluster() *Cluster { return &Cluster{ - Hasher: &jmphasher{}, - PartitionN: DefaultPartitionN, - ReplicaN: DefaultReplicaN, - MaxWritesPerRequest: DefaultMaxWritesPerRequest, - EventReceiver: NopEventReceiver, + Hasher: &jmphasher{}, + PartitionN: DefaultPartitionN, + ReplicaN: DefaultReplicaN, + EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel jobs: make(map[int64]*ResizeJob), diff --git a/config.go b/config.go index 84f199dc3..c03f03ac3 100644 --- a/config.go +++ b/config.go @@ -25,93 +25,8 @@ const ( ClusterGossip = "gossip" ) -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - - // DefaultHost is the default hostname to use. - DefaultHost = "localhost" - - // DefaultPort is the default port to use with the hostname. - DefaultPort = "10101" - - // DefaultClusterDisabled sets the node intercommunication method. - DefaultClusterDisabled = false - - // DefaultMetrics sets the internal metrics to no-op. - DefaultMetrics = "nop" - - // DefaultMaxWritesPerRequest is the default number of writes per request. - DefaultMaxWritesPerRequest = 5000 - - // Gossip config based on memberlist.Config. - - // Port indicates the port to which pilosa should bind for internal state sharing. - DefaultGossipPort = "14000" - - // StreamTimeout is the timeout for establishing a stream connection with - // a remote node for a full state sync, and for stream read and write - // operations. Maps to memberlist TCPTimeout. - DefaultGossipStreamTimeout = 10 * time.Second - - // SuspicionMult is the multiplier for determining the time an - // inaccessible node is considered suspect before declaring it dead. - // The actual timeout is calculated using the formula: - // - // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval - // - // This allows the timeout to scale properly with expected propagation - // delay with a larger cluster size. The higher the multiplier, the longer - // an inaccessible node is considered part of the cluster before declaring - // it dead, giving that suspect node more time to refute if it is indeed - // still alive. - DefaultGossipSuspicionMult = 4 - - // PushPullInterval is the interval between complete state syncs. - // Complete state syncs are done with a single node over TCP and are - // quite expensive relative to standard gossiped messages. Setting this - // to zero will disable state push/pull syncs completely. - // - // Setting this interval lower (more frequent) will increase convergence - // speeds across larger clusters at the expense of increased bandwidth - // usage. - DefaultGossipPushPullInterval = 30 * time.Second - - // ProbeInterval and ProbeTimeout are used to configure probing behavior - // for memberlist. - // - // ProbeInterval is the interval between random node probes. Setting - // this lower (more frequent) will cause the memberlist cluster to detect - // failed nodes more quickly at the expense of increased bandwidth usage. - // - // ProbeTimeout is the timeout to wait for an ack from a probed node - // before assuming it is unhealthy. This should be set to 99-percentile - // of RTT (round-trip time) on your network. - DefaultGossipProbeInterval = 1 * time.Second - DefaultGossipProbeTimeout = 500 * time.Millisecond - - // Interval and Nodes are used to configure the gossip - // behavior of memberlist. - // - // Interval is the interval between sending messages that need - // to be gossiped that haven't been able to piggyback on probing messages. - // If this is set to zero, non-piggyback gossip is disabled. By lowering - // this value (more frequent) gossip messages are propagated across - // the cluster more quickly at the expense of increased bandwidth. - // - // Nodes is the number of random nodes to send gossip messages to - // per Interval. Increasing this number causes the gossip messages - // to propagate across the cluster more quickly at the expense of - // increased bandwidth. - // - // ToTheDeadTime is the interval after which a node has died that - // we will still try to gossip to it. This gives it a chance to refute. - DefaultGossipInterval = 200 * time.Millisecond - DefaultGossipNodes = 3 - DefaultGossipToTheDeadTime = 30 * time.Second - - DefaultMetricPollInterval = 0 * time.Minute -) +// DefaultConfig is a Config structure that holds all the default values. +var DefaultConfig = NewConfig() // TLSConfig contains TLS configuration type TLSConfig struct { @@ -125,20 +40,27 @@ type TLSConfig struct { // Config represents the configuration for the command. type Config struct { + // DataDir is the default data directory. DataDir string `toml:"data-dir"` - Bind string `toml:"bind"` + // Bind is the host:port on which Pilosa will listen. + Bind string `toml:"bind"` - // Limits the number of mutating commands that can be in a single request to - // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. + // MaxWritesPerRequest limits the number of mutating commands that can be in + // a single request to the server. This includes SetBit, ClearBit, + // SetRowAttrs & SetColumnAttrs. MaxWritesPerRequest int `toml:"max-writes-per-request"` + // LogPath configures where Pilosa will write logs. LogPath string `toml:"log-path"` - Verbose bool `toml:"verbose"` + + // Verbose toggles verbose logging which can be useful for debugging. + Verbose bool `toml:"verbose"` // TLS TLS TLSConfig Cluster struct { + // Disabled controls whether clustering functionality is enabled. Disabled bool `toml:"disabled"` Coordinator bool `toml:"coordinator"` ReplicaN int `toml:"replicas"` @@ -146,18 +68,69 @@ type Config struct { LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` + // Gossip config is based around memberlist.Config. Gossip struct { - Port string `toml:"port"` - Seeds []string `toml:"seeds"` - Key string `toml:"key"` - StreamTimeout Duration `toml:"stream-timeout"` - SuspicionMult int `toml:"suspicion-mult"` + // Port indicates the port to which pilosa should bind for internal state sharing. + Port string `toml:"port"` + Seeds []string `toml:"seeds"` + Key string `toml:"key"` + // StreamTimeout is the timeout for establishing a stream connection with + // a remote node for a full state sync, and for stream read and write + // operations. Maps to memberlist TCPTimeout. + StreamTimeout Duration `toml:"stream-timeout"` + // SuspicionMult is the multiplier for determining the time an + // inaccessible node is considered suspect before declaring it dead. + // The actual timeout is calculated using the formula: + // + // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval + // + // This allows the timeout to scale properly with expected propagation + // delay with a larger cluster size. The higher the multiplier, the longer + // an inaccessible node is considered part of the cluster before declaring + // it dead, giving that suspect node more time to refute if it is indeed + // still alive. + SuspicionMult int `toml:"suspicion-mult"` + // PushPullInterval is the interval between complete state syncs. + // Complete state syncs are done with a single node over TCP and are + // quite expensive relative to standard gossiped messages. Setting this + // to zero will disable state push/pull syncs completely. + // + // Setting this interval lower (more frequent) will increase convergence + // speeds across larger clusters at the expense of increased bandwidth + // usage. PushPullInterval Duration `toml:"push-pull-interval"` - ProbeTimeout Duration `toml:"probe-timeout"` - ProbeInterval Duration `toml:"probe-interval"` - Nodes int `toml:"nodes"` - Interval Duration `toml:"interval"` - ToTheDeadTime Duration `toml:"to-the-dead-time"` + // ProbeInterval and ProbeTimeout are used to configure probing behavior + // for memberlist. + // + // ProbeInterval is the interval between random node probes. Setting + // this lower (more frequent) will cause the memberlist cluster to detect + // failed nodes more quickly at the expense of increased bandwidth usage. + // + // ProbeTimeout is the timeout to wait for an ack from a probed node + // before assuming it is unhealthy. This should be set to 99-percentile + // of RTT (round-trip time) on your network. + ProbeInterval Duration `toml:"probe-interval"` + ProbeTimeout Duration `toml:"probe-timeout"` + + // Interval and Nodes are used to configure the gossip + // behavior of memberlist. + // + // Interval is the interval between sending messages that need + // to be gossiped that haven't been able to piggyback on probing messages. + // If this is set to zero, non-piggyback gossip is disabled. By lowering + // this value (more frequent) gossip messages are propagated across + // the cluster more quickly at the expense of increased bandwidth. + // + // Nodes is the number of random nodes to send gossip messages to + // per Interval. Increasing this number causes the gossip messages + // to propagate across the cluster more quickly at the expense of + // increased bandwidth. + // + // ToTheDeadTime is the interval after which a node has died that + // we will still try to gossip to it. This gives it a chance to refute. + Interval Duration `toml:"interval"` + Nodes int `toml:"nodes"` + ToTheDeadTime Duration `toml:"to-the-dead-time"` } `toml:"gossip"` AntiEntropy struct { @@ -165,51 +138,55 @@ type Config struct { } `toml:"anti-entropy"` Metric struct { - Service string `toml:"service"` + // Service can be statsd, expvar, or none. + Service string `toml:"service"` + // Host tells the statsd client where to write. Host string `toml:"host"` PollInterval Duration `toml:"poll-interval"` - Diagnostics bool `toml:"diagnostics"` + // Diagnostics toggles sending some limited diagnostic information to + // Pilosa's developers. + Diagnostics bool `toml:"diagnostics"` } `toml:"metric"` } // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ - DataDir: DefaultDataDir, - Bind: ":" + DefaultPort, - MaxWritesPerRequest: DefaultMaxWritesPerRequest, + DataDir: "~/.pilosa", + Bind: ":10101", + MaxWritesPerRequest: 5000, // LogPath: "", // Verbose: false, TLS: TLSConfig{}, } // Cluster config. - c.Cluster.Disabled = DefaultClusterDisabled + c.Cluster.Disabled = false // c.Cluster.Coordinator = false c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Hosts = []string{} c.Cluster.LongQueryTime = Duration(time.Minute) // Gossip config. - // c.Gossip.Port = "" + c.Gossip.Port = "14000" // c.Gossip.Seeds = []string{} // c.Gossip.Key = "" - c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) - c.Gossip.SuspicionMult = DefaultGossipSuspicionMult - c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) - c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout) - c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval) - c.Gossip.Nodes = DefaultGossipNodes - c.Gossip.Interval = Duration(DefaultGossipInterval) - c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime) + c.Gossip.StreamTimeout = Duration(10 * time.Second) + c.Gossip.SuspicionMult = 4 + c.Gossip.PushPullInterval = Duration(30 * time.Second) + c.Gossip.ProbeInterval = Duration(1 * time.Second) + c.Gossip.ProbeTimeout = Duration(500 * time.Millisecond) + c.Gossip.Interval = Duration(200 * time.Millisecond) + c.Gossip.Nodes = 3 + c.Gossip.ToTheDeadTime = Duration(30 * time.Second) // AntiEntropy config. - c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) + c.AntiEntropy.Interval = Duration(10 * time.Minute) // Metric config. - c.Metric.Service = DefaultMetrics + c.Metric.Service = "none" // c.Metric.Host = "" - c.Metric.PollInterval = Duration(DefaultMetricPollInterval) + c.Metric.PollInterval = Duration(0 * time.Minute) c.Metric.Diagnostics = true return c diff --git a/config_test.go b/config_test.go index 1359e2e31..c07486ddb 100644 --- a/config_test.go +++ b/config_test.go @@ -25,7 +25,7 @@ import ( func Test_NewConfig(t *testing.T) { c := pilosa.NewConfig() - if c.Cluster.Disabled != pilosa.DefaultClusterDisabled { + if c.Cluster.Disabled { t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled) } diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 801ab4055..56f392bba 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -17,7 +17,6 @@ package ctl import ( "bytes" "context" - "github.com/pilosa/pilosa" "io" "os" "strings" @@ -35,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) { io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), pilosa.DefaultHost) { + } else if !strings.Contains(buf.String(), "localhost:10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import_test.go b/ctl/import_test.go index bd259e342..55ae6d243 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -134,7 +134,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = pilosa.DefaultHost + cm.Host = "anyhost" cm.Index = "i" cm.Frame = "f" file, err := ioutil.TempFile("", "import.csv") diff --git a/docs/configuration.md b/docs/configuration.md index 4d71dee8e..2528299a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -213,7 +213,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h ``` #### Metric Service -* Description: Which stats service to use. Choose from [statsd, expvar]. +* Description: Which stats service to use. Choose from [statsd, expvar, none]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' * Config: diff --git a/fragment.go b/fragment.go index c44a07380..56342690c 100644 --- a/fragment.go +++ b/fragment.go @@ -1832,15 +1832,19 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. total := len(set.ColumnIDs) + len(clear.ColumnIDs) - buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(s.Cluster.MaxWritesPerRequest)))) + maxWrites := s.Cluster.MaxWritesPerRequest + if maxWrites <= 0 { + maxWrites = 5000 + } + buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/server.go b/server.go index 21f974ad8..57ae423df 100644 --- a/server.go +++ b/server.go @@ -37,8 +37,7 @@ import ( // Default server settings. const ( - DefaultAntiEntropyInterval = 10 * time.Minute - DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" + DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Ensure Server implements interfaces. @@ -108,7 +107,7 @@ func NewServer() *Server { NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: DefaultAntiEntropyInterval, + AntiEntropyInterval: time.Duration(DefaultConfig.AntiEntropy.Interval), MetricInterval: 0, DiagnosticInterval: 0, diff --git a/server/server.go b/server/server.go index 62c7a70a5..7f66d31ed 100644 --- a/server/server.go +++ b/server/server.go @@ -20,7 +20,6 @@ package server import ( - "errors" "fmt" "io" "log" @@ -40,6 +39,7 @@ import ( "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/statsd" + "github.com/pkg/errors" ) func init() { @@ -255,13 +255,7 @@ func (m *Command) SetupNetworking() error { return nil } - // Set internal port (string). - gossipPortStr := pilosa.DefaultGossipPort - if m.Config.Gossip.Port != "" { - gossipPortStr = m.Config.Gossip.Port - } - - gossipPort, err := strconv.Atoi(gossipPortStr) + gossipPort, err := strconv.Atoi(m.Config.Gossip.Port) if err != nil { return err } @@ -318,7 +312,9 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { return pilosa.NewExpvarStatsClient(), nil case "statsd": return statsd.NewStatsClient(host) - default: + case "nop", "none": return pilosa.NopStatsClient, nil + default: + return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") } } From 0c35094bfc43b311239fc1da47a550445ebde44b Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 13 Apr 2018 10:21:47 -0500 Subject: [PATCH 236/311] few more docs tweaks --- docs/configuration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2528299a4..2d9d109b9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -197,12 +197,12 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h ```toml [profile] - cpu = "/path/to/somewhere" + cpu = "/path/to/somewhere" ``` #### Profile CPU Time -* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set. +* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set. * Flag: `--profile.cpu-time="30s"` * Env: `PILOSA_PROFILE_CPU_TIME="30s" * Config: @@ -236,7 +236,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Metric Poll Interval -* Description: Polling interval for runtime metrics. +* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected. * Flag: `metric.poll-interval=”0m15s”` * Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s` * Config: @@ -248,7 +248,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Metric Diagnostics -* Description: Enable diagnostic reporting. To disable diagnostics set to false. +* Description: Enable reporting of limited usage statistics to Pilosa developers. To disable, set to false. * Flag: `metric.diagnostics` * Env: `PILOSA_METRIC_DIAGNOSTICS` * Config: From 7233597f35951eb5827cc3d2a059b190213dba4c Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Sun, 15 Apr 2018 17:20:14 -0500 Subject: [PATCH 237/311] remove DefaultConfig global (only used once). Fix data-dir comment --- config.go | 6 ++---- server.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config.go b/config.go index c03f03ac3..483965916 100644 --- a/config.go +++ b/config.go @@ -25,9 +25,6 @@ const ( ClusterGossip = "gossip" ) -// DefaultConfig is a Config structure that holds all the default values. -var DefaultConfig = NewConfig() - // TLSConfig contains TLS configuration type TLSConfig struct { // CertificatePath contains the path to the certificate (.crt or .pem file) @@ -40,7 +37,8 @@ type TLSConfig struct { // Config represents the configuration for the command. type Config struct { - // DataDir is the default data directory. + // DataDir is the directory where Pilosa stores both indexed data and + // running state such as cluster topology information. DataDir string `toml:"data-dir"` // Bind is the host:port on which Pilosa will listen. Bind string `toml:"bind"` diff --git a/server.go b/server.go index 57ae423df..bb629b66a 100644 --- a/server.go +++ b/server.go @@ -107,7 +107,7 @@ func NewServer() *Server { NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: time.Duration(DefaultConfig.AntiEntropy.Interval), + AntiEntropyInterval: time.Duration(NewConfig().AntiEntropy.Interval), MetricInterval: 0, DiagnosticInterval: 0, From fe0ba6280f27f998e34502c8f81f426714e3e6ca Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 16 Apr 2018 16:08:04 -0500 Subject: [PATCH 238/311] deprecate RangeEnabled, but leave in API the RangeEnabled option now has no effect, but it still exists in the API. A few tests still use it to ensure this. This would only be considered a breaking change if someone was relying on Pilosa to enforce the RangeEnabled: false option to prevent fields being created in certain frames. This seems unlikely. --- client_test.go | 2 +- cluster_test.go | 1 - cmd/import.go | 2 +- ctl/import_test.go | 6 ++---- diagnostics.go | 6 ++---- docs/api-reference.md | 4 ++-- docs/tutorials.md | 1 - executor_test.go | 7 ------- frame.go | 35 +---------------------------------- frame_test.go | 5 ----- handler.go | 2 -- handler_test.go | 23 +++++++++++------------ index.go | 11 +---------- index_test.go | 24 ++++++++++-------------- pilosa.go | 23 ++++++++++------------- 15 files changed, 41 insertions(+), 111 deletions(-) diff --git a/client_test.go b/client_test.go index cdbaba788..4bd0bade6 100644 --- a/client_test.go +++ b/client_test.go @@ -310,7 +310,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true, Fields: []*pilosa.Field{&fld}}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}}) if err != nil { t.Fatal(err) } diff --git a/cluster_test.go b/cluster_test.go index c5c01d58d..6c5d41829 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -427,7 +427,6 @@ func TestCluster_ResizeStates(t *testing.T) { // Add Field Data to node0. if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{ InverseEnabled: false, - RangeEnabled: true, //CacheType: pilosa.CacheTypeNone, Fields: []*pilosa.Field{ { diff --git a/cmd/import.go b/cmd/import.go index 8692fa318..b6b78bdbc 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -63,7 +63,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") - flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame") + flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) diff --git a/ctl/import_test.go b/ctl/import_test.go index 55ae6d243..6cbc8271a 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -86,9 +86,7 @@ func TestImportCommand_Run(t *testing.T) { } // Ensure that the ImportValue path runs (note: we have specified a value -// for cm.Field. Because the handler doesn't return errors (it sends them -// to the logger), we don't get an error returned at `cm.Run()` even though -// we haven't setup frame `f` to be RangeEnabled. +// for cm.Field.) func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} @@ -117,7 +115,7 @@ func TestImportCommand_RunValue(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"rangeEnabled": true, "fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) cm.Index = "i" cm.Frame = "f" diff --git a/diagnostics.go b/diagnostics.go index 5d3f6699b..57423d0a5 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -219,10 +219,8 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { numIndexes += 1 for _, frame := range index.Frames() { numFrames += 1 - if frame.rangeEnabled { - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields) - } + if fields, err := frame.GetFields(); err == nil { + bsiFieldCount += len(fields) } if frame.TimeQuantum() != "" { timeQuantumEnabled = true diff --git a/docs/api-reference.md b/docs/api-reference.md index 446235cdc..0c0314f85 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -106,7 +106,7 @@ The request payload is in JSON, and may contain the `options` field. The `option * `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -* `rangeEnabled` (boolean): Enables range-encoded fields in this frame. +* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: @@ -130,7 +130,7 @@ curl localhost:10101/index/user/frame/language \ ``` request curl localhost:10101/index/repository/frame/stats \ -X POST \ - -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' + -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` ``` response {} diff --git a/docs/tutorials.md b/docs/tutorials.md index 5e0d5376d..601825fa9 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -259,7 +259,6 @@ In addition to storing rows of bits, a frame can also contain fields that store curl localhost:10101/index/patients/frame/measurements \ -X POST \ -d '{"options":{ - "rangeEnabled": true, "fields": [ {"name": "age", "type": "int", "min": 0, "max": 120}, {"name": "weight", "type": "int", "min": 0, "max": 500}, diff --git a/executor_test.go b/executor_test.go index f2e0d210d..1f8a38b99 100644 --- a/executor_test.go +++ b/executor_test.go @@ -280,7 +280,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2}, @@ -330,7 +329,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}, }, @@ -611,7 +609,6 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, @@ -621,7 +618,6 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, }, @@ -723,7 +719,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, @@ -733,7 +728,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, }, @@ -742,7 +736,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100}, }, diff --git a/frame.go b/frame.go index def41d1b1..6ed76d61c 100644 --- a/frame.go +++ b/frame.go @@ -33,7 +33,6 @@ import ( const ( DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false - DefaultRangeEnabled = false // Default ranked frame cache DefaultCacheSize = 50000 @@ -59,7 +58,6 @@ type Frame struct { cacheType string cacheSize uint32 timeQuantum TimeQuantum - rangeEnabled bool fields []*Field Logger Logger @@ -88,7 +86,6 @@ func NewFrame(path, index, name string) (*Frame, error) { cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, //timeQuantum - rangeEnabled: DefaultRangeEnabled, //fields Logger: NopLogger, @@ -145,11 +142,6 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } -// RangeEnabled returns true if range fields can be stored on this frame. -func (f *Frame) RangeEnabled() bool { - return f.rangeEnabled -} - // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -188,7 +180,6 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ InverseEnabled: f.inverseEnabled, - RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, @@ -268,7 +259,6 @@ func (f *Frame) loadMeta() error { f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize f.timeQuantum = "" - f.rangeEnabled = DefaultRangeEnabled //f.fields return nil } else if err != nil { @@ -287,7 +277,6 @@ func (f *Frame) loadMeta() error { } f.cacheSize = pb.CacheSize f.timeQuantum = TimeQuantum(pb.TimeQuantum) - f.rangeEnabled = pb.RangeEnabled f.fields = decodeFields(pb.Fields) return nil @@ -365,11 +354,6 @@ func (f *Frame) CreateField(field *Field) error { f.mu.Lock() defer f.mu.Unlock() - // Ensure frame supports fields. - if !f.RangeEnabled() { - return ErrFrameFieldsNotAllowed - } - // Append field. if err := f.addField(field); err != nil { return err @@ -402,11 +386,6 @@ func (f *Frame) GetFields() ([]*Field, error) { f.mu.RLock() defer f.mu.RUnlock() - // Ensure the frame supports fields. - if !f.RangeEnabled() { - return nil, ErrFrameFieldsNotAllowed - } - err := f.loadMeta() if err != nil { return nil, err @@ -420,11 +399,6 @@ func (f *Frame) DeleteField(name string) error { f.mu.Lock() defer f.mu.Unlock() - // Ensure frame supports fields. - if !f.RangeEnabled() { - return ErrFrameFieldsNotAllowed - } - // Remove field. if err := f.deleteField(name); err != nil { return err @@ -890,11 +864,6 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - // Verify that this frame is range-encoded. - if !f.RangeEnabled() { - return fmt.Errorf("Frame not RangeEnabled: %s", f.name) - } - viewName := ViewFieldPrefix + fieldName // Get the field so we know bitDepth. field := f.Field(fieldName) @@ -991,7 +960,7 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { InverseEnabled bool `json:"inverseEnabled,omitempty"` - RangeEnabled bool `json:"rangeEnabled,omitempty"` + RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` @@ -1009,7 +978,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { } return &internal.FrameMeta{ InverseEnabled: o.InverseEnabled, - RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, CacheSize: o.CacheSize, TimeQuantum: string(o.TimeQuantum), @@ -1023,7 +991,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { } return &FrameOptions{ InverseEnabled: options.InverseEnabled, - RangeEnabled: options.RangeEnabled, CacheType: options.CacheType, CacheSize: options.CacheSize, TimeQuantum: TimeQuantum(options.TimeQuantum), diff --git a/frame_test.go b/frame_test.go index 2107bd703..78014bbce 100644 --- a/frame_test.go +++ b/frame_test.go @@ -77,7 +77,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, @@ -123,7 +122,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, }, @@ -161,7 +159,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, }, @@ -181,7 +178,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, }, @@ -201,7 +197,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, }, diff --git a/handler.go b/handler.go index bc6c625ef..5e105073a 100644 --- a/handler.go +++ b/handler.go @@ -788,8 +788,6 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { fallthrough case ErrFrameNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case ErrFrameFieldsNotAllowed: - http.Error(w, err.Error(), http.StatusBadRequest) default: http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/handler_test.go b/handler_test.go index 063421667..efa931970 100644 --- a/handler_test.go +++ b/handler_test.go @@ -902,7 +902,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -927,7 +927,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrInvalidFieldType", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil { + if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -949,7 +949,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrInvalidFieldRange", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil { + if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -972,8 +972,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrFieldAlreadyExists", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, - Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, + Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, }); err != nil { t.Fatal(err) } @@ -1006,7 +1005,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { @@ -1034,7 +1033,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { t.Run("ErrFieldNotFound", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { @@ -1071,7 +1070,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil { @@ -1105,7 +1104,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{RangeEnabled: false}) + _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}) resp, err := http.Get(s.URL + "/index/i/frame/f1/fields") if err != nil { @@ -1113,12 +1112,12 @@ func TestHandler_Frame_GetFields(t *testing.T) { } if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusBadRequest { + } else if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %d", resp.StatusCode) } else if body, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatal(err) - } else if strings.TrimSpace(string(body)) != `frame fields not allowed` { - t.Fatalf("unexpected body: %q", body) + } else if strings.TrimSpace(string(body)) == `frame fields not allowed` { + t.Fatalf("shouldn't get frame fields not allowed error: %q", body) } }) diff --git a/index.go b/index.go index fef4eb92c..039816322 100644 --- a/index.go +++ b/index.go @@ -404,13 +404,7 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { // Validate mutually exclusive options if ranges are enabled. if opt.RangeEnabled { - if opt.InverseEnabled { - return nil, ErrInverseRangeNotAllowed - } - } else { - if len(opt.Fields) > 0 { - return nil, ErrFrameFieldsNotAllowed - } + i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame") } // Validate fields. @@ -452,9 +446,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } f.inverseEnabled = opt.InverseEnabled - f.rangeEnabled = opt.RangeEnabled - - f.rangeEnabled = opt.RangeEnabled // Set fields. f.fields = opt.Fields diff --git a/index_test.go b/index_test.go index c8cd238e8..439523c3c 100644 --- a/index_test.go +++ b/index_test.go @@ -99,7 +99,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Create frame with schema and verify it exists. if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, + RangeEnabled: false, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, @@ -124,14 +124,14 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { + t.Run("ErrInverseRangeAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - InverseEnabled: true, RangeEnabled: true, - }); err != pilosa.ErrInverseRangeNotAllowed { + InverseEnabled: true, + }); err != nil { t.Fatal(err) } }) @@ -141,8 +141,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeRanked, + CacheType: pilosa.CacheTypeRanked, }); err != nil { t.Fatal(err) } @@ -152,15 +151,14 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), }); err != nil { t.Fatal(err) } }) - t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { + t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() @@ -168,7 +166,7 @@ func TestIndex_CreateFrame(t *testing.T) { Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt}, }, - }); err != pilosa.ErrFrameFieldsNotAllowed { + }); err != nil { t.Fatal(err) } }) @@ -178,7 +176,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "", Type: pilosa.FieldTypeInt}, }, @@ -192,7 +189,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: "bad_type"}, }, @@ -206,7 +202,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, + RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, diff --git a/pilosa.go b/pilosa.go index ffb836ae8..307f9dadc 100644 --- a/pilosa.go +++ b/pilosa.go @@ -46,19 +46,16 @@ var ( ErrInputDefinitionActionRequired = errors.New("field definitions require an action") ErrInputDefinitionNotFound = errors.New("input-definition not found") - ErrFieldNotFound = errors.New("field not found") - ErrFieldExists = errors.New("field already exists") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") - ErrRangeCacheNotAllowed = errors.New("range cache not allowed") - ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") - ErrInvalidFieldValueType = errors.New("invalid field value type") - ErrFieldValueTooLow = errors.New("field value too low") - ErrFieldValueTooHigh = errors.New("field value too high") - ErrInvalidRangeOperation = errors.New("invalid range operation") - ErrInvalidBetweenValue = errors.New("invalid value for between operation") + ErrFieldNotFound = errors.New("field not found") + ErrFieldExists = errors.New("field already exists") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInvalidFieldValueType = errors.New("invalid field value type") + ErrFieldValueTooLow = errors.New("field value too low") + ErrFieldValueTooHigh = errors.New("field value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") + ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From c7f2d0f675adfe3027df89b2af02f83f65bfa9fe Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 16 Apr 2018 16:16:14 -0500 Subject: [PATCH 239/311] update unreleased section of CHANGELOG with rangeEnabled deprecation --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4d17174..950ecaad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) +- Deprecated RangeEnabled option. ([#1205](https://github.com/pilosa/pilosa/pull/1205)) ### Removed From 0a8d573fb396b05bfe06db217b08ae5552c1a6e7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 16 Apr 2018 17:12:28 -0500 Subject: [PATCH 240/311] WIP: Remove SecurityManager. Implement api restrictions in api package. --- api.go | 246 ++++++++++++++++++++++++++++++++++++++++- cluster.go | 12 +- handler.go | 59 +++------- handler_test.go | 2 +- security_manager.go | 32 ------ server.go | 4 +- server/cluster_test.go | 3 + test/cluster.go | 1 + test/handler.go | 2 - 9 files changed, 262 insertions(+), 99 deletions(-) delete mode 100644 security_manager.go diff --git a/api.go b/api.go index c447ef9f2..970766842 100644 --- a/api.go +++ b/api.go @@ -59,8 +59,30 @@ func NewAPI() *API { } } +// functionStates specifies the api functions that are valid for each +// cluster state. +var functionStates = map[string][]int{ + ClusterStateStarting: functionCommon, + ClusterStateNormal: append(functionCommon, functionNormal...), + ClusterStateResizing: append(functionCommon, functionResizing...), +} + +func (api *API) validate(f int) error { + state := api.Cluster.State() + for _, fnc := range functionStates[state] { + if f == fnc { + return nil + } + } + return fmt.Errorf("api function not allowed in state %s", state) +} + // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + if err := api.validate(apiQuery); err != nil { + return QueryResponse{}, errors.Wrap(err, "validate api function: query") + } + resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -125,6 +147,10 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + if err := api.validate(apiCreateIndex); err != nil { + return nil, errors.Wrap(err, "validate api function: create index") + } + // Create index. index, err := api.Holder.CreateIndex(indexName, options) if err != nil { @@ -146,6 +172,10 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { + if err := api.validate(apiIndex); err != nil { + return nil, errors.Wrap(err, "validate api function: index") + } + index := api.Holder.Index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -156,6 +186,10 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // DeleteIndex removes the named index. If the index is not found it does // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + if err := api.validate(apiDeleteIndex); err != nil { + return errors.Wrap(err, "validate api function: delete index") + } + // Delete index from the holder. err := api.Holder.DeleteIndex(indexName) if err != nil { @@ -176,6 +210,10 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { + if err := api.validate(apiCreateFrame); err != nil { + return nil, errors.Wrap(err, "validate api function: create frame") + } + // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -207,6 +245,10 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // found, an error is returned. If the frame is not found, it is ignored and no // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { + if err := api.validate(apiDeleteFrame); err != nil { + return errors.Wrap(err, "validate api function: delete frame") + } + // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -235,6 +277,10 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // ExportCSV encodes the fragment designated by the index,frame,view,slice as // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + if err := api.validate(apiExportCSV); err != nil { + return errors.Wrap(err, "validate api function: export csv") + } + // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) @@ -267,14 +313,22 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin } // SliceNodes returns the node and all replicas which should contain a slice's data. -func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) []*Node { - return api.Cluster.SliceNodes(indexName, slice) +func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { + if err := api.validate(apiSliceNodes); err != nil { + return nil, errors.Wrap(err, "validate api function: slice nodes") + } + + return api.Cluster.SliceNodes(indexName, slice), nil } // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { + if err := api.validate(apiMarshalFragment); err != nil { + return nil, errors.Wrap(err, "validate api function: marshal fragment") + } + // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -287,6 +341,10 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // Reader which was previously written by MarshalFragment to populate the // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { + if err := api.validate(apiUnmarshalFragment); err != nil { + return errors.Wrap(err, "validate api function: unmarshal fragment") + } + // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -316,6 +374,10 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + if err := api.validate(apiFragmentBlockData); err != nil { + return nil, errors.Wrap(err, "validate api function: fragment block data") + } + reqBytes, err := ioutil.ReadAll(body) if err != nil { return nil, BadRequestError{errors.Wrap(err, "read body error")} @@ -337,7 +399,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // Encode response. buf, err := proto.Marshal(&resp) if err != nil { - return nil, errors.Wrap(err, "merge block response encoding error: %s") + return nil, errors.Wrap(err, "merge block response encoding error") } return buf, nil @@ -345,6 +407,10 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { + if err := api.validate(apiFragmentBlocks); err != nil { + return nil, errors.Wrap(err, "validate api function: fragment blocks") + } + // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -359,6 +425,10 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // RestoreFrame reads all the data that this host should have for a given frame // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { + if err := api.validate(apiRestoreFrame); err != nil { + return errors.Wrap(err, "validate api function: restore frame") + } + // Create a client for the remote cluster. client := NewInternalHTTPClientFromURI(host, api.RemoteClient) @@ -433,6 +503,10 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + if err := api.validate(apiCreateInputDefinition); err != nil { + return errors.Wrap(err, "validate api function: create input definition") + } + api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. Please open an issue if you need to continue using it.`) // Find index. @@ -467,6 +541,10 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + if err := api.validate(apiInputDefinition); err != nil { + return nil, errors.Wrap(err, "validate api function: input definition") + } + api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) // Find index. index := api.Holder.Index(indexName) @@ -483,6 +561,10 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + if err := api.validate(apiDeleteInputDefinition); err != nil { + return errors.Wrap(err, "validate api function: delete input definition") + } + api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) @@ -508,6 +590,10 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + if err := api.validate(apiWriteInput); err != nil { + return errors.Wrap(err, "validate api function: write input") + } + api.Logger.Printf("WriteInput is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) @@ -532,6 +618,10 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { + if err := api.validate(apiRecalculateCaches); err != nil { + return errors.Wrap(err, "validate api function: recalculate caches") + } + err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") @@ -542,7 +632,11 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // PostClusterMessage is for internal use. It decodes a protobuf message out of // the body and forwards it to the BroadcastHandler. -func (api *API) PostClusterMessage(ctx context.Context, reqBody io.Reader) error { +func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { + if err := api.validate(apiClusterMessage); err != nil { + return errors.Wrap(err, "validate api function: cluster message") + } + // Read entire body. body, err := ioutil.ReadAll(reqBody) if err != nil { @@ -575,6 +669,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { + if err := api.validate(apiCreateField); err != nil { + return errors.Wrap(err, "validate api function: create field") + } + // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -601,6 +699,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { + return errors.Wrap(err, "validate api function: delete field") + } + // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -627,6 +729,10 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { + if err := api.validate(apiFields); err != nil { + return nil, errors.Wrap(err, "validate api function: fields") + } + index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -642,6 +748,10 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { + if err := api.validate(apiViews); err != nil { + return nil, errors.Wrap(err, "validate api function: views") + } + // Retrieve views. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -655,6 +765,10 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { + if err := api.validate(apiDeleteView); err != nil { + return errors.Wrap(err, "validate api function: delete view") + } + // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -685,6 +799,10 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiIndexAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api function: index attr diff") + } + // Retrieve index from holder. index := api.Holder.Index(indexName) if index == nil { @@ -715,6 +833,10 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At } func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFrameAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api function: frame attr diff") + } + // Retrieve index from holder. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -746,6 +868,10 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { + if err := api.validate(apiImport); err != nil { + return errors.Wrap(err, "validate api function: import") + } + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { return err @@ -771,6 +897,10 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { + if err := api.validate(apiImportValue); err != nil { + return errors.Wrap(err, "validate api function: import value") + } + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { return err @@ -786,6 +916,10 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { + if err := api.validate(apiModifyIndexTimeQuantum); err != nil { + return errors.Wrap(err, "validate api function: modify index time quantum") + } + // Retrieve index by name. index := api.Holder.Index(indexName) if index == nil { @@ -799,6 +933,10 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { + if err := api.validate(apiModifyFrameTimeQuantum); err != nil { + return errors.Wrap(err, "validate api function: modify frame time quantum") + } + // Retrieve index by name. frame := api.Holder.Frame(indexName, frameName) if frame == nil { @@ -933,6 +1071,10 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + if err := api.validate(apiSetCoordinator); err != nil { + return nil, nil, errors.Wrap(err, "validate api function: set coordinator") + } + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) newNode = api.Cluster.nodeByID(id) if newNode == nil { @@ -959,6 +1101,10 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // RemoveNode puts the cluster into the "RESIZING" state and begins the job of // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { + if err := api.validate(apiRemoveNode); err != nil { + return nil, errors.Wrap(err, "validate api function: remove node") + } + removeNode := api.Cluster.nodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") @@ -974,6 +1120,10 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { + if err := api.validate(apiResizeAbort); err != nil { + return errors.Wrap(err, "validate api function: resize abort") + } + if !api.Cluster.IsCoordinator() { return ErrNodeNotCoordinator } @@ -992,3 +1142,91 @@ func (api *API) State() string { func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } + +// API validation constants. +const ( + apiClusterMessage int = iota + apiCreateField + apiCreateFrame + apiCreateIndex + apiCreateInputDefinition + apiDeleteField + apiDeleteFrame + apiDeleteIndex + apiDeleteInputDefinition + apiDeleteView + apiExportCSV + apiFields + apiFragmentBlockData + apiFragmentBlocks + apiFrameAttrDiff + //apiHosts // not implemented + apiImport + apiImportValue + apiIndex + apiIndexAttrDiff + apiInputDefinition + //apiLocalID // not implemented + //apiLongQueryTime // not implemented + apiMarshalFragment + //apiMaxInverseSlices // not implemented + //apiMaxSlices // not implemented + apiModifyFrameTimeQuantum + apiModifyIndexTimeQuantum + apiQuery + apiRecalculateCaches + apiRemoveNode + apiResizeAbort + apiRestoreFrame + //apiSchema // not implemented + apiSetCoordinator + apiSliceNodes + //apiState // not implemented + //apiStatsWithTags // not implemented + apiUnmarshalFragment + //apiVersion // not implemented + apiViews + apiWriteInput +) + +var functionCommon = []int{ + apiClusterMessage, + apiMarshalFragment, + apiSetCoordinator, +} + +var functionResizing = []int{ + apiResizeAbort, +} + +var functionNormal = []int{ + apiCreateField, + apiCreateFrame, + apiCreateIndex, + apiCreateInputDefinition, + apiDeleteField, + apiDeleteFrame, + apiDeleteIndex, + apiDeleteInputDefinition, + apiDeleteView, + apiExportCSV, + apiFields, + apiFragmentBlockData, + apiFragmentBlocks, + apiFrameAttrDiff, + apiImport, + apiImportValue, + apiIndex, + apiIndexAttrDiff, + apiInputDefinition, + apiModifyFrameTimeQuantum, + apiModifyIndexTimeQuantum, + apiQuery, + apiRecalculateCaches, + apiRemoveNode, + apiRestoreFrame, + apiSliceNodes, + apiUnmarshalFragment, + apiViews, + apiWriteInput, +} diff --git a/cluster.go b/cluster.go index c7945d938..7e871698d 100644 --- a/cluster.go +++ b/cluster.go @@ -264,7 +264,6 @@ type Cluster struct { // Close management wg sync.WaitGroup closing chan struct{} - prefect SecurityManager Logger Logger @@ -285,8 +284,7 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), - Logger: NopLogger, - prefect: &NopSecurityManager{}, + Logger: NopLogger, } } @@ -433,19 +431,11 @@ func (c *Cluster) setState(state string) { var doCleanup bool switch state { - case ClusterStateResizing: - c.prefect.SetRestricted() case ClusterStateNormal: - c.prefect.SetNormal() - // Don't change routing for these states: - // - ClusterStateStarting - // If state is RESIZING -> NORMAL then run cleanup. if c.state == ClusterStateResizing { doCleanup = true } - default: - panic(fmt.Sprintf("invalid cluster state: %s", state)) } c.state = state diff --git a/handler.go b/handler.go index bc6c625ef..4e20fcb07 100644 --- a/handler.go +++ b/handler.go @@ -43,9 +43,7 @@ import ( type Handler struct { Router *mux.Router - FileSystem FileSystem - NormalRouter *mux.Router - RestrictedRouter *mux.Router + FileSystem FileSystem // The execution engine for running queries. Executor interface { @@ -83,29 +81,11 @@ func NewHandler() *Handler { FileSystem: NopFileSystem, Logger: NopLogger, } - BuildRouters(handler) + handler.Router = NewRouter(handler) handler.populateValidators() return handler } -// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints. -func BuildRouters(handler *Handler) { - router := mux.NewRouter() - loadCommon(router, handler) - loadNormal(router, handler) - handler.NormalRouter = router - router.Use(handler.queryArgValidator) - - // Restricted router. - router = mux.NewRouter() - loadCommon(router, handler) - loadRestricted(router, handler) - handler.RestrictedRouter = router - router.Use(handler.queryArgValidator) - - handler.SetRestricted() -} - func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") @@ -138,17 +118,9 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { }) } -// SetNormal is a method of the SecurityManager interface which provides normal URI routing. -func (h *Handler) SetNormal() { - h.Router = h.NormalRouter -} - -// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing. -func (h *Handler) SetRestricted() { - h.Router = h.RestrictedRouter -} - -func loadCommon(router *mux.Router, handler *Handler) { +// NewRouter creates a new mux http router. +func NewRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() router.HandleFunc("/", handler.handleWebUI).Methods("GET") router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") @@ -162,16 +134,9 @@ func loadCommon(router *mux.Router, handler *Handler) { router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - router.Use(handler.queryArgValidator) -} -func loadRestricted(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted) - router.Use(handler.queryArgValidator) -} -func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") @@ -212,10 +177,8 @@ func loadNormal(router *mux.Router, handler *Handler) { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") -} - -func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.API.State()), http.StatusMethodNotAllowed) + router.Use(handler.queryArgValidator) + return router } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { @@ -1169,7 +1132,11 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.API.SliceNodes(r.Context(), index, slice) + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1727,7 +1694,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - err := h.API.PostClusterMessage(r.Context(), r.Body) + err := h.API.ClusterMessage(r.Context(), r.Body) if err != nil { // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/handler_test.go b/handler_test.go index 063421667..78f9e470f 100644 --- a/handler_test.go +++ b/handler_test.go @@ -160,7 +160,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { h := test.NewHandler() h.API.Cluster = test.NewCluster(1) - h.SetRestricted() + h.API.Cluster.SetState(pilosa.ClusterStateResizing) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) diff --git a/security_manager.go b/security_manager.go deleted file mode 100644 index 80b696c38..000000000 --- a/security_manager.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -// SecurityManager provides the ability to limit access to restricted endpoints -// during cluster configuration. -type SecurityManager interface { - SetRestricted() - SetNormal() -} - -// NopSecurityManager provides a no-op implementation of the SecurityManager interface. -type NopSecurityManager struct { -} - -// SetRestricted no-op. -func (sdm *NopSecurityManager) SetRestricted() {} - -// SetNormal no-op. -func (sdm *NopSecurityManager) SetNormal() {} diff --git a/server.go b/server.go index bb629b66a..049672686 100644 --- a/server.go +++ b/server.go @@ -169,10 +169,8 @@ func (s *Server) Open() error { s.Handler.API.StatusHandler = s s.Handler.API.URI = s.URI s.Handler.API.Cluster = s.Cluster - s.Handler.Executor = e - - s.Cluster.prefect = s.Handler s.Handler.API.Executor = e + s.Handler.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster diff --git a/server/cluster_test.go b/server/cluster_test.go index 04d53b438..7f4886612 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -101,6 +101,9 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal(err) } + m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) + m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Expected indexes and Frames diff --git a/test/cluster.go b/test/cluster.go index 308a7e69c..77cf496a8 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -50,6 +50,7 @@ func NewCluster(n int) *pilosa.Cluster { c.Node = c.Nodes[0] c.Coordinator = c.Nodes[0].ID + c.SetState(pilosa.ClusterStateNormal) return c } diff --git a/test/handler.go b/test/handler.go index cb90cd383..c325d3243 100644 --- a/test/handler.go +++ b/test/handler.go @@ -47,8 +47,6 @@ func NewHandler() *Handler { // Handler test messages can no-op. h.API.Broadcaster = pilosa.NopBroadcaster - h.SetNormal() - return h } From 58492c7eab45744f64deb995caef554523c9578f Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 17 Apr 2018 07:54:25 -0500 Subject: [PATCH 241/311] remove period in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 950ecaad7..09562dc06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Deprecated RangeEnabled option. ([#1205](https://github.com/pilosa/pilosa/pull/1205)) +- Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) ### Removed From e56b8717b5d8359ee30cac3ae9b54758ab54fabe Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 17 Apr 2018 09:00:57 -0500 Subject: [PATCH 242/311] add better inverseEnabled with field test --- index_test.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/index_test.go b/index_test.go index 439523c3c..25e093d4b 100644 --- a/index_test.go +++ b/index_test.go @@ -128,12 +128,43 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + frame, err := index.CreateFrame("f", pilosa.FrameOptions{ RangeEnabled: true, InverseEnabled: true, - }); err != nil { + Fields: []*pilosa.Field{ + &pilosa.Field{ + Name: "myfield", + Type: pilosa.FieldTypeInt, + Min: -20, + Max: 100, + }, + }, + }) + if err != nil { t.Fatal(err) } + + ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil) + if !ch || err != nil { + t.Fatal(ch, err) + } + ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil) + if !ch || err != nil { + t.Fatal(ch, err) + } + ch, err = frame.SetFieldValue(1, "myfield", 87) + if !ch || err != nil { + t.Fatal(ch, err) + } + views := frame.Views() + if len(views) != 3 { + var names string + for _, v := range views { + names = names + v.Name() + " " + } + t.Fatalf("Unexpected views: %s", names) + } + }) t.Run("ErrRangeCacheAllowed", func(t *testing.T) { From 54680652137aee9b16cb4999b15d66d6c27020fa Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Apr 2018 13:01:36 -0500 Subject: [PATCH 243/311] improve apiFunc error handling. change slice to map in function validation. --- Makefile | 6 +- api.go | 173 +++++++++++++++++++++++++--------------------- apifunc_string.go | 16 +++++ pilosa.go | 6 ++ 4 files changed, 120 insertions(+), 81 deletions(-) create mode 100644 apifunc_string.go diff --git a/Makefile b/Makefile index 9e2c6fa3f..cab932fba 100644 --- a/Makefile +++ b/Makefile @@ -80,8 +80,12 @@ generate-protoc: require-protoc require-protoc-gen-gofast generate-statik: require-statik go generate github.com/pilosa/pilosa/statik +# `go generate` stringers +generate-stringer: + go generate github.com/pilosa/pilosa + # `go generate` all needed packages -generate: generate-protoc generate-statik +generate: generate-protoc generate-statik generate-stringer # Create Docker image from Dockerfile docker: diff --git a/api.go b/api.go index 970766842..6852a1f87 100644 --- a/api.go +++ b/api.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:generate stringer -type=apiFunc + package pilosa import ( @@ -61,26 +63,35 @@ func NewAPI() *API { // functionStates specifies the api functions that are valid for each // cluster state. -var functionStates = map[string][]int{ +var functionStates = map[string]map[apiFunc]struct{}{ ClusterStateStarting: functionCommon, - ClusterStateNormal: append(functionCommon, functionNormal...), - ClusterStateResizing: append(functionCommon, functionResizing...), + ClusterStateNormal: appendMap(functionCommon, functionNormal), + ClusterStateResizing: appendMap(functionCommon, functionResizing), } -func (api *API) validate(f int) error { - state := api.Cluster.State() - for _, fnc := range functionStates[state] { - if f == fnc { - return nil - } +func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { + r := make(map[apiFunc]struct{}) + for k, v := range a { + r[k] = v } - return fmt.Errorf("api function not allowed in state %s", state) + for k, v := range b { + r[k] = v + } + return r +} + +func (api *API) validate(f apiFunc) error { + state := api.Cluster.State() + if _, ok := functionStates[state][f]; ok { + return nil + } + return ApiFunctionNotAllowedError{errors.Errorf("api function %s not allowed in state %s", f, state)} } // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { if err := api.validate(apiQuery); err != nil { - return QueryResponse{}, errors.Wrap(err, "validate api function: query") + return QueryResponse{}, errors.Wrap(err, "validate api function") } resp := QueryResponse{} @@ -148,7 +159,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { - return nil, errors.Wrap(err, "validate api function: create index") + return nil, errors.Wrap(err, "validate api function") } // Create index. @@ -173,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { if err := api.validate(apiIndex); err != nil { - return nil, errors.Wrap(err, "validate api function: index") + return nil, errors.Wrap(err, "validate api function") } index := api.Holder.Index(indexName) @@ -187,7 +198,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { if err := api.validate(apiDeleteIndex); err != nil { - return errors.Wrap(err, "validate api function: delete index") + return errors.Wrap(err, "validate api function") } // Delete index from the holder. @@ -211,7 +222,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { if err := api.validate(apiCreateFrame); err != nil { - return nil, errors.Wrap(err, "validate api function: create frame") + return nil, errors.Wrap(err, "validate api function") } // Find index. @@ -246,7 +257,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { if err := api.validate(apiDeleteFrame); err != nil { - return errors.Wrap(err, "validate api function: delete frame") + return errors.Wrap(err, "validate api function") } // Find index. @@ -278,7 +289,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { - return errors.Wrap(err, "validate api function: export csv") + return errors.Wrap(err, "validate api function") } // Validate that this handler owns the slice. @@ -315,7 +326,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // SliceNodes returns the node and all replicas which should contain a slice's data. func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { if err := api.validate(apiSliceNodes); err != nil { - return nil, errors.Wrap(err, "validate api function: slice nodes") + return nil, errors.Wrap(err, "validate api function") } return api.Cluster.SliceNodes(indexName, slice), nil @@ -326,7 +337,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validate api function: marshal fragment") + return nil, errors.Wrap(err, "validate api function") } // Retrieve fragment from holder. @@ -342,7 +353,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validate api function: unmarshal fragment") + return errors.Wrap(err, "validate api function") } // Retrieve frame. @@ -375,7 +386,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { if err := api.validate(apiFragmentBlockData); err != nil { - return nil, errors.Wrap(err, "validate api function: fragment block data") + return nil, errors.Wrap(err, "validate api function") } reqBytes, err := ioutil.ReadAll(body) @@ -408,7 +419,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { - return nil, errors.Wrap(err, "validate api function: fragment blocks") + return nil, errors.Wrap(err, "validate api function") } // Retrieve fragment from holder. @@ -426,7 +437,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validate api function: restore frame") + return errors.Wrap(err, "validate api function") } // Create a client for the remote cluster. @@ -504,7 +515,7 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { if err := api.validate(apiCreateInputDefinition); err != nil { - return errors.Wrap(err, "validate api function: create input definition") + return errors.Wrap(err, "validate api function") } api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. @@ -542,7 +553,7 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { if err := api.validate(apiInputDefinition); err != nil { - return nil, errors.Wrap(err, "validate api function: input definition") + return nil, errors.Wrap(err, "validate api function") } api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) @@ -562,7 +573,7 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { if err := api.validate(apiDeleteInputDefinition); err != nil { - return errors.Wrap(err, "validate api function: delete input definition") + return errors.Wrap(err, "validate api function") } api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") @@ -591,7 +602,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { if err := api.validate(apiWriteInput); err != nil { - return errors.Wrap(err, "validate api function: write input") + return errors.Wrap(err, "validate api function") } api.Logger.Printf("WriteInput is deprecated and will be removed.") @@ -619,7 +630,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { - return errors.Wrap(err, "validate api function: recalculate caches") + return errors.Wrap(err, "validate api function") } err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) @@ -634,7 +645,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { if err := api.validate(apiClusterMessage); err != nil { - return errors.Wrap(err, "validate api function: cluster message") + return errors.Wrap(err, "validate api function") } // Read entire body. @@ -670,7 +681,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validate api function: create field") + return errors.Wrap(err, "validate api function") } // Retrieve frame by name. @@ -700,7 +711,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validate api function: delete field") + return errors.Wrap(err, "validate api function") } // Retrieve frame by name. @@ -730,7 +741,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validate api function: fields") + return nil, errors.Wrap(err, "validate api function") } index := api.Holder.index(indexName) @@ -749,7 +760,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { - return nil, errors.Wrap(err, "validate api function: views") + return nil, errors.Wrap(err, "validate api function") } // Retrieve views. @@ -766,7 +777,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { - return errors.Wrap(err, "validate api function: delete view") + return errors.Wrap(err, "validate api function") } // Retrieve frame. @@ -800,7 +811,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function: index attr diff") + return nil, errors.Wrap(err, "validate api function") } // Retrieve index from holder. @@ -834,7 +845,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiFrameAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function: frame attr diff") + return nil, errors.Wrap(err, "validate api function") } // Retrieve index from holder. @@ -869,7 +880,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { - return errors.Wrap(err, "validate api function: import") + return errors.Wrap(err, "validate api function") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -898,7 +909,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { - return errors.Wrap(err, "validate api function: import value") + return errors.Wrap(err, "validate api function") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -917,7 +928,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function: modify index time quantum") + return errors.Wrap(err, "validate api function") } // Retrieve index by name. @@ -934,7 +945,7 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function: modify frame time quantum") + return errors.Wrap(err, "validate api function") } // Retrieve index by name. @@ -1072,7 +1083,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validate api function: set coordinator") + return nil, nil, errors.Wrap(err, "validate api function") } oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) @@ -1102,7 +1113,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validate api function: remove node") + return nil, errors.Wrap(err, "validate api function") } removeNode := api.Cluster.nodeByID(id) @@ -1121,7 +1132,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validate api function: resize abort") + return errors.Wrap(err, "validate api function") } if !api.Cluster.IsCoordinator() { @@ -1143,9 +1154,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } +type apiFunc int + // API validation constants. const ( - apiClusterMessage int = iota + apiClusterMessage apiFunc = iota apiCreateField apiCreateFrame apiCreateIndex @@ -1189,44 +1202,44 @@ const ( apiWriteInput ) -var functionCommon = []int{ - apiClusterMessage, - apiMarshalFragment, - apiSetCoordinator, +var functionCommon = map[apiFunc]struct{}{ + apiClusterMessage: struct{}{}, + apiMarshalFragment: struct{}{}, + apiSetCoordinator: struct{}{}, } -var functionResizing = []int{ - apiResizeAbort, +var functionResizing = map[apiFunc]struct{}{ + apiResizeAbort: struct{}{}, } -var functionNormal = []int{ - apiCreateField, - apiCreateFrame, - apiCreateIndex, - apiCreateInputDefinition, - apiDeleteField, - apiDeleteFrame, - apiDeleteIndex, - apiDeleteInputDefinition, - apiDeleteView, - apiExportCSV, - apiFields, - apiFragmentBlockData, - apiFragmentBlocks, - apiFrameAttrDiff, - apiImport, - apiImportValue, - apiIndex, - apiIndexAttrDiff, - apiInputDefinition, - apiModifyFrameTimeQuantum, - apiModifyIndexTimeQuantum, - apiQuery, - apiRecalculateCaches, - apiRemoveNode, - apiRestoreFrame, - apiSliceNodes, - apiUnmarshalFragment, - apiViews, - apiWriteInput, +var functionNormal = map[apiFunc]struct{}{ + apiCreateField: struct{}{}, + apiCreateFrame: struct{}{}, + apiCreateIndex: struct{}{}, + apiCreateInputDefinition: struct{}{}, + apiDeleteField: struct{}{}, + apiDeleteFrame: struct{}{}, + apiDeleteIndex: struct{}{}, + apiDeleteInputDefinition: struct{}{}, + apiDeleteView: struct{}{}, + apiExportCSV: struct{}{}, + apiFields: struct{}{}, + apiFragmentBlockData: struct{}{}, + apiFragmentBlocks: struct{}{}, + apiFrameAttrDiff: struct{}{}, + apiImport: struct{}{}, + apiImportValue: struct{}{}, + apiIndex: struct{}{}, + apiIndexAttrDiff: struct{}{}, + apiInputDefinition: struct{}{}, + apiModifyFrameTimeQuantum: struct{}{}, + apiModifyIndexTimeQuantum: struct{}{}, + apiQuery: struct{}{}, + apiRecalculateCaches: struct{}{}, + apiRemoveNode: struct{}{}, + apiRestoreFrame: struct{}{}, + apiSliceNodes: struct{}{}, + apiUnmarshalFragment: struct{}{}, + apiViews: struct{}{}, + apiWriteInput: struct{}{}, } diff --git a/apifunc_string.go b/apifunc_string.go new file mode 100644 index 000000000..8fb9b16e0 --- /dev/null +++ b/apifunc_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiFunc"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiFunc_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiFunc_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} + +func (i apiFunc) String() string { + if i < 0 || i >= apiFunc(len(_apiFunc_index)-1) { + return fmt.Sprintf("apiFunc(%d)", i) + } + return _apiFunc_name[_apiFunc_index[i]:_apiFunc_index[i+1]] +} diff --git a/pilosa.go b/pilosa.go index ffb836ae8..c3a6919db 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,6 +82,12 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) +// InvalidApiFunctionError wraps an error value indicating that a particular +// API function is not allowed in the current cluster state. +type ApiFunctionNotAllowedError struct { + error +} + // BadRequestError wraps an error value to signify that a request could not be // read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest // would be returned. From e60d11a23e2d93ba5d594dac15f3d40a3d40eb0e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Apr 2018 17:07:41 -0500 Subject: [PATCH 244/311] change references from "function" to "method" --- api.go | 98 ++++++++++++++++++++++----------------------- apifunc_string.go | 16 -------- apimethod_string.go | 16 ++++++++ pilosa.go | 6 +-- 4 files changed, 68 insertions(+), 68 deletions(-) delete mode 100644 apifunc_string.go create mode 100644 apimethod_string.go diff --git a/api.go b/api.go index 6852a1f87..ed41328bf 100644 --- a/api.go +++ b/api.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate stringer -type=apiFunc +//go:generate stringer -type=apiMethod package pilosa @@ -61,16 +61,16 @@ func NewAPI() *API { } } -// functionStates specifies the api functions that are valid for each +// validAPIMethods specifies the api methods that are valid for each // cluster state. -var functionStates = map[string]map[apiFunc]struct{}{ - ClusterStateStarting: functionCommon, - ClusterStateNormal: appendMap(functionCommon, functionNormal), - ClusterStateResizing: appendMap(functionCommon, functionResizing), +var validAPIMethods = map[string]map[apiMethod]struct{}{ + ClusterStateStarting: methodsCommon, + ClusterStateNormal: appendMap(methodsCommon, methodsNormal), + ClusterStateResizing: appendMap(methodsCommon, methodsResizing), } -func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { - r := make(map[apiFunc]struct{}) +func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { + r := make(map[apiMethod]struct{}) for k, v := range a { r[k] = v } @@ -80,18 +80,18 @@ func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { return r } -func (api *API) validate(f apiFunc) error { +func (api *API) validate(f apiMethod) error { state := api.Cluster.State() - if _, ok := functionStates[state][f]; ok { + if _, ok := validAPIMethods[state][f]; ok { return nil } - return ApiFunctionNotAllowedError{errors.Errorf("api function %s not allowed in state %s", f, state)} + return ApiMethodNotAllowedError{errors.Errorf("api method %s not allowed in state %s", f, state)} } // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { if err := api.validate(apiQuery); err != nil { - return QueryResponse{}, errors.Wrap(err, "validate api function") + return QueryResponse{}, errors.Wrap(err, "validate api method") } resp := QueryResponse{} @@ -159,7 +159,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Create index. @@ -184,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { if err := api.validate(apiIndex); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } index := api.Holder.Index(indexName) @@ -198,7 +198,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { if err := api.validate(apiDeleteIndex); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Delete index from the holder. @@ -222,7 +222,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { if err := api.validate(apiCreateFrame); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Find index. @@ -257,7 +257,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { if err := api.validate(apiDeleteFrame); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Find index. @@ -289,7 +289,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Validate that this handler owns the slice. @@ -326,7 +326,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // SliceNodes returns the node and all replicas which should contain a slice's data. func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { if err := api.validate(apiSliceNodes); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } return api.Cluster.SliceNodes(indexName, slice), nil @@ -337,7 +337,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve fragment from holder. @@ -353,7 +353,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame. @@ -386,7 +386,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { if err := api.validate(apiFragmentBlockData); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } reqBytes, err := ioutil.ReadAll(body) @@ -419,7 +419,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve fragment from holder. @@ -437,7 +437,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Create a client for the remote cluster. @@ -515,7 +515,7 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { if err := api.validate(apiCreateInputDefinition); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. @@ -553,7 +553,7 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { if err := api.validate(apiInputDefinition); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) @@ -573,7 +573,7 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { if err := api.validate(apiDeleteInputDefinition); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") @@ -602,7 +602,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { if err := api.validate(apiWriteInput); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf("WriteInput is deprecated and will be removed.") @@ -630,7 +630,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) @@ -645,7 +645,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { if err := api.validate(apiClusterMessage); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Read entire body. @@ -681,7 +681,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame by name. @@ -711,7 +711,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame by name. @@ -741,7 +741,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } index := api.Holder.index(indexName) @@ -760,7 +760,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve views. @@ -777,7 +777,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame. @@ -811,7 +811,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve index from holder. @@ -845,7 +845,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiFrameAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve index from holder. @@ -880,7 +880,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -909,7 +909,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -928,7 +928,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve index by name. @@ -945,7 +945,7 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve index by name. @@ -1083,7 +1083,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validate api function") + return nil, nil, errors.Wrap(err, "validate api method") } oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) @@ -1113,7 +1113,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } removeNode := api.Cluster.nodeByID(id) @@ -1132,7 +1132,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } if !api.Cluster.IsCoordinator() { @@ -1154,11 +1154,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } -type apiFunc int +type apiMethod int // API validation constants. const ( - apiClusterMessage apiFunc = iota + apiClusterMessage apiMethod = iota apiCreateField apiCreateFrame apiCreateIndex @@ -1202,17 +1202,17 @@ const ( apiWriteInput ) -var functionCommon = map[apiFunc]struct{}{ +var methodsCommon = map[apiMethod]struct{}{ apiClusterMessage: struct{}{}, apiMarshalFragment: struct{}{}, apiSetCoordinator: struct{}{}, } -var functionResizing = map[apiFunc]struct{}{ +var methodsResizing = map[apiMethod]struct{}{ apiResizeAbort: struct{}{}, } -var functionNormal = map[apiFunc]struct{}{ +var methodsNormal = map[apiMethod]struct{}{ apiCreateField: struct{}{}, apiCreateFrame: struct{}{}, apiCreateIndex: struct{}{}, diff --git a/apifunc_string.go b/apifunc_string.go deleted file mode 100644 index 8fb9b16e0..000000000 --- a/apifunc_string.go +++ /dev/null @@ -1,16 +0,0 @@ -// Code generated by "stringer -type=apiFunc"; DO NOT EDIT. - -package pilosa - -import "fmt" - -const _apiFunc_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" - -var _apiFunc_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} - -func (i apiFunc) String() string { - if i < 0 || i >= apiFunc(len(_apiFunc_index)-1) { - return fmt.Sprintf("apiFunc(%d)", i) - } - return _apiFunc_name[_apiFunc_index[i]:_apiFunc_index[i+1]] -} diff --git a/apimethod_string.go b/apimethod_string.go new file mode 100644 index 000000000..8a3dce195 --- /dev/null +++ b/apimethod_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiMethod"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} + +func (i apiMethod) String() string { + if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { + return fmt.Sprintf("apiMethod(%d)", i) + } + return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] +} diff --git a/pilosa.go b/pilosa.go index c3a6919db..347fbed67 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,9 +82,9 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) -// InvalidApiFunctionError wraps an error value indicating that a particular -// API function is not allowed in the current cluster state. -type ApiFunctionNotAllowedError struct { +// ApiMethodNotAllowedError wraps an error value indicating that a particular +// API method is not allowed in the current cluster state. +type ApiMethodNotAllowedError struct { error } From e7151eb2a8a029ddf849b045c6d0070c3159c6b1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 18 Apr 2018 08:17:26 -0500 Subject: [PATCH 245/311] Remove Index.TimeQuantum --- api.go | 18 ---- apimethod_string.go | 4 +- cmd/import.go | 1 - handler.go | 40 -------- handler_test.go | 20 ---- holder.go | 1 - holder_test.go | 16 ---- index.go | 63 +++--------- index_test.go | 50 +--------- internal/private.pb.go | 213 ++++++++++++++++------------------------- internal/private.proto | 1 - server.go | 4 +- 12 files changed, 98 insertions(+), 333 deletions(-) diff --git a/api.go b/api.go index ed41328bf..a5953217d 100644 --- a/api.go +++ b/api.go @@ -925,22 +925,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return err } -// ModifyIndexTimeQuantum changes the default time quantum on the given index. -func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { - if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api method") - } - - // Retrieve index by name. - index := api.Holder.Index(indexName) - if index == nil { - return ErrIndexNotFound - } - - // Set default time quantum on index. - return index.SetTimeQuantum(timeQuantum) -} - // ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { @@ -1185,7 +1169,6 @@ const ( //apiMaxInverseSlices // not implemented //apiMaxSlices // not implemented apiModifyFrameTimeQuantum - apiModifyIndexTimeQuantum apiQuery apiRecalculateCaches apiRemoveNode @@ -1233,7 +1216,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIndexAttrDiff: struct{}{}, apiInputDefinition: struct{}{}, apiModifyFrameTimeQuantum: struct{}{}, - apiModifyIndexTimeQuantum: struct{}{}, apiQuery: struct{}{}, apiRecalculateCaches: struct{}{}, apiRemoveNode: struct{}{}, diff --git a/apimethod_string.go b/apimethod_string.go index 8a3dce195..8f77b69e7 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 352, 372, 385, 399, 414, 431, 444, 464, 472, 485} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/cmd/import.go b/cmd/import.go index b6b78bdbc..93fa92ee0 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -60,7 +60,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") diff --git a/handler.go b/handler.go index aa630e52c..bd97c1d1a 100644 --- a/handler.go +++ b/handler.go @@ -168,7 +168,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") // TODO: Apply MethodNotAllowed statuses to all endpoints. @@ -457,45 +456,6 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { } } -// handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. -func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req patchIndexTimeQuantumRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate quantum. - tq, err := ParseTimeQuantum(req.TimeQuantum) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err = h.API.ModifyIndexTimeQuantum(r.Context(), indexName, tq); err != nil { - if err == ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type patchIndexTimeQuantumRequest struct { - TimeQuantum string `json:"timeQuantum"` -} - -type patchIndexTimeQuantumResponse struct{} - // handlePostIndexAttrDiff handles POST /index/attr/diff requests. func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] diff --git a/handler_test.go b/handler_test.go index c9044aee0..8c719feb1 100644 --- a/handler_test.go +++ b/handler_test.go @@ -748,26 +748,6 @@ func TestHandler_DeleteFrame(t *testing.T) { } } -// Ensure handler can set the Index time quantum. -func TestHandler_SetIndexTimeQuantum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - - h := test.NewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected time quantum: %s", q) - } -} - // Ensure handler can set the frame time quantum. func TestHandler_SetFrameTimeQuantum(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/holder.go b/holder.go index 570433ce4..fa93767a3 100644 --- a/holder.go +++ b/holder.go @@ -359,7 +359,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } // Update options. - index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index diff --git a/holder_test.go b/holder_test.go index 24594025d..8b1f6166a 100644 --- a/holder_test.go +++ b/holder_test.go @@ -73,22 +73,6 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { - h := test.MustOpenHolder() - defer h.Close() - - if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { - t.Fatal(err) - } - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") { - t.Fatalf("unexpected error: %s", err) - } - }) t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() diff --git a/index.go b/index.go index 039816322..4676e1b43 100644 --- a/index.go +++ b/index.go @@ -39,10 +39,6 @@ type Index struct { path string name string - // Default time quantum for all frames in index. - // This can be overridden by individual frames. - timeQuantum TimeQuantum - // Frames by name. frames map[string]*Frame @@ -106,9 +102,7 @@ func (i *Index) Options() IndexOptions { } func (i *Index) options() IndexOptions { - return IndexOptions{ - TimeQuantum: i.timeQuantum, - } + return IndexOptions{} } // Open opens and initializes the index. @@ -175,7 +169,6 @@ func (i *Index) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta")) if os.IsNotExist(err) { - i.timeQuantum = "" return nil } else if err != nil { return err @@ -186,17 +179,18 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. - i.timeQuantum = TimeQuantum(pb.TimeQuantum) return nil } +// NOTE: Until we introduce new attributes to store in the index .meta file, +// we don't need to actually write the file. The code related to index.options +// and the index meta file are left in place for future use. +/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{ - TimeQuantum: string(i.timeQuantum), - }) + buf, err := proto.Marshal(&internal.IndexMeta{}) if err != nil { return err } @@ -208,6 +202,7 @@ func (i *Index) saveMeta() error { return nil } +*/ // Close closes the index and its frames. func (i *Index) Close() error { @@ -278,34 +273,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) { i.remoteMaxInverseSlice = v } -// TimeQuantum returns the default time quantum for the index. -func (i *Index) TimeQuantum() TimeQuantum { - i.mu.RLock() - defer i.mu.RUnlock() - return i.timeQuantum -} - -// SetTimeQuantum sets the default time quantum for the index. -func (i *Index) SetTimeQuantum(q TimeQuantum) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Validate input. - if !q.Valid() { - return ErrInvalidTimeQuantum - } - - // Update value on index. - i.timeQuantum = q - - // Perist meta data to disk. - if err := i.saveMeta(); err != nil { - return err - } - - return nil -} - // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } @@ -425,12 +392,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, err } - // Default the time quantum to what is set on the Index. - timeQuantum := i.timeQuantum - if opt.TimeQuantum != "" { - timeQuantum = opt.TimeQuantum - } - if err := f.SetTimeQuantum(timeQuantum); err != nil { + // Set the time quantum. + if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { f.Close() return nil, err } @@ -577,15 +540,11 @@ func encodeIndex(d *Index) *internal.Index { } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct { - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` -} +type IndexOptions struct{} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{ - TimeQuantum: string(i.TimeQuantum), - } + return &internal.IndexMeta{} } // hasTime returns true if a contains a non-nil time. diff --git a/index_test.go b/index_test.go index 25e093d4b..b0a5b3506 100644 --- a/index_test.go +++ b/index_test.go @@ -58,11 +58,6 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - // Create frame with explicit quantum. f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) if err != nil { @@ -71,24 +66,6 @@ func TestIndex_CreateFrame(t *testing.T) { t.Fatalf("unexpected frame time quantum: %s", q) } }) - - t.Run("Inherited", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - - // Create frame. - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { - t.Fatalf("unexpected frame time quantum: %s", q) - } - }) }) // Ensure frame can include range columns. @@ -267,26 +244,6 @@ func TestIndex_DeleteFrame(t *testing.T) { } } -// Ensure index can set the default time quantum. -func TestIndex_SetTimeQuantum(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - // Set & retrieve time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload index and verify that it is persisted. - if err := index.Reopen(); err != nil { - t.Fatal(err) - } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - // Ensure index can delete a frame. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") @@ -404,18 +361,13 @@ func TestIndex_InputBits(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - err := index.InputBits("f", bits) if !strings.Contains(err.Error(), "Frame not found") { t.Fatalf("Expected Frame not found error, actual error: %s", err) } // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil { t.Fatal(err) } diff --git a/internal/private.pb.go b/internal/private.pb.go index 0d0f3124a..2e4d39239 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -68,7 +68,6 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -76,13 +75,6 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -func (m *IndexMeta) GetTimeQuantum() string { - if m != nil { - return m.TimeQuantum - } - return "" -} - type FrameMeta struct { InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` @@ -1232,12 +1224,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } return i, nil } @@ -2746,10 +2732,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l - l = len(m.TimeQuantum) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } return n } @@ -3439,35 +3421,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8573,87 +8526,87 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1308 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15, - 0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3, - 0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03, - 0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08, - 0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66, - 0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a, - 0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5, - 0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e, - 0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b, - 0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65, - 0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff, - 0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1, - 0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22, - 0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66, - 0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d, - 0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd, - 0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13, - 0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8, - 0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73, - 0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2, - 0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14, - 0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8, - 0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a, - 0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13, - 0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8, - 0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63, - 0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee, - 0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76, - 0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25, - 0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78, - 0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7, - 0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70, - 0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a, - 0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66, - 0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b, - 0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c, - 0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29, - 0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99, - 0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6, - 0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5, - 0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde, - 0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c, - 0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e, - 0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6, - 0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa, - 0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66, - 0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed, - 0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78, - 0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab, - 0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29, - 0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0, - 0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4, - 0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e, - 0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3, - 0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a, - 0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9, - 0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80, - 0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3, - 0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80, - 0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9, - 0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4, - 0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f, - 0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69, - 0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e, - 0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a, - 0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2, - 0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0, - 0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7, - 0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34, - 0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a, - 0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10, - 0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6, - 0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e, - 0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e, - 0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16, - 0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5, - 0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0, - 0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b, - 0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d, - 0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06, - 0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00, + // 1307 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xc5, + 0x1b, 0xff, 0xaf, 0xd7, 0x76, 0xe2, 0xc7, 0x71, 0xe2, 0x4c, 0xd3, 0xfc, 0x37, 0x55, 0xe4, 0x9a, + 0x51, 0xa1, 0xa6, 0x12, 0x51, 0x49, 0x25, 0x44, 0x03, 0x95, 0x4a, 0x62, 0x57, 0x5d, 0x20, 0x51, + 0x19, 0x27, 0x41, 0x42, 0x02, 0x69, 0x62, 0x0f, 0xe9, 0x2a, 0xeb, 0x5d, 0xb3, 0x3b, 0x4e, 0xe2, + 0x1e, 0x38, 0x22, 0x24, 0xc4, 0x1d, 0x71, 0xe5, 0xcb, 0x70, 0xe4, 0x13, 0x20, 0x14, 0x3e, 0x04, + 0x47, 0xd0, 0xbc, 0xed, 0xae, 0xdf, 0x92, 0x26, 0x70, 0xdb, 0xe7, 0xfd, 0x37, 0xcf, 0xdb, 0xcc, + 0x42, 0xa5, 0x1f, 0x79, 0xa7, 0x94, 0xb3, 0x8d, 0x7e, 0x14, 0xf2, 0x10, 0xcd, 0x7b, 0x01, 0x67, + 0x51, 0x40, 0x7d, 0x5c, 0x86, 0x92, 0x1b, 0x74, 0xd9, 0xf9, 0x2e, 0xe3, 0x14, 0xff, 0x6e, 0x41, + 0xe9, 0x59, 0x44, 0x7b, 0x4c, 0x50, 0xe8, 0x2d, 0x58, 0x74, 0x83, 0x53, 0x16, 0xc5, 0xac, 0x15, + 0xd0, 0x23, 0x9f, 0x75, 0x9d, 0x5c, 0xdd, 0x6a, 0xcc, 0x93, 0x31, 0x2e, 0x5a, 0x87, 0xd2, 0x0e, + 0xed, 0xbc, 0x64, 0xfb, 0xc3, 0x3e, 0x73, 0xec, 0xba, 0xd5, 0x28, 0x91, 0x94, 0x91, 0x48, 0xdb, + 0xde, 0x2b, 0xe6, 0xe4, 0xeb, 0x56, 0xa3, 0x42, 0x52, 0x06, 0xaa, 0x43, 0x79, 0xdf, 0xeb, 0xb1, + 0xcf, 0x06, 0x34, 0xe0, 0x83, 0x9e, 0x53, 0x90, 0xd6, 0x59, 0x16, 0xc2, 0xb0, 0x40, 0x68, 0x70, + 0x9c, 0x60, 0x28, 0x4a, 0x0c, 0x23, 0x3c, 0x74, 0x1f, 0x8a, 0xcf, 0x3c, 0xe6, 0x77, 0x63, 0x67, + 0xae, 0x6e, 0x37, 0xca, 0x9b, 0x4b, 0x1b, 0xe6, 0x7c, 0x1b, 0x92, 0x4f, 0xb4, 0x18, 0x63, 0x58, + 0x74, 0x7b, 0xfd, 0x30, 0xe2, 0x84, 0xc5, 0xfd, 0x30, 0x88, 0x19, 0xaa, 0x82, 0xdd, 0x8a, 0x22, + 0xc7, 0x92, 0x81, 0xc5, 0x27, 0xfe, 0x16, 0xaa, 0xdb, 0x7e, 0xd8, 0x39, 0x69, 0x52, 0x4e, 0x09, + 0xfb, 0x66, 0xc0, 0x62, 0x8e, 0x56, 0xa0, 0x20, 0xb3, 0xa4, 0xf5, 0x14, 0x21, 0xb8, 0x32, 0x5b, + 0x32, 0x2f, 0x25, 0xa2, 0x08, 0xc1, 0x95, 0xf6, 0x32, 0x15, 0x79, 0xa2, 0x08, 0xc1, 0x6d, 0xfb, + 0x5e, 0x47, 0xa5, 0x20, 0x4f, 0x14, 0x81, 0x10, 0xe4, 0x0f, 0x3d, 0x76, 0xa6, 0xcf, 0x2d, 0xbf, + 0xb1, 0x0b, 0xcb, 0x99, 0xf8, 0x1a, 0xe6, 0x2a, 0x14, 0x49, 0x78, 0xe6, 0x36, 0x63, 0xc7, 0xaa, + 0xdb, 0x8d, 0x3c, 0xd1, 0x94, 0xcc, 0x6e, 0xe8, 0x0f, 0x7a, 0x81, 0x10, 0xe5, 0xa4, 0x28, 0x65, + 0xe0, 0x35, 0x28, 0xc8, 0x54, 0x8b, 0x53, 0xa6, 0xb6, 0xe2, 0x13, 0xff, 0x6d, 0x41, 0x69, 0x97, + 0x9e, 0x4b, 0x18, 0x31, 0x7a, 0x02, 0xf3, 0x6d, 0x4e, 0x83, 0x2e, 0x8d, 0xba, 0x52, 0xa9, 0xbc, + 0xf9, 0x46, 0x9a, 0xc2, 0x44, 0x6d, 0xc3, 0xe8, 0xb4, 0x02, 0x1e, 0x0d, 0x49, 0x62, 0x82, 0xb6, + 0x60, 0x4e, 0xf7, 0x84, 0xc4, 0x50, 0xde, 0xac, 0x4f, 0xb3, 0x4e, 0xda, 0x46, 0x18, 0x1b, 0x83, + 0x3b, 0x1f, 0x40, 0x65, 0xc4, 0xad, 0xc0, 0x7a, 0xc2, 0x86, 0xa6, 0x22, 0x27, 0x6c, 0x28, 0x72, + 0x77, 0x4a, 0xfd, 0x81, 0xca, 0x73, 0x9e, 0x28, 0x62, 0x2b, 0xf7, 0xbe, 0x75, 0x67, 0x0b, 0x16, + 0xb2, 0x5e, 0xaf, 0x63, 0x8b, 0xbf, 0x02, 0xb4, 0x13, 0x31, 0xca, 0x99, 0x84, 0xb7, 0xcb, 0xe2, + 0x98, 0x1e, 0xb3, 0xd9, 0x95, 0x56, 0xd5, 0xcb, 0x65, 0xab, 0xb7, 0x0e, 0x25, 0x37, 0x36, 0x07, + 0xb7, 0x65, 0x5f, 0xa6, 0x0c, 0xfc, 0x00, 0x50, 0x93, 0xf9, 0x8c, 0x33, 0x3d, 0x5f, 0x97, 0xf8, + 0xc7, 0x6d, 0x83, 0xe5, 0x6a, 0x5d, 0x74, 0x1f, 0xf2, 0x62, 0x3c, 0x25, 0x94, 0xf2, 0xe6, 0xad, + 0x34, 0xd3, 0xc9, 0x1c, 0x13, 0xa9, 0x80, 0x3d, 0xe3, 0x54, 0x8f, 0xf4, 0x15, 0x07, 0x9c, 0xd2, + 0xca, 0x26, 0x94, 0x3d, 0x1e, 0x2a, 0x59, 0x12, 0x3a, 0xd4, 0x53, 0x73, 0xd6, 0x9b, 0x86, 0xc2, + 0xc7, 0x09, 0x58, 0x31, 0xa9, 0x37, 0x01, 0xfb, 0x26, 0x14, 0xa4, 0xad, 0x46, 0x3b, 0xb1, 0x03, + 0x94, 0x14, 0x1f, 0x26, 0x50, 0x6f, 0x1a, 0x68, 0x25, 0x1b, 0xa8, 0x64, 0xfc, 0x7e, 0xa1, 0x75, + 0xc5, 0x4c, 0xef, 0x09, 0x1b, 0xe5, 0x49, 0x7e, 0xcf, 0xae, 0xd9, 0x58, 0x22, 0x85, 0x6f, 0xb1, + 0x04, 0x62, 0xc7, 0xae, 0xdb, 0xc2, 0xb7, 0x24, 0xf0, 0x23, 0x28, 0xb6, 0x3b, 0x2f, 0x59, 0x8f, + 0xa2, 0xb7, 0xc5, 0xa4, 0x75, 0xd9, 0x39, 0x8b, 0xf5, 0x9c, 0x2e, 0x8d, 0xd5, 0x9f, 0x18, 0x39, + 0xfe, 0xc1, 0xd2, 0x67, 0x9a, 0x81, 0xa8, 0x28, 0x63, 0xc7, 0x4e, 0x7e, 0x62, 0x65, 0x0a, 0x3e, + 0xd1, 0x62, 0xd4, 0x82, 0xaa, 0x1b, 0xf4, 0x07, 0xbc, 0xc9, 0xbe, 0xf6, 0x02, 0x8f, 0x7b, 0x61, + 0x10, 0x3b, 0x45, 0x69, 0xb2, 0x96, 0x0d, 0x3d, 0xa2, 0x41, 0x26, 0x4c, 0xf0, 0x77, 0x16, 0x2c, + 0x8d, 0x31, 0xaf, 0xc0, 0x95, 0xbb, 0x1c, 0xd7, 0x7b, 0xc9, 0xce, 0xb7, 0xa5, 0x62, 0x6d, 0x26, + 0x9a, 0xd1, 0x2b, 0xe0, 0x17, 0x0b, 0x56, 0xa6, 0x29, 0x4c, 0x45, 0x53, 0x03, 0x78, 0x11, 0x79, + 0x3d, 0x1a, 0x0d, 0x3f, 0x61, 0x43, 0x7d, 0xfd, 0x65, 0x38, 0xe8, 0x73, 0x58, 0x1d, 0xf3, 0xf5, + 0x51, 0x47, 0xa5, 0x48, 0x81, 0xba, 0x3b, 0x13, 0x94, 0xd2, 0x23, 0x33, 0xcc, 0xf1, 0x5f, 0x16, + 0xdc, 0x9e, 0x2a, 0x4a, 0x7b, 0xd2, 0xca, 0xf6, 0xe4, 0x03, 0xa8, 0x1e, 0x8a, 0xcd, 0xd6, 0x64, + 0x31, 0xf7, 0x02, 0x2a, 0x34, 0x75, 0xd3, 0x4e, 0xf0, 0x91, 0x0b, 0xf3, 0x92, 0xb7, 0x4b, 0xfb, + 0x1a, 0xe6, 0x3b, 0x57, 0xc0, 0xdc, 0x30, 0xfa, 0x7a, 0xf1, 0x1b, 0x52, 0x80, 0x91, 0x17, 0x91, + 0xb9, 0xd5, 0x24, 0x21, 0x56, 0xfa, 0x88, 0xc1, 0xb5, 0xd6, 0x72, 0x08, 0xeb, 0x66, 0x15, 0x8e, + 0x20, 0xb9, 0x7c, 0x52, 0x1f, 0x03, 0xa4, 0xaa, 0x7a, 0x03, 0x5c, 0xd2, 0x9f, 0x19, 0x65, 0xfc, + 0x1c, 0xd6, 0xcd, 0x9e, 0xbe, 0x46, 0x40, 0xd3, 0x2d, 0xb9, 0xb4, 0x5b, 0x70, 0x0b, 0xec, 0x03, + 0xe2, 0x8a, 0xbb, 0x5a, 0x4e, 0xab, 0x29, 0x91, 0xa6, 0x84, 0xc9, 0xf3, 0x30, 0xe6, 0xc6, 0x44, + 0x7c, 0x0b, 0xde, 0x8b, 0x30, 0xe2, 0x12, 0x71, 0x85, 0xc8, 0x6f, 0xfc, 0x25, 0xe4, 0xf7, 0xc2, + 0x2e, 0x43, 0x8b, 0x90, 0x73, 0x9b, 0xda, 0x47, 0xce, 0x6d, 0xa2, 0xbb, 0xd2, 0xbd, 0xde, 0x21, + 0x95, 0xf4, 0x70, 0x07, 0xc4, 0x25, 0x32, 0xf0, 0x3d, 0xa8, 0xb8, 0xf1, 0x4e, 0x18, 0x46, 0x5d, + 0x51, 0xea, 0x30, 0xd2, 0x77, 0xd2, 0x28, 0x13, 0x3f, 0x85, 0xaa, 0x70, 0xdf, 0xe6, 0x94, 0x27, + 0x9b, 0x7a, 0x15, 0x8a, 0x82, 0x97, 0x84, 0xd3, 0x94, 0xbc, 0xf7, 0x84, 0x9e, 0x59, 0x80, 0x92, + 0xc0, 0x9f, 0x2a, 0x0f, 0xad, 0x53, 0x16, 0xf0, 0x4c, 0x96, 0x24, 0x2d, 0x1d, 0x54, 0x88, 0x22, + 0x10, 0x56, 0x47, 0xd1, 0x98, 0x17, 0x53, 0xcc, 0x82, 0x4b, 0xa4, 0x0c, 0xff, 0x68, 0x01, 0x18, + 0x40, 0x83, 0x38, 0x31, 0xb1, 0x66, 0x9b, 0xa0, 0x77, 0x33, 0x6f, 0x97, 0xc9, 0x9d, 0x9a, 0x88, + 0x48, 0xe6, 0x85, 0xd3, 0x30, 0x2b, 0x54, 0x37, 0x47, 0x35, 0xd5, 0x57, 0x7c, 0x5d, 0x26, 0x71, + 0x6d, 0x56, 0x76, 0xfc, 0x41, 0xcc, 0x59, 0xa4, 0x11, 0x89, 0x37, 0x96, 0x62, 0x24, 0xf9, 0x49, + 0x19, 0xd3, 0x53, 0x84, 0xee, 0x41, 0x41, 0x20, 0x35, 0x7b, 0x60, 0xfc, 0x18, 0x4a, 0x88, 0xdb, + 0xfa, 0x26, 0x99, 0xba, 0x7b, 0x10, 0xe4, 0xe5, 0x8b, 0x5a, 0xb7, 0x8b, 0x7c, 0x4c, 0x57, 0xc1, + 0xde, 0xf5, 0x54, 0x7f, 0xdb, 0x44, 0x7c, 0x4a, 0x0e, 0x3d, 0x97, 0xf3, 0x27, 0x38, 0x54, 0xbc, + 0x25, 0x96, 0xd5, 0x00, 0x89, 0xbb, 0xe3, 0x26, 0xf7, 0x9b, 0x79, 0x94, 0xda, 0x99, 0x47, 0x69, + 0x1b, 0x96, 0xd5, 0x90, 0xfc, 0x97, 0x4e, 0x7f, 0xce, 0xc1, 0x32, 0x61, 0xb1, 0xf7, 0x8a, 0xb9, + 0x41, 0xcc, 0xa3, 0x41, 0xb2, 0xe0, 0x3e, 0x0e, 0x8f, 0x74, 0xaa, 0x6d, 0xa2, 0x88, 0xd7, 0xe9, + 0x24, 0xf4, 0x10, 0xca, 0xe3, 0xdd, 0x3f, 0xa9, 0x9a, 0x55, 0x41, 0x0f, 0x61, 0xae, 0x1d, 0x0e, + 0xa2, 0x4e, 0x72, 0x0d, 0xae, 0xa6, 0xda, 0x0a, 0x99, 0x12, 0x13, 0xa3, 0x96, 0xe9, 0xa3, 0xc2, + 0xe5, 0x7d, 0x84, 0x9e, 0x8c, 0xf5, 0x91, 0xfc, 0x73, 0x29, 0x6f, 0xfe, 0x3f, 0x35, 0x18, 0x11, + 0x93, 0x51, 0x6d, 0xfc, 0xbd, 0x05, 0x0b, 0x59, 0x08, 0xaf, 0x35, 0x18, 0x49, 0x45, 0x72, 0x53, + 0x2b, 0x62, 0x4f, 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0xce, 0x2d, 0x64, 0xde, 0xb9, 0xf8, 0x04, + 0xd6, 0x26, 0xca, 0xb4, 0x13, 0xf6, 0xfa, 0xa2, 0x1f, 0xfe, 0x45, 0xb9, 0xc4, 0xca, 0x88, 0x22, + 0x5d, 0xa8, 0x12, 0x51, 0x04, 0x7e, 0x0c, 0xb7, 0xdb, 0x8c, 0x67, 0x8a, 0x64, 0xba, 0xad, 0x0e, + 0xf6, 0x1e, 0x3b, 0x9b, 0x71, 0x7c, 0x21, 0xc2, 0x1f, 0x82, 0x73, 0xd0, 0xef, 0x52, 0xce, 0x6e, + 0x64, 0xbd, 0x0d, 0xf3, 0xfb, 0x61, 0x3f, 0xf4, 0xc3, 0xe3, 0xe1, 0x15, 0x23, 0xef, 0xc0, 0x9c, + 0xda, 0x8f, 0xea, 0x91, 0x52, 0x22, 0x86, 0xc4, 0xb7, 0x44, 0x43, 0x77, 0xa8, 0xdf, 0x19, 0xf8, + 0x02, 0x86, 0xf8, 0xf7, 0x8a, 0xb7, 0xab, 0xbf, 0x5e, 0xd4, 0xac, 0xdf, 0x2e, 0x6a, 0xd6, 0x1f, + 0x17, 0x35, 0xeb, 0xa7, 0x3f, 0x6b, 0xff, 0x3b, 0x2a, 0xca, 0xbf, 0xf0, 0x47, 0xff, 0x04, 0x00, + 0x00, 0xff, 0xff, 0xc3, 0xb3, 0xdc, 0xe3, 0x96, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 7126811ac..bd3e52b26 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package internal; message IndexMeta { - string TimeQuantum = 2; } message FrameMeta { diff --git a/server.go b/server.go index 049672686..f28ca3864 100644 --- a/server.go +++ b/server.go @@ -367,9 +367,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { idx.SetRemoteMaxSlice(obj.Slice) } case *internal.CreateIndexMessage: - opt := IndexOptions{ - TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), - } + opt := IndexOptions{} _, err := s.Holder.CreateIndex(obj.Index, opt) if err != nil { return err From 8c2387b45ee52667c85aeb22be26d0d65be423a7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 18 Apr 2018 11:06:53 -0500 Subject: [PATCH 246/311] remove references to Input Defintion from the docs --- docs/api-reference.md | 123 --------------------------------------- docs/getting-started.md | 9 --- docs/input-definition.md | 11 ---- 3 files changed, 143 deletions(-) delete mode 100644 docs/input-definition.md diff --git a/docs/api-reference.md b/docs/api-reference.md index 0c0314f85..f7c3641b9 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -198,129 +198,6 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \ {} ``` -### Create input definition - -
-Input definition is deprecated as of v0.9. -
- -`POST /index//input-definition/` - -Creates an input definition in the given index with the given name. - -The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options: - -* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. -* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. -* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. - -The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa. - -* `name` (string): Maps the source data field to actions that process the field's corresponding value. -* `actions` (array): List of actions that will process the field's value. - -The `action` describes how the field value will be processed. Each `action` may contain: - -* `frame` (string): The Frame that will contain this action's set bits. -* `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame. -* `valueDestination` (string): The mapping rule used for this data. - - `value-to-row`: The value should be an integer and will map directly to a RowID. - - `single-row-boolean`: If the value is true set a bit using the `rowid`. - - `mapping`: Map the value to a RowID in the `valueMap`. -* `valueMap` (object): string and integer pairs used to map field values to RowID's. - -``` request -curl localhost:10101/index/user/input-definition/stargazer-input \ - -X POST \ - -d '{ - "frames":[ - { - "name": "language", - "options": {"inverseEnabled": true} - } - ], - "fields":[ - { - "name": "repo_id", - "primaryKey":true - }, - { - "name": "language_id", - "actions":[ - { - "frame": "language", - "valueDestination": "mapping", - "valueMap": { - "Go": 5, - "Python": 17, - "C++": 10 - } - } - ] - } - ] - }' -``` -``` response -{} -``` - -### Get input definition - -
-Input definition is deprecated as of v0.9. -
- -`GET /index//input-definition/` - -Returns the given input definition as JSON. - -``` request -curl -XGET localhost:10101/index/user/input-definition/stargazer-input -``` -``` response -{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} -``` - -### Remove input definition - -
-Input definition is deprecated as of v0.9. -
- -`DELETE /index//input-definition/` - -Removes the given input definition. - -``` request -curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input -``` -``` response -{} -``` - -### Process input data - -
-Input definition is deprecated as of v0.9. -
- -`POST /index//input/` - -Processes the JSON payload using the given input definition. - -The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition. - -``` request -curl localhost:10101/index/user/input/stargazer-input \ - -X POST \ - -d '[{"language_id": "Go", "repo_id": 92274475}]' -``` -``` response -{} -``` - ### List hosts `GET /hosts` diff --git a/docs/getting-started.md b/docs/getting-started.md index c9cdb8aba..bee165643 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -4,7 +4,6 @@ weight = 3 nav = [ "Starting Pilosa", "Sample Project", - "Input Definition", "What's Next?", ] +++ @@ -111,14 +110,6 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages. -### Input Definition - -
-Input definition is deprecated as of v0.9. -
- -Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. - #### Make Some Queries
diff --git a/docs/input-definition.md b/docs/input-definition.md deleted file mode 100644 index 9d0e5fe6b..000000000 --- a/docs/input-definition.md +++ /dev/null @@ -1,11 +0,0 @@ -+++ -title = "Input Definition" -weight = 8 -+++ - -## Input Definition -
-Input definition is deprecated as of Pilosa v0.9.
-
-The previous version of this page is still available here. -
From ff8fea804c17abf7f081fec94b9c4ad88a0b0bf6 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 18 Apr 2018 11:22:59 -0500 Subject: [PATCH 247/311] remove outdated pdk docs, and write some new ones incomplete, but an improvement? --- docs/pdk.md | 111 +++++++++++++++++++++++++++------------------------- 1 file changed, 58 insertions(+), 53 deletions(-) diff --git a/docs/pdk.md b/docs/pdk.md index 8c02dc279..a9df97832 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -2,66 +2,71 @@ title = "PDK" weight = 11 nav = [ + "Examples and Executables", "Library", - "Examples", ] +++ ## PDK -The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly. +The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively. + +### Examples and Executables +Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here. + +#### Kafka +`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the +Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each +record from Kafka is assigned a Pilosa column, and each value in a record is +assigned a row or field. Frame and field names are built from the "path" through +the record to arrive at that field. For example: + +```json +{ + "name": "jill", + "favorite_foods": ["corn chips", "chipotle dip"], + "location": { + "city": "Austin", + "state": "Texas", + "latitude": 3754, + "longitude": 4526 + }, + "active": true, + "age": 27 +} +``` + +This JSON object would result in the following Pilosa schema: + +| Name | Field | Type | Size/Min | Max | +|----------------|-----------|--------|----------|------------| +| name | | ranked | 100000 | | +| favorite_foods | | ranked | 100000 | | +| default | | Ranked | 100000 | | +| | age | int | 0 | 2147483647 | +| location | | ranked | 1000 | | +| | latitude | int | 0 | 2147483647 | +| | longitude | int | 0 | 2147483647 | +| location-city | | ranked | 100000 | | +| location-state | | ranked | 100000 | | + +All frames are created as ranked frames by default, and fields are created with +a minmum size of zero and a fixed maximum of 2147483647. Fields at the top level +are created in the default frame. Frames are a dash-separated concatenation of +all key values in the path - you can see this with frames like location-city. + + +Most the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts, +kafka topics, kafka group, etc.), but there are a few options that give some +control over the way data is indexed, and ingestion performance. + +* `--batch-size`: The batch size control how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. +* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state". +* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it. +* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used. +* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb. -The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary. ### Library -#### Mapping - -Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts: - -* an enumeration of field names -* a list of parsers that are used to parse strings in the CSV to values -* a list of commonly used, named, mapper functions -* a list of ParserMappers - objects that encapsulate all of the work related to a single frame. - -This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK. - -With this definition available, the PDK tool can run the import, which consists of these steps: - -- create the index -- create all frames -- for each CSV file, read all rows -- for each CSV record: - - generate a columnID - - apply all ParserMappers, generating a list of (frame, ID) pairs - - set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID) - -The process is summarized in this flowchart: - -![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg) - - -Some of the simple mapper functions available with PDK include: - -* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year. -* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11]. -* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6]. -* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23]. -* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins. -* BoolMapper: Maps a boolean value to the range [0, 1]. -* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain. -* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small. -* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets. -* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket. -* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example. -* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper. - -### Examples - -Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help ` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality. - - +For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation. From 72eb3d239ac264a69ab32c7a60e100a675327f18 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 18 Apr 2018 14:18:57 -0500 Subject: [PATCH 248/311] Remove references to inverse frames --- docs/administration.md | 1 - docs/api-reference.md | 5 +---- docs/data-model.md | 16 +--------------- docs/examples.md | 4 ++-- docs/glossary.md | 2 +- docs/query-language.md | 13 +------------ docs/webui.md | 4 ++-- 7 files changed, 8 insertions(+), 37 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 9d8c5b70c..c9d932df0 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -201,7 +201,6 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **NumCPU:** Number of Cores per Node - **BSIEnabled:** Bit Slice Index Frames in use. - **TimeQuantumEnabled:** Time Quantum Frames in use. -- **InverseEnabled:** Inverse Frames in use. - **NumIndexes:** Number of Indexes in the Cluster. - **NumFrames:** Number of Frames in the Cluster. - **NumSlices:** Number of Slices in the Cluster. diff --git a/docs/api-reference.md b/docs/api-reference.md index f7c3641b9..a2697f311 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -103,7 +103,6 @@ Creates a frame in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. @@ -119,9 +118,7 @@ Each individual `field` contains the following: Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. ``` request -curl localhost:10101/index/user/frame/language \ - -X POST \ - -d '{"options": {"inverseEnabled": true}}' +curl localhost:10101/index/user/frame/language -X POST ``` ``` response {} diff --git a/docs/data-model.md b/docs/data-model.md index 6a57a96a1..dc8646a85 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -77,26 +77,12 @@ Columns are sharded on a preset width, and each shard is referred to as a Slice. ### View -Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. +Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. #### Standard The standard View contains the same Row/Column format as the input data. -#### Inverse - -The Inverse View contains the same data with the Row and Column swapped. - -For example, the following `SetBit()` queries will result in the data described in the illustration below: -``` -SetBit(frame="A", rowID=8, columnID=3) -SetBit(frame="A", rowID=11, columnID=3) -SetBit(frame="A", rowID=19, columnID=5) -``` - -![inverse frame diagram](/img/docs/frame-inverse.svg) -*Inverse frame diagram* - #### Time Quantums If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: diff --git a/docs/examples.md b/docs/examples.md index 225e0f07c..f2d458b30 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -205,7 +205,7 @@ For more examples and details, see this [ipython notebook](https://github.com/pi The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. -We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. +We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. ``` @@ -218,7 +218,7 @@ All source code to calculate tanimoto for molecule fingerprint using Pilosa is a #### Data model -We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. +We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. diff --git a/docs/glossary.md b/docs/glossary.md index 612c65dd4..d222cb511 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -64,4 +64,4 @@ nav = [] [TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). -[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. diff --git a/docs/query-language.md b/docs/query-language.md index 6b0584320..f26cf021d 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -386,13 +386,12 @@ Return `2` ``` TopN([BITMAP_CALL], , [n=UINT], - [inverse=true], [, ]) + [, ]) ``` **Description:** Return the id and count of the top `n` bitmaps (by count of bits) in the frame. -`inverse=true` specifies that the call should operate on the [inverse view ](../data-model/#inverse). The `field` and `filters` arguments work together to only return Bitmaps which have the attribute specified by `field` with one of the values specified in `filters`. @@ -419,16 +418,6 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1} * count is amount of repositories * Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository. -``` -TopN(frame="stargazer", inverse=true) -``` - -Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]` - -* key is a repository ID -* count is amount of users -* Results are the number of users that starred each repository in descending order for all respositories in the stargazer frame. - ``` TopN(frame="stargazer", n=2) ``` diff --git a/docs/webui.md b/docs/webui.md index 738fb20fd..0f14934b4 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -32,9 +32,9 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). +Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). -- `:create frame inverseEnabled=true cacheSize=10000` +- `:create frame cacheSize=10000` ### Cluster Admin From 84e6493d7a94a6db7a6ffcea79455a9d44b08d3f Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 19 Apr 2018 10:31:01 -0500 Subject: [PATCH 249/311] Add deprecation warning to examples doc --- docs/examples.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/examples.md b/docs/examples.md index f2d458b30..d7830a2a8 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -201,6 +201,10 @@ For more examples and details, see this [ipython notebook](https://github.com/pi ### Chemical similarity search +
+This example uses the inverse frames feature, which is deprecated as of v0.9.0. This will soon be updated to reflect the current Pilosa API. +
+ #### Overview The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. From 876ed56e306f57c08b49688a3186d412bb2061c1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 19 Apr 2018 14:51:42 -0500 Subject: [PATCH 250/311] move pilosa.Config to pilosa/server.Config step 1 of #1203 The Config object is really just a specification of the options to pilosa server, so it makes sense to have it in that package. --- broadcast_test.go | 5 +- cluster.go | 5 +- cmd/server_test.go | 8 +- config.go | 226 ------------------------ ctl/backup.go | 5 +- ctl/bench.go | 5 +- ctl/common.go | 3 +- ctl/config.go | 3 +- ctl/config_test.go | 4 +- ctl/export.go | 5 +- ctl/import.go | 5 +- ctl/restore.go | 5 +- gossip/gossip.go | 105 ++++++++--- pilosa.go | 4 - server.go | 25 ++- server/cluster_test.go | 4 +- server/config.go | 132 ++++++++++++++ config_test.go => server/config_test.go | 17 +- server/server.go | 14 +- server/server_test.go | 5 +- toml/toml.go | 30 ++++ 21 files changed, 308 insertions(+), 307 deletions(-) delete mode 100644 config.go create mode 100644 server/config.go rename config_test.go => server/config_test.go (78%) create mode 100644 toml/toml.go diff --git a/broadcast_test.go b/broadcast_test.go index 8f0245c88..f4ff4c0fa 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -53,7 +53,10 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { - s := pilosa.NewServer() + s, err := pilosa.NewServer() + if err != nil { + t.Fatalf("getting new server: %v", err) + } sbr := NewSimpleBroadcastReceiver() sbh := NewSimpleBroadcastHandler() diff --git a/cluster.go b/cluster.go index 7e871698d..be668a6bc 100644 --- a/cluster.go +++ b/cluster.go @@ -40,9 +40,6 @@ const ( // DefaultPartitionN is the default number of partitions in a cluster. DefaultPartitionN = 256 - // DefaultReplicaN is the default number of replicas per partition. - DefaultReplicaN = 1 - // ClusterState represents the state returned in the /status endpoint. ClusterStateStarting = "STARTING" ClusterStateNormal = "NORMAL" @@ -276,7 +273,7 @@ func NewCluster() *Cluster { return &Cluster{ Hasher: &jmphasher{}, PartitionN: DefaultPartitionN, - ReplicaN: DefaultReplicaN, + ReplicaN: 1, EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel diff --git a/cmd/server_test.go b/cmd/server_test.go index b1f39e921..e69ce1134 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/cmd" _ "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa/toml" ) func TestServerHelp(t *testing.T) { @@ -65,7 +65,7 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) - v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) + v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() }, @@ -86,7 +86,7 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) - v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) + v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) return v.Error() }, }, @@ -113,7 +113,7 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) - v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11)) + v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11)) v.Check(cmd.Server.CPUProfile, profFile.Name()) v.Check(cmd.Server.CPUTime, time.Minute) v.Check(cmd.Server.Config.LogPath, logFile.Name()) diff --git a/config.go b/config.go deleted file mode 100644 index 483965916..000000000 --- a/config.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "time" -) - -// Cluster types. -const ( - ClusterNone = "" - ClusterStatic = "static" - ClusterGossip = "gossip" -) - -// TLSConfig contains TLS configuration -type TLSConfig struct { - // CertificatePath contains the path to the certificate (.crt or .pem file) - CertificatePath string `toml:"certificate-path"` - // CertificateKeyPath contains the path to the certificate key (.key file) - CertificateKeyPath string `toml:"certificate-key-path"` - // SkipVerify disables verification for self-signed certificates - SkipVerify bool `toml:"skip-verify"` -} - -// Config represents the configuration for the command. -type Config struct { - // DataDir is the directory where Pilosa stores both indexed data and - // running state such as cluster topology information. - DataDir string `toml:"data-dir"` - // Bind is the host:port on which Pilosa will listen. - Bind string `toml:"bind"` - - // MaxWritesPerRequest limits the number of mutating commands that can be in - // a single request to the server. This includes SetBit, ClearBit, - // SetRowAttrs & SetColumnAttrs. - MaxWritesPerRequest int `toml:"max-writes-per-request"` - - // LogPath configures where Pilosa will write logs. - LogPath string `toml:"log-path"` - - // Verbose toggles verbose logging which can be useful for debugging. - Verbose bool `toml:"verbose"` - - // TLS - TLS TLSConfig - - Cluster struct { - // Disabled controls whether clustering functionality is enabled. - Disabled bool `toml:"disabled"` - Coordinator bool `toml:"coordinator"` - ReplicaN int `toml:"replicas"` - Hosts []string `toml:"hosts"` - LongQueryTime Duration `toml:"long-query-time"` - } `toml:"cluster"` - - // Gossip config is based around memberlist.Config. - Gossip struct { - // Port indicates the port to which pilosa should bind for internal state sharing. - Port string `toml:"port"` - Seeds []string `toml:"seeds"` - Key string `toml:"key"` - // StreamTimeout is the timeout for establishing a stream connection with - // a remote node for a full state sync, and for stream read and write - // operations. Maps to memberlist TCPTimeout. - StreamTimeout Duration `toml:"stream-timeout"` - // SuspicionMult is the multiplier for determining the time an - // inaccessible node is considered suspect before declaring it dead. - // The actual timeout is calculated using the formula: - // - // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval - // - // This allows the timeout to scale properly with expected propagation - // delay with a larger cluster size. The higher the multiplier, the longer - // an inaccessible node is considered part of the cluster before declaring - // it dead, giving that suspect node more time to refute if it is indeed - // still alive. - SuspicionMult int `toml:"suspicion-mult"` - // PushPullInterval is the interval between complete state syncs. - // Complete state syncs are done with a single node over TCP and are - // quite expensive relative to standard gossiped messages. Setting this - // to zero will disable state push/pull syncs completely. - // - // Setting this interval lower (more frequent) will increase convergence - // speeds across larger clusters at the expense of increased bandwidth - // usage. - PushPullInterval Duration `toml:"push-pull-interval"` - // ProbeInterval and ProbeTimeout are used to configure probing behavior - // for memberlist. - // - // ProbeInterval is the interval between random node probes. Setting - // this lower (more frequent) will cause the memberlist cluster to detect - // failed nodes more quickly at the expense of increased bandwidth usage. - // - // ProbeTimeout is the timeout to wait for an ack from a probed node - // before assuming it is unhealthy. This should be set to 99-percentile - // of RTT (round-trip time) on your network. - ProbeInterval Duration `toml:"probe-interval"` - ProbeTimeout Duration `toml:"probe-timeout"` - - // Interval and Nodes are used to configure the gossip - // behavior of memberlist. - // - // Interval is the interval between sending messages that need - // to be gossiped that haven't been able to piggyback on probing messages. - // If this is set to zero, non-piggyback gossip is disabled. By lowering - // this value (more frequent) gossip messages are propagated across - // the cluster more quickly at the expense of increased bandwidth. - // - // Nodes is the number of random nodes to send gossip messages to - // per Interval. Increasing this number causes the gossip messages - // to propagate across the cluster more quickly at the expense of - // increased bandwidth. - // - // ToTheDeadTime is the interval after which a node has died that - // we will still try to gossip to it. This gives it a chance to refute. - Interval Duration `toml:"interval"` - Nodes int `toml:"nodes"` - ToTheDeadTime Duration `toml:"to-the-dead-time"` - } `toml:"gossip"` - - AntiEntropy struct { - Interval Duration `toml:"interval"` - } `toml:"anti-entropy"` - - Metric struct { - // Service can be statsd, expvar, or none. - Service string `toml:"service"` - // Host tells the statsd client where to write. - Host string `toml:"host"` - PollInterval Duration `toml:"poll-interval"` - // Diagnostics toggles sending some limited diagnostic information to - // Pilosa's developers. - Diagnostics bool `toml:"diagnostics"` - } `toml:"metric"` -} - -// NewConfig returns an instance of Config with default options. -func NewConfig() *Config { - c := &Config{ - DataDir: "~/.pilosa", - Bind: ":10101", - MaxWritesPerRequest: 5000, - // LogPath: "", - // Verbose: false, - TLS: TLSConfig{}, - } - - // Cluster config. - c.Cluster.Disabled = false - // c.Cluster.Coordinator = false - c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.Hosts = []string{} - c.Cluster.LongQueryTime = Duration(time.Minute) - - // Gossip config. - c.Gossip.Port = "14000" - // c.Gossip.Seeds = []string{} - // c.Gossip.Key = "" - c.Gossip.StreamTimeout = Duration(10 * time.Second) - c.Gossip.SuspicionMult = 4 - c.Gossip.PushPullInterval = Duration(30 * time.Second) - c.Gossip.ProbeInterval = Duration(1 * time.Second) - c.Gossip.ProbeTimeout = Duration(500 * time.Millisecond) - c.Gossip.Interval = Duration(200 * time.Millisecond) - c.Gossip.Nodes = 3 - c.Gossip.ToTheDeadTime = Duration(30 * time.Second) - - // AntiEntropy config. - c.AntiEntropy.Interval = Duration(10 * time.Minute) - - // Metric config. - c.Metric.Service = "none" - // c.Metric.Host = "" - c.Metric.PollInterval = Duration(0 * time.Minute) - c.Metric.Diagnostics = true - - return c -} - -// Validate that all configuration permutations are compatible with each other. -func (c *Config) Validate() error { - if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 { - return ErrConfigClusterEnabledHosts - } - return nil -} - -// Duration is a TOML wrapper type for time.Duration. -type Duration time.Duration - -// String returns the string representation of the duration. -func (d Duration) String() string { return time.Duration(d).String() } - -// UnmarshalText parses a TOML value into a duration value. -func (d *Duration) UnmarshalText(text []byte) error { - v, err := time.ParseDuration(string(text)) - if err != nil { - return err - } - - *d = Duration(v) - return nil -} - -// MarshalText writes duration value in text format. -func (d Duration) MarshalText() (text []byte, err error) { - return []byte(d.String()), nil -} - -// MarshalTOML write duration into valid TOML. -func (d Duration) MarshalTOML() ([]byte, error) { - return []byte(d.String()), nil -} diff --git a/ctl/backup.go b/ctl/backup.go index 8e725b41e..1b16391e0 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -21,6 +21,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // BackupCommand represents a command for backing up a view. @@ -39,7 +40,7 @@ type BackupCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewBackupCommand returns a new instance of BackupCommand. @@ -88,6 +89,6 @@ func (cmd *BackupCommand) TLSHost() string { return cmd.Host } -func (cmd *BackupCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/bench.go b/ctl/bench.go index 01e07cc14..9e37fb704 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -24,6 +24,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/server" ) // BenchCommand represents a command for benchmarking index operations. @@ -42,7 +43,7 @@ type BenchCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewBenchCommand returns a new instance of BenchCommand. @@ -110,6 +111,6 @@ func (cmd *BenchCommand) TLSHost() string { return cmd.Host } -func (cmd *BenchCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/common.go b/ctl/common.go index 9bba68657..6ebcd3132 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -18,13 +18,14 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" "github.com/spf13/pflag" ) // CommandWithTLSSupport is the interface for commands which has TLS settings type CommandWithTLSSupport interface { TLSHost() string - TLSConfiguration() pilosa.TLSConfig + TLSConfiguration() server.TLSConfig } // SetTLSConfig creates common TLS flags diff --git a/ctl/config.go b/ctl/config.go index 2047941c7..7277f74ea 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -21,12 +21,13 @@ import ( toml "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ConfigCommand represents a command for printing a default config. type ConfigCommand struct { *pilosa.CmdIO - Config *pilosa.Config + Config *server.Config } // NewConfigCommand returns a new instance of ConfigCommand. diff --git a/ctl/config_test.go b/ctl/config_test.go index b54446e34..ca08c273b 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -22,7 +22,7 @@ import ( "strings" "testing" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) func TestConfigCommand_Run(t *testing.T) { @@ -30,7 +30,7 @@ func TestConfigCommand_Run(t *testing.T) { stdin := bytes.NewReader(rder) r, w, _ := os.Pipe() cm := NewConfigCommand(stdin, w, os.Stderr) - cm.Config = pilosa.NewConfig() + cm.Config = server.NewConfig() err := cm.Run(context.Background()) w.Close() diff --git a/ctl/export.go b/ctl/export.go index 84b91edec..8c219a078 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -21,6 +21,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ExportCommand represents a command for bulk exporting data from a server. @@ -38,7 +39,7 @@ type ExportCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewExportCommand returns a new instance of ExportCommand. @@ -114,6 +115,6 @@ func (cmd *ExportCommand) TLSHost() string { return cmd.Host } -func (cmd *ExportCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *ExportCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/import.go b/ctl/import.go index 8103bf673..c3c117eb3 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -27,6 +27,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ImportCommand represents a command for bulk importing data. @@ -66,7 +67,7 @@ type ImportCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewImportCommand returns a new instance of ImportCommand. @@ -447,6 +448,6 @@ func (cmd *ImportCommand) TLSHost() string { return cmd.Host } -func (cmd *ImportCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *ImportCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/restore.go b/ctl/restore.go index 38528c2b5..89ee8ac8d 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -21,6 +21,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // RestoreCommand represents a command for restoring a frame from a backup. @@ -39,7 +40,7 @@ type RestoreCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewRestoreCommand returns a new instance of RestoreCommand. @@ -81,6 +82,6 @@ func (cmd *RestoreCommand) TLSHost() string { return cmd.Host } -func (cmd *RestoreCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/gossip/gossip.go b/gossip/gossip.go index a27da24c8..8ad00a8a7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -29,6 +29,7 @@ import ( "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -167,7 +168,7 @@ func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { +func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { g := &GossipMemberSet{ Logger: server.Logger, @@ -181,17 +182,11 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, } if g.transport == nil { - port, err := strconv.Atoi(cfg.Gossip.Port) + port, err := strconv.Atoi(cfg.Port) if err != nil { return nil, fmt.Errorf("convert port: %s", err) } - bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) - if err != nil { - return nil, fmt.Errorf("getting uri from bind address: %s", err) - } - host := bindURI.Host() - // Set up the transport. transport, err := NewTransport(host, port, g.logger) if err != nil { @@ -203,15 +198,10 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, port := g.transport.Net.GetAutoBindPort() - bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) - if err != nil { - return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err) - } - host := bindURI.Host() - var gossipKey []byte - if cfg.Gossip.Key != "" { - gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key) + var err error + if cfg.Key != "" { + gossipKey, err = ioutil.ReadFile(cfg.Key) if err != nil { return nil, fmt.Errorf("reading gossip key: %s", err) } @@ -226,14 +216,14 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, conf.AdvertisePort = port conf.AdvertiseAddr = pilosa.HostToIP(host) // - conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout) - conf.SuspicionMult = cfg.Gossip.SuspicionMult - conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval) - conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout) - conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval) - conf.GossipNodes = cfg.Gossip.Nodes - conf.GossipInterval = time.Duration(cfg.Gossip.Interval) - conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime) + conf.TCPTimeout = time.Duration(cfg.StreamTimeout) + conf.SuspicionMult = cfg.SuspicionMult + conf.PushPullInterval = time.Duration(cfg.PushPullInterval) + conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout) + conf.ProbeInterval = time.Duration(cfg.ProbeInterval) + conf.GossipNodes = cfg.Nodes + conf.GossipInterval = time.Duration(cfg.Interval) + conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime) // conf.Delegate = g conf.SecretKey = gossipKey @@ -242,7 +232,7 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, g.config = &gossipConfig{ memberlistConfig: conf, - gossipSeeds: cfg.Gossip.Seeds, + gossipSeeds: cfg.Seeds, } g.statusHandler = server @@ -526,3 +516,68 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { return nt, nil } + +// Config holds toml-friendly memberlist configuration. +type Config struct { + // Port indicates the port to which pilosa should bind for internal state sharing. + Port string `toml:"port"` + Seeds []string `toml:"seeds"` + Key string `toml:"key"` + // StreamTimeout is the timeout for establishing a stream connection with + // a remote node for a full state sync, and for stream read and write + // operations. Maps to memberlist TCPTimeout. + StreamTimeout toml.Duration `toml:"stream-timeout"` + // SuspicionMult is the multiplier for determining the time an + // inaccessible node is considered suspect before declaring it dead. + // The actual timeout is calculated using the formula: + // + // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval + // + // This allows the timeout to scale properly with expected propagation + // delay with a larger cluster size. The higher the multiplier, the longer + // an inaccessible node is considered part of the cluster before declaring + // it dead, giving that suspect node more time to refute if it is indeed + // still alive. + SuspicionMult int `toml:"suspicion-mult"` + // PushPullInterval is the interval between complete state syncs. + // Complete state syncs are done with a single node over TCP and are + // quite expensive relative to standard gossiped messages. Setting this + // to zero will disable state push/pull syncs completely. + // + // Setting this interval lower (more frequent) will increase convergence + // speeds across larger clusters at the expense of increased bandwidth + // usage. + PushPullInterval toml.Duration `toml:"push-pull-interval"` + // ProbeInterval and ProbeTimeout are used to configure probing behavior + // for memberlist. + // + // ProbeInterval is the interval between random node probes. Setting + // this lower (more frequent) will cause the memberlist cluster to detect + // failed nodes more quickly at the expense of increased bandwidth usage. + // + // ProbeTimeout is the timeout to wait for an ack from a probed node + // before assuming it is unhealthy. This should be set to 99-percentile + // of RTT (round-trip time) on your network. + ProbeInterval toml.Duration `toml:"probe-interval"` + ProbeTimeout toml.Duration `toml:"probe-timeout"` + + // Interval and Nodes are used to configure the gossip + // behavior of memberlist. + // + // Interval is the interval between sending messages that need + // to be gossiped that haven't been able to piggyback on probing messages. + // If this is set to zero, non-piggyback gossip is disabled. By lowering + // this value (more frequent) gossip messages are propagated across + // the cluster more quickly at the expense of increased bandwidth. + // + // Nodes is the number of random nodes to send gossip messages to + // per Interval. Increasing this number causes the gossip messages + // to propagate across the cluster more quickly at the expense of + // increased bandwidth. + // + // ToTheDeadTime is the interval after which a node has died that + // we will still try to gossip to it. This gives it a chance to refute. + Interval toml.Duration `toml:"interval"` + Nodes int `toml:"nodes"` + ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` +} diff --git a/pilosa.go b/pilosa.go index 2d98b910a..62eb88113 100644 --- a/pilosa.go +++ b/pilosa.go @@ -68,10 +68,6 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed") - ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") - ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") - ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") diff --git a/server.go b/server.go index f28ca3864..6d25e3320 100644 --- a/server.go +++ b/server.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "crypto/tls" - "errors" "fmt" "net" "net/http" @@ -31,6 +30,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -45,6 +45,16 @@ var _ Broadcaster = &Server{} var _ BroadcastHandler = &Server{} var _ StatusHandler = &Server{} +// ServerOption is a functional option type for pilosa.Server +type ServerOption func(s *Server) error + +func OptServerLogger(l Logger) ServerOption { + return func(s *Server) error { + s.Logger = l + return nil + } +} + // Server represents a holder wrapped by a running HTTP server. type Server struct { ln net.Listener @@ -90,7 +100,7 @@ type Server struct { } // NewServer returns a new instance of Server. -func NewServer() *Server { +func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ closing: make(chan struct{}), @@ -107,16 +117,23 @@ func NewServer() *Server { NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: time.Duration(NewConfig().AntiEntropy.Interval), + AntiEntropyInterval: time.Minute * 10, MetricInterval: 0, DiagnosticInterval: 0, Logger: NopLogger, } + for _, opt := range opts { + err := opt(s) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + s.Handler.API = NewAPI() s.Handler.API.Holder = s.Holder - return s + return s, nil } // Open opens and initializes the server. diff --git a/server/cluster_test.go b/server/cluster_test.go index 7f4886612..5b0321285 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -55,7 +55,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.Coordinator = m0.Server.NodeID m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger) - gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) + gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Server.URI.Host(), m0.Config.Gossip, m0.Server) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Server.Cluster.Coordinator = m0.Server.NodeID m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger) - gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) + gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Server.URI.Host(), m1.Config.Gossip, m1.Server) if err != nil { t.Fatal(err) } diff --git a/server/config.go b/server/config.go new file mode 100644 index 000000000..2e2feeeb2 --- /dev/null +++ b/server/config.go @@ -0,0 +1,132 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "time" + + "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/toml" +) + +// Cluster types. +const ( + ClusterNone = "" + ClusterStatic = "static" + ClusterGossip = "gossip" +) + +// TLSConfig contains TLS configuration +type TLSConfig struct { + // CertificatePath contains the path to the certificate (.crt or .pem file) + CertificatePath string `toml:"certificate-path"` + // CertificateKeyPath contains the path to the certificate key (.key file) + CertificateKeyPath string `toml:"certificate-key-path"` + // SkipVerify disables verification for self-signed certificates + SkipVerify bool `toml:"skip-verify"` +} + +// Config represents the configuration for the command. +type Config struct { + // DataDir is the directory where Pilosa stores both indexed data and + // running state such as cluster topology information. + DataDir string `toml:"data-dir"` + // Bind is the host:port on which Pilosa will listen. + Bind string `toml:"bind"` + + // MaxWritesPerRequest limits the number of mutating commands that can be in + // a single request to the server. This includes SetBit, ClearBit, + // SetRowAttrs & SetColumnAttrs. + MaxWritesPerRequest int `toml:"max-writes-per-request"` + + // LogPath configures where Pilosa will write logs. + LogPath string `toml:"log-path"` + + // Verbose toggles verbose logging which can be useful for debugging. + Verbose bool `toml:"verbose"` + + // TLS + TLS TLSConfig + + Cluster struct { + // Disabled controls whether clustering functionality is enabled. + Disabled bool `toml:"disabled"` + Coordinator bool `toml:"coordinator"` + ReplicaN int `toml:"replicas"` + Hosts []string `toml:"hosts"` + LongQueryTime toml.Duration `toml:"long-query-time"` + } `toml:"cluster"` + + // Gossip config is based around memberlist.Config. + Gossip gossip.Config `toml:"gossip"` + + AntiEntropy struct { + Interval toml.Duration `toml:"interval"` + } `toml:"anti-entropy"` + + Metric struct { + // Service can be statsd, expvar, or none. + Service string `toml:"service"` + // Host tells the statsd client where to write. + Host string `toml:"host"` + PollInterval toml.Duration `toml:"poll-interval"` + // Diagnostics toggles sending some limited diagnostic information to + // Pilosa's developers. + Diagnostics bool `toml:"diagnostics"` + } `toml:"metric"` +} + +// NewConfig returns an instance of Config with default options. +func NewConfig() *Config { + c := &Config{ + DataDir: "~/.pilosa", + Bind: ":10101", + MaxWritesPerRequest: 5000, + // LogPath: "", + // Verbose: false, + TLS: TLSConfig{}, + } + + // Cluster config. + c.Cluster.Disabled = false + // c.Cluster.Coordinator = false + c.Cluster.ReplicaN = 1 + c.Cluster.Hosts = []string{} + c.Cluster.LongQueryTime = toml.Duration(time.Minute) + + // Gossip config. + c.Gossip.Port = "14000" + // c.Gossip.Seeds = []string{} + // c.Gossip.Key = "" + c.Gossip.StreamTimeout = toml.Duration(10 * time.Second) + c.Gossip.SuspicionMult = 4 + c.Gossip.PushPullInterval = toml.Duration(30 * time.Second) + c.Gossip.ProbeInterval = toml.Duration(1 * time.Second) + c.Gossip.ProbeTimeout = toml.Duration(500 * time.Millisecond) + c.Gossip.Interval = toml.Duration(200 * time.Millisecond) + c.Gossip.Nodes = 3 + c.Gossip.ToTheDeadTime = toml.Duration(30 * time.Second) + + // AntiEntropy config. + c.AntiEntropy.Interval = toml.Duration(10 * time.Minute) + + // Metric config. + c.Metric.Service = "none" + // c.Metric.Host = "" + c.Metric.PollInterval = toml.Duration(0 * time.Minute) + c.Metric.Diagnostics = true + + return c +} diff --git a/config_test.go b/server/config_test.go similarity index 78% rename from config_test.go rename to server/config_test.go index c07486ddb..2027ef35a 100644 --- a/config_test.go +++ b/server/config_test.go @@ -12,34 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package server_test import ( "reflect" "testing" "time" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/toml" ) func Test_NewConfig(t *testing.T) { - c := pilosa.NewConfig() + c := server.NewConfig() if c.Cluster.Disabled { t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled) } - - // Ensure that hosts can't be specificed on a non-disabled cluster. - c.Cluster.Hosts = []string{c.Bind, "localhost:10102"} - - // Change cluster type from the default (gossip) to an invalid string. - if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts { - t.Fatal(err) - } } func TestDuration(t *testing.T) { - d := pilosa.Duration(time.Second * 182) + d := toml.Duration(time.Second * 182) if d.String() != "3m2s" { t.Fatalf("Unexpected time Duration %s", d) } diff --git a/server/server.go b/server/server.go index 7f66d31ed..8f4ab6147 100644 --- a/server/server.go +++ b/server/server.go @@ -56,7 +56,7 @@ type Command struct { Server *pilosa.Server // Configuration. - Config *pilosa.Config + Config *Config // Profiling options. CPUProfile string @@ -80,9 +80,10 @@ type Command struct { // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { + s, _ := pilosa.NewServer() return &Command{ - Server: pilosa.NewServer(), - Config: pilosa.NewConfig(), + Server: s, + Config: NewConfig(), CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), @@ -150,11 +151,6 @@ func (m *Command) SetupLogger() error { // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - err := m.Config.Validate() - if err != nil { - return err - } - m.Server.Handler.Logger = m.Server.Logger m.Server.Holder.Logger = m.Server.Logger m.Server.Holder.Stats.SetLogger(m.Server.Logger) @@ -278,7 +274,7 @@ func (m *Command) SetupNetworking() error { } m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger) - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) if err != nil { return err } diff --git a/server/server_test.go b/server/server_test.go index 497ffd7da..532257c13 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,6 +30,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -489,8 +490,8 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { } // ParseConfig parses s into a Config. -func ParseConfig(s string) (pilosa.Config, error) { - var c pilosa.Config +func ParseConfig(s string) (server.Config, error) { + var c server.Config _, err := toml.Decode(s, &c) return c, err } diff --git a/toml/toml.go b/toml/toml.go new file mode 100644 index 000000000..5193ad787 --- /dev/null +++ b/toml/toml.go @@ -0,0 +1,30 @@ +package toml + +import "time" + +// Duration is a TOML wrapper type for time.Duration. +type Duration time.Duration + +// String returns the string representation of the duration. +func (d Duration) String() string { return time.Duration(d).String() } + +// UnmarshalText parses a TOML value into a duration value. +func (d *Duration) UnmarshalText(text []byte) error { + v, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + + *d = Duration(v) + return nil +} + +// MarshalText writes duration value in text format. +func (d Duration) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// MarshalTOML write duration into valid TOML. +func (d Duration) MarshalTOML() ([]byte, error) { + return []byte(d.String()), nil +} From 23f6acc165f9e1395337a44a891ec0091caf86c1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 19 Apr 2018 15:37:08 -0500 Subject: [PATCH 251/311] panic if NewCommand errors on NewServer --- server/server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 8f4ab6147..2606ff248 100644 --- a/server/server.go +++ b/server/server.go @@ -80,7 +80,11 @@ type Command struct { // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { - s, _ := pilosa.NewServer() + s, err := pilosa.NewServer() + if err != nil { + panic(err) + } + return &Command{ Server: s, Config: NewConfig(), From 5177b243f3133b95fb90061627bf68799c5270c6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 19 Apr 2018 17:03:49 -0500 Subject: [PATCH 252/311] use ValCount return type (instead of SumCount, MinCount, and MaxCount) --- executor.go | 168 ++++++++++++++++++++---------------------- executor_test.go | 8 +- handler.go | 8 +- internal/public.pb.go | 142 +++++++++++++++++------------------ internal/public.proto | 6 +- 5 files changed, 160 insertions(+), 172 deletions(-) diff --git a/executor.go b/executor.go index 83f35e7d3..ac163cf5c 100644 --- a/executor.go +++ b/executor.go @@ -205,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) { +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return SumCount{}, errors.New("Sum(): frame required") + return ValCount{}, errors.New("Sum(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return SumCount{}, errors.New("Sum(): field required") + return ValCount{}, errors.New("Sum(): field required") } if len(c.Children) > 1 { - return SumCount{}, errors.New("Sum() only accepts a single bitmap input") + return ValCount{}, errors.New("Sum() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -223,32 +223,32 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(SumCount) - return other.Add(v.(SumCount)) + other, _ := prev.(ValCount) + return other.Add(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return SumCount{}, err + return ValCount{}, err } - other, _ := result.(SumCount) + other, _ := result.(ValCount) if other.Count == 0 { - return SumCount{}, nil + return ValCount{}, nil } return other, nil } // executeFieldMin executes a Min() call. -func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MinCount, error) { +func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return MinCount{}, errors.New("Min(): frame required") + return ValCount{}, errors.New("Min(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return MinCount{}, errors.New("Min(): field required") + return ValCount{}, errors.New("Min(): field required") } if len(c.Children) > 1 { - return MinCount{}, errors.New("Min() only accepts a single bitmap input") + return ValCount{}, errors.New("Min() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -258,32 +258,32 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(MinCount) - return other.Smaller(v.(MinCount)) + other, _ := prev.(ValCount) + return other.Smaller(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return MinCount{}, err + return ValCount{}, err } - other, _ := result.(MinCount) + other, _ := result.(ValCount) if other.Count == 0 { - return MinCount{}, nil + return ValCount{}, nil } return other, nil } // executeFieldMax executes a Max() call. -func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MaxCount, error) { +func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return MaxCount{}, errors.New("Max(): frame required") + return ValCount{}, errors.New("Max(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return MaxCount{}, errors.New("Max(): field required") + return ValCount{}, errors.New("Max(): field required") } if len(c.Children) > 1 { - return MaxCount{}, errors.New("Max() only accepts a single bitmap input") + return ValCount{}, errors.New("Max() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -293,18 +293,18 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(MaxCount) - return other.Larger(v.(MaxCount)) + other, _ := prev.(ValCount) + return other.Larger(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return MaxCount{}, err + return ValCount{}, err } - other, _ := result.(MaxCount) + other, _ := result.(ValCount) if other.Count == 0 { - return MaxCount{}, nil + return ValCount{}, nil } return other, nil } @@ -395,12 +395,12 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } // executeSumCountSlice calculates the sum and count for fields on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { +func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return SumCount{}, err + return ValCount{}, err } filter = bm } @@ -410,36 +410,36 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return SumCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return SumCount{}, nil + return ValCount{}, nil } fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) if fragment == nil { - return SumCount{}, nil + return ValCount{}, nil } vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { - return SumCount{}, err + return ValCount{}, err } - return SumCount{ - Sum: int64(vsum) + (int64(vcount) * field.Min), + return ValCount{ + Val: int64(vsum) + (int64(vcount) * field.Min), Count: int64(vcount), }, nil } // executeFieldMinSlice calculates the min for fields on a slice. -func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MinCount, error) { +func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return MinCount{}, err + return ValCount{}, err } filter = bm } @@ -449,36 +449,36 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return MinCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return MinCount{}, nil + return ValCount{}, nil } fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) if fragment == nil { - return MinCount{}, nil + return ValCount{}, nil } fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) if err != nil { - return MinCount{}, err + return ValCount{}, err } - return MinCount{ - Min: int64(fmin) + field.Min, + return ValCount{ + Val: int64(fmin) + field.Min, Count: int64(fcount), }, nil } // executeFieldMaxSlice calculates the max for fields on a slice. -func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MaxCount, error) { +func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return MaxCount{}, err + return ValCount{}, err } filter = bm } @@ -488,25 +488,25 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return MaxCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return MaxCount{}, nil + return ValCount{}, nil } fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) if fragment == nil { - return MaxCount{}, nil + return ValCount{}, nil } fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) if err != nil { - return MaxCount{}, err + return ValCount{}, err } - return MaxCount{ - Max: int64(fmax) + field.Min, + return ValCount{ + Val: int64(fmax) + field.Min, Count: int64(fcount), }, nil } @@ -1509,7 +1509,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * switch call.Name { case "Average", "Sum": - v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil + v, err = decodeValCount(pb.Results[i].GetValCount()), nil case "TopN": v, err = decodePairs(pb.Results[i].GetPairs()), nil case "Count": @@ -1747,63 +1747,51 @@ func needsSlices(calls []*pql.Call) bool { return false } -// SumCount represents a grouping of sum & count for Sum() and Average() calls. -type SumCount struct { - Sum int64 `json:"sum"` +// ValCount represents a grouping of sum & count for Sum() and Average() calls. +type ValCount struct { + Val int64 `json:"value"` Count int64 `json:"count"` } -func (sc *SumCount) Add(other SumCount) SumCount { - return SumCount{ - Sum: sc.Sum + other.Sum, - Count: sc.Count + other.Count, +func (vc *ValCount) Add(other ValCount) ValCount { + return ValCount{ + Val: vc.Val + other.Val, + Count: vc.Count + other.Count, } } -func encodeSumCount(sc SumCount) *internal.SumCount { - return &internal.SumCount{ - Sum: sc.Sum, - Count: sc.Count, +func encodeValCount(vc ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, } } -func decodeSumCount(pb *internal.SumCount) SumCount { - return SumCount{ - Sum: pb.Sum, +func decodeValCount(pb *internal.ValCount) ValCount { + return ValCount{ + Val: pb.Val, Count: pb.Count, } } -// MinCount represents a grouping of min and count for Min() calls. -type MinCount struct { - Min int64 `json:"min"` - Count int64 `json:"count"` -} - -// Smaller returns the smaller of the two MinCounts. -func (mc *MinCount) Smaller(other MinCount) MinCount { - if mc.Count == 0 || (other.Min < mc.Min && other.Count > 0) { +// Smaller returns the smaller of the two ValCounts. +func (vc *ValCount) Smaller(other ValCount) ValCount { + if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { return other } - return MinCount{ - Min: mc.Min, - Count: mc.Count, + return ValCount{ + Val: vc.Val, + Count: vc.Count, } } -// MaxCount represents a grouping of max and count for Max() calls. -type MaxCount struct { - Max int64 `json:"max"` - Count int64 `json:"count"` -} - -// Larger returns the larger of the two MaxCounts. -func (mc *MaxCount) Larger(other MaxCount) MaxCount { - if mc.Count == 0 || (other.Max > mc.Max && other.Count > 0) { +// Larger returns the larger of the two ValCounts. +func (vc *ValCount) Larger(other ValCount) ValCount { + if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { return other } - return MaxCount{ - Max: mc.Max, - Count: mc.Count, + return ValCount{ + Val: vc.Val, + Count: vc.Count, } } diff --git a/executor_test.go b/executor_test.go index d5fa6e5d6..181a1a438 100644 --- a/executor_test.go +++ b/executor_test.go @@ -658,7 +658,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.MinCount{Min: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -684,7 +684,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.MaxCount{Max: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -738,7 +738,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("NoFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -746,7 +746,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("WithFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) diff --git a/handler.go b/handler.go index d7536f738..919008d57 100644 --- a/handler.go +++ b/handler.go @@ -1372,7 +1372,7 @@ const ( QueryResultTypeNil uint32 = iota QueryResultTypeBitmap QueryResultTypePairs - QueryResultTypeSumCount + QueryResultTypeValCount QueryResultTypeUint64 QueryResultTypeBool ) @@ -1461,9 +1461,9 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Type = QueryResultTypePairs pb.Results[i].Pairs = encodePairs(result) - case SumCount: - pb.Results[i].Type = QueryResultTypeSumCount - pb.Results[i].SumCount = encodeSumCount(result) + case ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = encodeValCount(result) case uint64: pb.Results[i].Type = QueryResultTypeUint64 pb.Results[i].N = result diff --git a/internal/public.pb.go b/internal/public.pb.go index dfb9a819e..31447b9f0 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,7 +11,7 @@ It has these top-level messages: Bitmap Pair - SumCount + ValCount Bit ColumnAttrSet Attr @@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 { return 0 } -type SumCount struct { - Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` +type ValCount struct { + Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *SumCount) Reset() { *m = SumCount{} } -func (m *SumCount) String() string { return proto.CompactTextString(m) } -func (*SumCount) ProtoMessage() {} -func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (m *ValCount) Reset() { *m = ValCount{} } +func (m *ValCount) String() string { return proto.CompactTextString(m) } +func (*ValCount) ProtoMessage() {} +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } -func (m *SumCount) GetSum() int64 { +func (m *ValCount) GetVal() int64 { if m != nil { - return m.Sum + return m.Val } return 0 } -func (m *SumCount) GetCount() int64 { +func (m *ValCount) GetCount() int64 { if m != nil { return m.Count } @@ -358,7 +358,7 @@ type QueryResult struct { Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } @@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetSumCount() *SumCount { +func (m *QueryResult) GetValCount() *ValCount { if m != nil { - return m.SumCount + return m.ValCount } return nil } @@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") - proto.RegisterType((*SumCount)(nil), "internal.SumCount") + proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") @@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *SumCount) Marshal() (dAtA []byte, err error) { +func (m *ValCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *SumCount) MarshalTo(dAtA []byte) (int, error) { +func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.Sum != 0 { + if m.Val != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + i = encodeVarintPublic(dAtA, i, uint64(m.Val)) } if m.Count != 0 { dAtA[i] = 0x10 @@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.SumCount != nil { + if m.ValCount != nil { dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size())) - n6, err := m.SumCount.MarshalTo(dAtA[i:]) + i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) + n6, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) { return n } -func (m *SumCount) Size() (n int) { +func (m *ValCount) Size() (n int) { var l int _ = l - if m.Sum != 0 { - n += 1 + sovPublic(uint64(m.Sum)) + if m.Val != 0 { + n += 1 + sovPublic(uint64(m.Val)) } if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) @@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) { if m.Changed { n += 2 } - if m.SumCount != nil { - l = m.SumCount.Size() + if m.ValCount != nil { + l = m.ValCount.Size() n += 1 + l + sovPublic(uint64(l)) } if m.Type != 0 { @@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } -func (m *SumCount) Unmarshal(dAtA []byte) error { +func (m *ValCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: SumCount: wiretype end group for non-group") + return fmt.Errorf("proto: ValCount: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType) } - m.Sum = 0 + m.Val = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= (int64(b) & 0x7F) << shift + m.Val |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { m.Changed = bool(v != 0) case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.SumCount == nil { - m.SumCount = &SumCount{} + if m.ValCount == nil { + m.ValCount = &ValCount{} } - if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{ // 705 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, 0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, - 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3, + 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3, 0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76, 0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce, 0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6, - 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c, - 0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85, - 0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45, - 0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a, - 0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7, - 0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7, - 0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a, - 0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9, - 0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb, - 0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21, - 0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, - 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94, - 0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, + 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce, + 0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42, + 0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22, + 0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15, + 0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb, + 0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b, + 0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45, + 0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec, + 0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed, + 0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10, + 0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, + 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94, + 0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, 0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb, 0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80, 0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd, 0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76, - 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36, - 0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc, - 0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b, - 0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4, - 0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a, - 0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad, - 0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d, - 0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2, - 0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5, - 0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23, - 0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8, - 0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b, - 0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03, - 0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad, - 0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8, - 0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f, - 0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91, - 0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc, - 0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f, - 0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4, - 0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00, + 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d, + 0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8, + 0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7, + 0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69, + 0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55, + 0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b, + 0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b, + 0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85, + 0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a, + 0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47, + 0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0, + 0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56, + 0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06, + 0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b, + 0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0, + 0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f, + 0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23, + 0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9, + 0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde, + 0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8, + 0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 5fa40cfab..026a3748b 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,8 +14,8 @@ message Pair { uint64 Count = 2; } -message SumCount { - int64 Sum = 1; +message ValCount { + int64 Val = 1; int64 Count = 2; } @@ -64,7 +64,7 @@ message QueryResult { Bitmap Bitmap = 1; uint64 N = 2; repeated Pair Pairs = 3; - SumCount SumCount = 5; + ValCount ValCount = 5; bool Changed = 4; } From edee152a9bc06756fc2a32180f4a1f981f868b52 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 20 Apr 2018 13:20:56 -0500 Subject: [PATCH 253/311] remove Index.MergeSchemas() method --- index.go | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/index.go b/index.go index 4676e1b43..00ee98f11 100644 --- a/index.go +++ b/index.go @@ -481,46 +481,6 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// MergeSchemas combines indexes and frames from a and b into one schema. -func MergeSchemas(a, b []*IndexInfo) []*IndexInfo { - // Generate a map from both schemas. - m := make(map[string]map[string]map[string]struct{}) - for _, idxs := range [][]*IndexInfo{a, b} { - for _, idx := range idxs { - if m[idx.Name] == nil { - m[idx.Name] = make(map[string]map[string]struct{}) - } - for _, frame := range idx.Frames { - if m[idx.Name][frame.Name] == nil { - m[idx.Name][frame.Name] = make(map[string]struct{}) - } - for _, view := range frame.Views { - m[idx.Name][frame.Name][view.Name] = struct{}{} - } - } - } - } - - // Generate new schema from map. - idxs := make([]*IndexInfo, 0, len(m)) - for idx, frames := range m { - di := &IndexInfo{Name: idx} - for frame, views := range frames { - fi := &FrameInfo{Name: frame} - for view := range views { - fi.Views = append(fi.Views, &ViewInfo{Name: view}) - } - sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) - } - sort.Sort(frameInfoSlice(di.Frames)) - idxs = append(idxs, di) - } - sort.Sort(indexInfoSlice(idxs)) - - return idxs -} - // EncodeIndexes converts a into its internal representation. func EncodeIndexes(a []*Index) []*internal.Index { other := make([]*internal.Index, len(a)) From f2729b90d7ba5e033a8e2aa13ac8804bf0d73379 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 10:43:06 -0500 Subject: [PATCH 254/311] vendor github.com/golang/groupcache/lru. rebuild Gopkg.lock --- Gopkg.lock | 58 +++++++++++++++++------ NOTICE | 36 +++++++++++++++ cache.go | 2 +- lru/lru.go | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 15 deletions(-) create mode 100644 NOTICE create mode 100644 lru/lru.go diff --git a/Gopkg.lock b/Gopkg.lock index af6174c08..f29660b22 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -57,7 +57,10 @@ [[projects]] name = "github.com/go-ole/go-ole" - packages = [".","oleutil"] + packages = [ + ".", + "oleutil" + ] revision = "0e87ea779d9deb219633b828a023b32e1244dd57" version = "v1.2.0" @@ -67,12 +70,6 @@ revision = "100ba4e885062801d56799d78530b73b178a78f3" version = "v0.4" -[[projects]] - branch = "master" - name = "github.com/golang/groupcache" - packages = ["lru"] - revision = "84a468cf14b4376def5d68c722b139b881c450a4" - [[projects]] branch = "master" name = "github.com/golang/protobuf" @@ -130,7 +127,17 @@ [[projects]] branch = "master" name = "github.com/hashicorp/hcl" - packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] @@ -154,7 +161,10 @@ [[projects]] branch = "master" name = "github.com/miekg/dns" - packages = [".","internal/socket"] + packages = [ + ".", + "internal/socket" + ] revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba" [[projects]] @@ -195,7 +205,14 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["cpu","host","internal/common","mem","net","process"] + packages = [ + "cpu", + "host", + "internal/common", + "mem", + "net", + "process" + ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" @@ -208,7 +225,10 @@ [[projects]] branch = "master" name = "github.com/spf13/afero" - packages = [".","mem"] + packages = [ + ".", + "mem" + ] revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" [[projects]] @@ -256,13 +276,23 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows"] + packages = [ + "unix", + "windows" + ] revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] + packages = [ + "internal/gen", + "internal/triegen", + "internal/ucd", + "transform", + "unicode/cldr", + "unicode/norm" + ] revision = "88f656faf3f37f690df1a32515b479415e1a6769" [[projects]] @@ -274,6 +304,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b" + inputs-digest = "61a00007fe7398d958fc8668543d42293f1310f9f9b74c63ff96f5a892eab1e7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..7613b11e7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,36 @@ +Software license +================ + +Copyright 2017 Pilosa Corp. + +Licensed under the Apache License, Version 2.0 (the "License"). +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Third-party software licenses +============================= + +The file /pilosa/lru/lru.go contains a redistribution of lru +(github.com/golang/groupcache/lru); the license follows: + + Copyright 2013 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cache.go b/cache.go index 9c909ced4..e27d2f93c 100644 --- a/cache.go +++ b/cache.go @@ -22,8 +22,8 @@ import ( "sync" "time" - "github.com/golang/groupcache/lru" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/lru" ) const ( diff --git a/lru/lru.go b/lru/lru.go new file mode 100644 index 000000000..532cc45e6 --- /dev/null +++ b/lru/lru.go @@ -0,0 +1,133 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package lru implements an LRU cache. +package lru + +import "container/list" + +// Cache is an LRU cache. It is not safe for concurrent access. +type Cache struct { + // MaxEntries is the maximum number of cache entries before + // an item is evicted. Zero means no limit. + MaxEntries int + + // OnEvicted optionally specificies a callback function to be + // executed when an entry is purged from the cache. + OnEvicted func(key Key, value interface{}) + + ll *list.List + cache map[interface{}]*list.Element +} + +// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators +type Key interface{} + +type entry struct { + key Key + value interface{} +} + +// New creates a new Cache. +// If maxEntries is zero, the cache has no limit and it's assumed +// that eviction is done by the caller. +func New(maxEntries int) *Cache { + return &Cache{ + MaxEntries: maxEntries, + ll: list.New(), + cache: make(map[interface{}]*list.Element), + } +} + +// Add adds a value to the cache. +func (c *Cache) Add(key Key, value interface{}) { + if c.cache == nil { + c.cache = make(map[interface{}]*list.Element) + c.ll = list.New() + } + if ee, ok := c.cache[key]; ok { + c.ll.MoveToFront(ee) + ee.Value.(*entry).value = value + return + } + ele := c.ll.PushFront(&entry{key, value}) + c.cache[key] = ele + if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { + c.RemoveOldest() + } +} + +// Get looks up a key's value from the cache. +func (c *Cache) Get(key Key) (value interface{}, ok bool) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.ll.MoveToFront(ele) + return ele.Value.(*entry).value, true + } + return +} + +// Remove removes the provided key from the cache. +func (c *Cache) Remove(key Key) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.removeElement(ele) + } +} + +// RemoveOldest removes the oldest item from the cache. +func (c *Cache) RemoveOldest() { + if c.cache == nil { + return + } + ele := c.ll.Back() + if ele != nil { + c.removeElement(ele) + } +} + +func (c *Cache) removeElement(e *list.Element) { + c.ll.Remove(e) + kv := e.Value.(*entry) + delete(c.cache, kv.key) + if c.OnEvicted != nil { + c.OnEvicted(kv.key, kv.value) + } +} + +// Len returns the number of items in the cache. +func (c *Cache) Len() int { + if c.cache == nil { + return 0 + } + return c.ll.Len() +} + +// Clear purges all stored items from the cache. +func (c *Cache) Clear() { + if c.OnEvicted != nil { + for _, e := range c.cache { + kv := e.Value.(*entry) + c.OnEvicted(kv.key, kv.value) + } + } + c.ll = nil + c.cache = nil +} From 3282cf8cbe8e9776ad81e506c7a0065c6c38ce55 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 11:32:41 -0500 Subject: [PATCH 255/311] remove PATCH frame endpoint --- api.go | 73 ++++++++++++++++--------------------------- apimethod_string.go | 4 +-- docs/api-reference.md | 29 ----------------- handler.go | 41 ------------------------ handler_test.go | 24 -------------- 5 files changed, 29 insertions(+), 142 deletions(-) diff --git a/api.go b/api.go index a5953217d..700b21d25 100644 --- a/api.go +++ b/api.go @@ -925,23 +925,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return err } -// ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: -// what happens if there is already data in the frame? -func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { - if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api method") - } - - // Retrieve index by name. - frame := api.Holder.Frame(indexName, frameName) - if frame == nil { - return ErrFrameNotFound - } - - // Set default time quantum on index. - return frame.SetTimeQuantum(timeQuantum) -} - // MaxSlices returns the maximum slice number for each index in a map. func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() @@ -1168,7 +1151,6 @@ const ( apiMarshalFragment //apiMaxInverseSlices // not implemented //apiMaxSlices // not implemented - apiModifyFrameTimeQuantum apiQuery apiRecalculateCaches apiRemoveNode @@ -1196,32 +1178,31 @@ var methodsResizing = map[apiMethod]struct{}{ } var methodsNormal = map[apiMethod]struct{}{ - apiCreateField: struct{}{}, - apiCreateFrame: struct{}{}, - apiCreateIndex: struct{}{}, - apiCreateInputDefinition: struct{}{}, - apiDeleteField: struct{}{}, - apiDeleteFrame: struct{}{}, - apiDeleteIndex: struct{}{}, - apiDeleteInputDefinition: struct{}{}, - apiDeleteView: struct{}{}, - apiExportCSV: struct{}{}, - apiFields: struct{}{}, - apiFragmentBlockData: struct{}{}, - apiFragmentBlocks: struct{}{}, - apiFrameAttrDiff: struct{}{}, - apiImport: struct{}{}, - apiImportValue: struct{}{}, - apiIndex: struct{}{}, - apiIndexAttrDiff: struct{}{}, - apiInputDefinition: struct{}{}, - apiModifyFrameTimeQuantum: struct{}{}, - apiQuery: struct{}{}, - apiRecalculateCaches: struct{}{}, - apiRemoveNode: struct{}{}, - apiRestoreFrame: struct{}{}, - apiSliceNodes: struct{}{}, - apiUnmarshalFragment: struct{}{}, - apiViews: struct{}{}, - apiWriteInput: struct{}{}, + apiCreateField: struct{}{}, + apiCreateFrame: struct{}{}, + apiCreateIndex: struct{}{}, + apiCreateInputDefinition: struct{}{}, + apiDeleteField: struct{}{}, + apiDeleteFrame: struct{}{}, + apiDeleteIndex: struct{}{}, + apiDeleteInputDefinition: struct{}{}, + apiDeleteView: struct{}{}, + apiExportCSV: struct{}{}, + apiFields: struct{}{}, + apiFragmentBlockData: struct{}{}, + apiFragmentBlocks: struct{}{}, + apiFrameAttrDiff: struct{}{}, + apiImport: struct{}{}, + apiImportValue: struct{}{}, + apiIndex: struct{}{}, + apiIndexAttrDiff: struct{}{}, + apiInputDefinition: struct{}{}, + apiQuery: struct{}{}, + apiRecalculateCaches: struct{}{}, + apiRemoveNode: struct{}{}, + apiRestoreFrame: struct{}{}, + apiSliceNodes: struct{}{}, + apiUnmarshalFragment: struct{}{}, + apiViews: struct{}{}, + apiWriteInput: struct{}{}, } diff --git a/apimethod_string.go b/apimethod_string.go index 8f77b69e7..8ee574f8d 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 352, 372, 385, 399, 414, 431, 444, 464, 472, 485} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/docs/api-reference.md b/docs/api-reference.md index f7c3641b9..3bcc33559 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -149,35 +149,6 @@ curl -XDELETE localhost:10101/index/user/frame/language {} ``` -### Change frame time quantum - -`PATCH /index//frame//time-quantum` - -Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame. - -The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are: - -* (Empty string) -* Y: year -* M: month -* D: day -* H: hour -* YM: year and month -* MD: month and day -* DH: day and hour -* YMD: year, month and day -* MDH: month, day and hour -* YMDH: year, month, day and hour - -``` request -curl localhost:10101/index/user/frame/language/time-quantum \ - -X POST \ - -d '{"timeQuantum": "YM"}' -``` -``` response -{} -``` - ### Create Field `POST /index//frame//field/` diff --git a/handler.go b/handler.go index bd97c1d1a..da0360906 100644 --- a/handler.go +++ b/handler.go @@ -157,7 +157,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore") - router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") @@ -596,46 +595,6 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { type deleteFrameResponse struct{} -// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. -func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req patchFrameTimeQuantumRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate quantum. - tq, err := ParseTimeQuantum(req.TimeQuantum) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.ModifyFrameTimeQuantum(r.Context(), indexName, frameName, tq); err != nil { - if err == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type patchFrameTimeQuantumRequest struct { - TimeQuantum string `json:"timeQuantum"` -} - -type patchFrameTimeQuantumResponse struct{} - // handlePostFrameField handles POST /frame/field request. func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] diff --git a/handler_test.go b/handler_test.go index 8c719feb1..4187b3303 100644 --- a/handler_test.go +++ b/handler_test.go @@ -748,30 +748,6 @@ func TestHandler_DeleteFrame(t *testing.T) { } } -// Ensure handler can set the frame time quantum. -func TestHandler_SetFrameTimeQuantum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - h := test.NewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected time quantum: %s", q) - } -} - // Ensure the handler can return data in differing blocks for an index. func TestHandler_Index_AttrStore_Diff(t *testing.T) { hldr := test.MustOpenHolder() From 23f5c7166bb6a1aaabe3541050c7331ab2b8d7cb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 13:23:35 -0500 Subject: [PATCH 256/311] cleat up flipBitmap and add tests --- roaring/roaring.go | 52 +++++++++++++++++++++-------- roaring/roaring_helpers_test.go | 5 ++- roaring/roaring_internal_test.go | 57 ++++++++++++++------------------ 3 files changed, 66 insertions(+), 48 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 0087e6fdf..2a8b6645c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1674,19 +1674,6 @@ func (c *container) clone() *container { return other } -// flipBitmap returns a new bitmap containter containing the inverse of all -// bits in c. -func (c *container) flipBitmap() *container { - other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - - for i, bitmap := range c.bitmap { - other.bitmap[i] = ^bitmap - } - - other.n = other.count() - return other -} - // WriteTo writes c to w. func (c *container) WriteTo(w io.Writer) (n int64, err error) { if c.isArray() { @@ -1812,6 +1799,43 @@ type ContainerInfo struct { Pointer unsafe.Pointer // offset within the mmap } +// flip returns a new container containing the inverse of all +// bits in a. +func flip(a *container) *container { + if a.isArray() { + return flipArray(a) + } else if a.isRun() { + return flipRun(a) + } else { + return flipBitmap(a) + } +} + +func flipArray(b *container) *container { + // TODO: actually implement this + x := b.clone() + x.arrayToBitmap() + return flipBitmap(x) +} + +func flipBitmap(b *container) *container { + other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + + for i, bitmap := range b.bitmap { + other.bitmap[i] = ^bitmap + } + + other.n = other.count() + return other +} + +func flipRun(b *container) *container { + // TODO: actually implement this + x := b.clone() + x.runToBitmap() + return flipBitmap(x) +} + func intersectionCount(a, b *container) int { if a.isArray() { if b.isArray() { @@ -2571,7 +2595,7 @@ RUNLOOP: func differenceRunBitmap(a, b *container) *container { // If a is full, difference is the flip of b. if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { - return b.flipBitmap() + return flipBitmap(b) } output := &container{containerType: ContainerRun} output.n = a.n diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index ea6d86cd9..417ac7db6 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -220,8 +220,11 @@ func runEvenBitsSet() []interval16 { /////////////////////////////////////////////////////////////////////////// +// f is a container function taking either one or two containers as input +// func(a *container) *container +// func(a, b *container) *container type testOp struct { - f func(a, b *container) *container + f interface{} x string y string exp string diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index dca413811..216aa7eb7 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2020,38 +2020,6 @@ func TestXorRunRun(t *testing.T) { } } -func TestBitmapFlip(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - - ttable := []struct { - original uint64 - flipped uint64 - }{ - {0x0000000000000000, 0xFFFFFFFFFFFFFFFF}, - {0xFFFFFFFFFFFFFFFF, 0x0000000000000000}, - {0xFFFFFFFFFFFFFFF0, 0x000000000000000F}, - {0xFFFFFFEFFFFFFFFF, 0x0000001000000000}, - {0x0000001000000000, 0xFFFFFFEFFFFFFFFF}, - } - - expectedN := int(65536) - for i, tt := range ttable { - c.bitmap[i] = tt.original - expectedN -= int(popcount(tt.original)) - } - - o := c.flipBitmap() - - for i, tt := range ttable { - if o.bitmap[i] != tt.flipped { - t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i]) - } - } - if o.n != expectedN { - t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n) - } -} - func TestBitmapXorRange(t *testing.T) { c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { @@ -3185,12 +3153,24 @@ func TestContainerCombinations(t *testing.T) { //{xor, "evenBitsSet", "outerBitsSet", ""}, {xor, "evenBitsSet", "oddBitsSet", "full"}, {xor, "evenBitsSet", "evenBitsSet", "empty"}, + + // flip + {flip, "empty", "", "full"}, + {flip, "full", "", "empty"}, + {flip, "firstBitSet", "", "firstBitUnset"}, + {flip, "lastBitSet", "", "lastBitUnset"}, + {flip, "firstBitUnset", "", "firstBitSet"}, + {flip, "lastBitUnset", "", "lastBitSet"}, + {flip, "innerBitsSet", "", "outerBitsSet"}, + {flip, "outerBitsSet", "", "innerBitsSet"}, + {flip, "oddBitsSet", "", "evenBitsSet"}, + {flip, "evenBitsSet", "", "oddBitsSet"}, } for _, testOp := range testOps { for _, x := range containerTypes { for _, y := range containerTypes { desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) - ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y]) + ret := runContainerFunc(testOp.f, cts[x][testOp.x], cts[y][testOp.y]) exp := testOp.exp // Convert to all container types and check result. @@ -3240,3 +3220,14 @@ func TestContainerCombinations(t *testing.T) { } } } + +//func getFunc(func(a, b *container) *container, m, n *container) *container { +func runContainerFunc(f interface{}, c ...*container) *container { + switch f.(type) { + case func(*container) *container: + return f.(func(*container) *container)(c[0]) + case func(*container, *container) *container: + return f.(func(a, b *container) *container)(c[0], c[1]) + } + return nil +} From 28acc29a109433c69927889624099c2aed740b67 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 10:37:20 -0500 Subject: [PATCH 257/311] refactoring pilosa/server trying to separate internal an external concerns in pilosa.Server - it should handle Cluster, Holder, etc. while pilosa/server handles things with external deps - e.g. Logger, Stats, Handler, etc. Using functional options in pilosa.Server now. --- broadcast_test.go | 11 +- client_test.go | 3 +- cmd/server.go | 54 +--------- cmd/server_test.go | 12 +-- ctl/common.go | 2 +- ctl/server.go | 4 - handler.go | 7 -- holder.go | 1 - holder_test.go | 5 +- server.go | 228 +++++++++++++++++++++++++---------------- server/cluster_test.go | 69 +------------ server/server.go | 215 ++++++++++++++++++++++++-------------- server/server_test.go | 6 +- test/executor.go | 3 +- test/pilosa.go | 23 ++--- test/test.go | 7 -- 16 files changed, 311 insertions(+), 339 deletions(-) diff --git a/broadcast_test.go b/broadcast_test.go index f4ff4c0fa..187e50b4e 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -15,12 +15,16 @@ package pilosa_test import ( + "bytes" "reflect" "testing" + "io/ioutil" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/server" ) // Ensure a message can be marshaled and unmarshaled. @@ -52,11 +56,12 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { - - s, err := pilosa.NewServer() + com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) + err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO if err != nil { - t.Fatalf("getting new server: %v", err) + t.Fatalf("setting up server: %v", err) } + s := com.Server sbr := NewSimpleBroadcastReceiver() sbh := NewSimpleBroadcastHandler() diff --git a/client_test.go b/client_test.go index 4bd0bade6..d11ab16a9 100644 --- a/client_test.go +++ b/client_test.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -47,7 +48,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { var defaultClient *http.Client func init() { - defaultClient = pilosa.GetHTTPClient(nil) + defaultClient = server.GetHTTPClient(nil) } diff --git a/cmd/server.go b/cmd/server.go index 81633ea01..93ecb543b 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -15,17 +15,11 @@ package cmd import ( - "fmt" "io" - "os" - "os/signal" - "runtime/pprof" - "syscall" - "time" + "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/ctl" "github.com/pilosa/pilosa/server" ) @@ -45,52 +39,10 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - // Set up the logger. - if err := Server.SetupLogger(); err != nil { - return fmt.Errorf("error setting up the logger: %v", err) - } - logger := Server.Server.Logger - logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) - - // Start CPU profiling. - if Server.CPUProfile != "" { - f, err := os.Create(Server.CPUProfile) - if err != nil { - return fmt.Errorf("create cpu profile: %v", err) - } - defer f.Close() - - fmt.Fprintln(Server.Stderr, "Starting cpu profile") - pprof.StartCPUProfile(f) - time.AfterFunc(Server.CPUTime, func() { - fmt.Fprintln(Server.Stderr, "Stopping cpu profile") - pprof.StopCPUProfile() - f.Close() - }) - } - - // Execute the program. if err := Server.Run(); err != nil { - return fmt.Errorf("error running server: %v", err) + return errors.Wrap(err, "running server") } - - // First SIGKILL causes server to shut down gracefully. - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - select { - case sig := <-c: - logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) - - // Second signal causes a hard shutdown. - go func() { <-c; os.Exit(1) }() - - if err := Server.Close(); err != nil { - return err - } - case <-Server.Done: - logger.Printf("Server closed externally") - } - return nil + return errors.Wrap(Server.Wait(), "waiting on Server") }, } diff --git a/cmd/server_test.go b/cmd/server_test.go index e69ce1134..abbe8d7a4 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -37,8 +37,6 @@ func TestServerHelp(t *testing.T) { func TestServerConfig(t *testing.T) { actualDataDir, err := ioutil.TempDir("", "") failErr(t, err, "making data dir") - profFile, err := ioutil.TempFile("", "") - failErr(t, err, "making temp file") logFile, err := ioutil.TempFile("", "") failErr(t, err, "making log file") tests := []commandTest{ @@ -93,7 +91,7 @@ func TestServerConfig(t *testing.T) { // TEST 2 { args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"}, - env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"}, + env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" data-dir = "` + actualDataDir + `" @@ -103,9 +101,6 @@ func TestServerConfig(t *testing.T) { ] [anti-entropy] interval = "11m0s" - [profile] - cpu = "` + profFile.Name() + `" - cpu-time = "35s" [metric] service = "statsd" host = "127.0.0.1:8125" @@ -114,8 +109,6 @@ func TestServerConfig(t *testing.T) { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11)) - v.Check(cmd.Server.CPUProfile, profFile.Name()) - v.Check(cmd.Server.CPUTime, time.Minute) v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") @@ -147,6 +140,9 @@ func TestServerConfig(t *testing.T) { case <-cmd.Server.Started: case <-executed: } + if execErr != nil { + t.Fatalf("executing server command: %v", execErr) + } err := cmd.Server.Close() failErr(t, err, "closing pilosa server command") <-executed diff --git a/ctl/common.go b/ctl/common.go index 6ebcd3132..3629e233e 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -49,7 +49,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig)) + client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) if err != nil { return nil, err } diff --git a/ctl/server.go b/ctl/server.go index 1c60a7c9c..4816f6170 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -61,8 +61,4 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.") flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.") - - // CPU Profiling - flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") - flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") } diff --git a/handler.go b/handler.go index bd97c1d1a..52cbff094 100644 --- a/handler.go +++ b/handler.go @@ -15,7 +15,6 @@ package pilosa import ( - "context" "encoding/json" "expvar" "fmt" @@ -35,7 +34,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -45,11 +43,6 @@ type Handler struct { FileSystem FileSystem - // The execution engine for running queries. - Executor interface { - Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) - } - Logger Logger // Keeps the query argument validators for each handler diff --git a/holder.go b/holder.go index fa93767a3..2279a3e19 100644 --- a/holder.go +++ b/holder.go @@ -530,7 +530,6 @@ func (h *Holder) setFileLimit() { func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, "ID") nodeID := "" - h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { return "", err diff --git a/holder_test.go b/holder_test.go index 8b1f6166a..7053fa1f2 100644 --- a/holder_test.go +++ b/holder_test.go @@ -25,6 +25,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -367,7 +368,7 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := test.NewCluster(2) - client := pilosa.GetHTTPClient(nil) + client := server.GetHTTPClient(nil) // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -451,7 +452,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { Holder: hldr0.Holder, Node: cluster.Nodes[0], Cluster: cluster, - RemoteClient: pilosa.GetHTTPClient(nil), + RemoteClient: server.GetHTTPClient(nil), Stats: pilosa.NopStatsClient, } diff --git a/server.go b/server.go index 6d25e3320..f022d562a 100644 --- a/server.go +++ b/server.go @@ -55,55 +55,154 @@ func OptServerLogger(l Logger) ServerOption { } } +func OptServerReplicaN(n int) ServerOption { + return func(s *Server) error { + s.Cluster.ReplicaN = n + return nil + } +} + +func OptServerDataDir(dir string) ServerOption { + return func(s *Server) error { + s.Cluster.Path = dir + s.Holder.Path = dir + return nil + } +} + +func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { + return func(s *Server) error { + s.NewAttrStore = af + s.Holder.NewAttrStore = af + return nil + } +} + +func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { + return func(s *Server) error { + s.AntiEntropyInterval = interval + return nil + } +} + +func OptServerLongQueryTime(dur time.Duration) ServerOption { + return func(s *Server) error { + s.Cluster.LongQueryTime = dur + return nil + } +} + +func OptServerHandler(h *Handler) ServerOption { + return func(s *Server) error { + s.Handler = h + return nil + } +} + +func OptServerMaxWritesPerRequest(n int) ServerOption { + return func(s *Server) error { + s.MaxWritesPerRequest = n + return nil + } +} + +func OptServerMetricInterval(dur time.Duration) ServerOption { + return func(s *Server) error { + s.MetricInterval = dur + return nil + } +} + +func OptServerSystemInfo(si SystemInfo) ServerOption { + return func(s *Server) error { + s.SystemInfo = si + return nil + } +} + +func OptServerGCNotifier(gcn GCNotifier) ServerOption { + return func(s *Server) error { + s.GCNotifier = gcn + return nil + } +} + +func OptServerRemoteClient(c *http.Client) ServerOption { + return func(s *Server) error { + s.RemoteClient = c + s.Cluster.RemoteClient = c + return nil + } +} + +func OptServerStatsClient(sc StatsClient) ServerOption { + return func(s *Server) error { + s.Holder.Stats = sc + return nil + } +} + +func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { + return func(s *Server) error { + s.DiagnosticInterval = dur + return nil + } +} + +func OptServerListener(ln net.Listener) ServerOption { + return func(s *Server) error { + s.ln = ln + + return nil + } +} + +func OptServerURI(uri *URI) ServerOption { + return func(s *Server) error { + s.URI = *uri + return nil + } +} + // Server represents a holder wrapped by a running HTTP server. type Server struct { - ln net.Listener - // Close management. wg sync.WaitGroup closing chan struct{} - // Data storage and HTTP interface. - Holder *Holder + // Internal + Holder *Holder + Cluster *Cluster + diagnostics *DiagnosticsCollector + + // External Handler *Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper RemoteClient *http.Client + SystemInfo SystemInfo + GCNotifier GCNotifier + NewAttrStore func(string) AttrStore + Logger Logger + TLS *tls.Config + ln net.Listener - // Cluster configuration. - Network string - NodeID string - URI URI - Cluster *Cluster - diagnostics *DiagnosticsCollector - SystemInfo SystemInfo - - GCNotifier GCNotifier - - NewAttrStore func(string) AttrStore - - // Background monitoring intervals. + NodeID string + URI URI AntiEntropyInterval time.Duration MetricInterval time.Duration DiagnosticInterval time.Duration - - // TLS configuration - TLS *tls.Config - - // Misc options. MaxWritesPerRequest int - Logger Logger - defaultClient InternalClient } // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ - closing: make(chan struct{}), - + closing: make(chan struct{}), + Cluster: NewCluster(), Holder: NewHolder(), Handler: NewHandler(), Broadcaster: NopBroadcaster, @@ -111,8 +210,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), SystemInfo: NewNopSystemInfo(), - Network: "tcp", - GCNotifier: NopGCNotifier, NewAttrStore: NewNopAttrStore, @@ -131,20 +228,25 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } - s.Handler.API = NewAPI() - s.Handler.API.Holder = s.Holder + s.Holder.Logger = s.Logger + s.Holder.Stats.SetLogger(s.Logger) + + s.Cluster.Logger = s.Logger + s.Cluster.Holder = s.Holder + s.Cluster.RemoteClient = s.RemoteClient + // update URI port with actual listener port. TODO this should probably be done outside of here. + if s.URI.Port() == 0 { + s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) + } return s, nil } // Open opens and initializes the server. func (s *Server) Open() error { - s.Handler.API.Logger = s.Logger // TODO do this in NewServer with functional options s.Logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { - if err := s.OpenListener(); err != nil { - return err - } + return errors.New("Must pass a listener option to NewServer") } // Get or create NodeID. @@ -181,13 +283,13 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest // Initialize HTTP handler. + s.Handler.API.Holder = s.Holder s.Handler.API.Broadcaster = s.Broadcaster s.Handler.API.BroadcastHandler = s s.Handler.API.StatusHandler = s s.Handler.API.URI = s.URI s.Handler.API.Cluster = s.Cluster s.Handler.API.Executor = e - s.Handler.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster @@ -234,43 +336,6 @@ func (s *Server) Open() error { return nil } -// OpenListener opens a listener for the Server. -func (s *Server) OpenListener() error { - s.Logger.Printf("open server listener: %s", s.URI) - if s.ln != nil { - return fmt.Errorf("a listener already exists for server: %s", s.URI) - } - - var ln net.Listener - var err error - - // If bind URI has the https scheme, enable TLS - if s.URI.Scheme() == "https" && s.TLS != nil { - ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS) - if err != nil { - return err - } - } else if s.URI.Scheme() == "http" { - // Open HTTP listener to determine port (if specified as :0). - ln, err = net.Listen(s.Network, s.URI.HostPort()) - if err != nil { - return fmt.Errorf("net.Listen: %v", err) - } - } else { - return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme()) - } - - s.ln = ln - - if s.URI.Port() == 0 { - // If the port is 0, it is set automatically. - // Find out automatically set port and update the host. - s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) - } - - return nil -} - // Close closes the server and waits for it to shutdown. func (s *Server) Close() error { // Notify goroutines to stop. @@ -311,25 +376,6 @@ func (s *Server) Addr() net.Addr { } return s.ln.Addr() } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} func (s *Server) monitorAntiEntropy() { ticker := time.NewTicker(s.AntiEntropyInterval) diff --git a/server/cluster_test.go b/server/cluster_test.go index 5b0321285..1a64ce305 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -26,81 +26,16 @@ import ( "golang.org/x/sync/errgroup" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/test" ) // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { - - m0 := test.MustRunMain() + ms := test.MustRunMainWithCluster(t, 2) + m0, m1 := ms[0], ms[1] defer m0.Close() - - m1 := test.MustRunMain() defer m1.Close() - // Update cluster config - m0.Server.Cluster.Nodes = []*pilosa.Node{ - {ID: m0.Server.NodeID, URI: m0.Server.URI}, - {ID: m1.Server.NodeID, URI: m1.Server.URI}, - } - m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes - - // Configure node0 - - // get the host portion of addr to use for binding - m0.Config.Gossip.Port = "0" - m0.Config.Gossip.Seeds = []string{} - - m0.Server.Cluster.Coordinator = m0.Server.NodeID - m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} - m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger) - gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Server.URI.Host(), m0.Config.Gossip, m0.Server) - if err != nil { - t.Fatal(err) - } - m0.Server.Cluster.MemberSet = gossipMemberSet0 - m0.Server.Broadcaster = m0.Server - m0.Server.Gossiper = gossipMemberSet0 - m0.Server.Handler.API.Broadcaster = m0.Server.Broadcaster - m0.Server.Holder.Broadcaster = m0.Server.Broadcaster - m0.Server.BroadcastReceiver = gossipMemberSet0 - - if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { - t.Fatal(err) - } - // Open Cluster management. - if err := m0.Server.Cluster.Open(); err != nil { - t.Fatal(err) - } - - // Configure node1 - - // get the host portion of addr to use for binding - m1.Config.Gossip.Port = "0" - m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()} - - m1.Server.Cluster.Coordinator = m0.Server.NodeID - m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger) - gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Server.URI.Host(), m1.Config.Gossip, m1.Server) - if err != nil { - t.Fatal(err) - } - m1.Server.Cluster.MemberSet = gossipMemberSet1 - m1.Server.Broadcaster = m1.Server - m1.Server.Gossiper = gossipMemberSet1 - m1.Server.Handler.API.Broadcaster = m1.Server.Broadcaster - m1.Server.Holder.Broadcaster = m1.Server.Broadcaster - m1.Server.BroadcastReceiver = gossipMemberSet1 - - if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { - t.Fatal(err) - } - // Open Cluster management. - if err := m1.Server.Cluster.Open(); err != nil { - t.Fatal(err) - } - m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) diff --git a/server/server.go b/server/server.go index 2606ff248..829676fd5 100644 --- a/server/server.go +++ b/server/server.go @@ -24,10 +24,14 @@ import ( "io" "log" "math/rand" + "net" + "net/http" "os" + "os/signal" "path/filepath" "strconv" "strings" + "syscall" "time" "crypto/tls" @@ -46,10 +50,10 @@ func init() { rand.Seed(time.Now().UTC().UnixNano()) } -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" -) +type loggerLogger interface { + pilosa.Logger + Logger() *log.Logger +} // Command represents the state of the pilosa server command. type Command struct { @@ -58,10 +62,6 @@ type Command struct { // Configuration. Config *Config - // Profiling options. - CPUProfile string - CPUTime time.Duration - // Gossip transport GossipTransport *gossip.Transport @@ -75,18 +75,12 @@ type Command struct { // Passed to the Gossip implementation. logOutput io.Writer - logger *log.Logger + logger loggerLogger } // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { - s, err := pilosa.NewServer() - if err != nil { - panic(err) - } - return &Command{ - Server: s, Config: NewConfig(), CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), @@ -97,7 +91,7 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { } // Run executes the pilosa server. -func (m *Command) Run(args ...string) (err error) { +func (m *Command) Run(args ...string) (err error) { // TODO args WTF defer close(m.Started) prefix := "~" + string(filepath.Separator) if strings.HasPrefix(m.Config.DataDir, prefix) { @@ -125,76 +119,72 @@ func (m *Command) Run(args ...string) (err error) { return fmt.Errorf("server.Open: %v", err) } - m.Server.Logger.Printf("Listening as %s\n", m.Server.URI) + m.logger.Printf("Listening as %s\n", m.Server.URI) + return nil } +// Wait waits for the server to be closed or interrupted. +func (m *Command) Wait() error { + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + select { + case sig := <-c: + m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + return errors.Wrap(m.Close(), "closing command") + case <-m.Done: + m.logger.Printf("Server closed externally") + return nil + } +} + // SetupLogger sets up the logger based on the configuration. -func (m *Command) SetupLogger() error { +func (m *Command) SetupLogger() (pilosa.Logger, error) { + if m.logger != nil { + return m.logger, nil + } var err error if m.Config.LogPath == "" { m.logOutput = m.Stderr } else { m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { - return err + return nil, errors.Wrap(err, "opening file") } } if m.Config.Verbose { - vbl := pilosa.NewVerboseLogger(m.logOutput) - m.logger = vbl.Logger() - m.Server.Logger = vbl + m.logger = pilosa.NewVerboseLogger(m.logOutput) } else { - sl := pilosa.NewStandardLogger(m.logOutput) - m.logger = sl.Logger() - m.Server.Logger = sl + m.logger = pilosa.NewStandardLogger(m.logOutput) } - return nil + return m.logger, nil } // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - m.Server.Handler.Logger = m.Server.Logger - m.Server.Holder.Logger = m.Server.Logger - m.Server.Holder.Stats.SetLogger(m.Server.Logger) + if m.logger == nil { + _, err := m.SetupLogger() + if err != nil { + return errors.Wrap(err, "setting up logger") + } + + } + + handler := pilosa.NewHandler() + handler.Logger = m.logger + handler.FileSystem = &statik.FileSystem{} + handler.API = pilosa.NewAPI() + handler.API.Logger = m.logger uri, err := pilosa.AddressWithDefaults(m.Config.Bind) - if err != nil { - return err + return errors.Wrap(err, "processing bind address") } - m.Server.URI = *uri - - cluster := pilosa.NewCluster() - cluster.ReplicaN = m.Config.Cluster.ReplicaN - cluster.Holder = m.Server.Holder - cluster.Logger = m.Server.Logger - - m.Server.Cluster = cluster - - // Configure data directory (for Cluster .topology) - m.Server.Cluster.Path = m.Config.DataDir - - m.Server.NewAttrStore = boltdb.NewAttrStore - m.Server.Holder.NewAttrStore = boltdb.NewAttrStore - - // Configure holder. - m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir) - m.Server.Holder.Path = m.Config.DataDir - m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) - if m.Config.Metric.Diagnostics { - m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval) - } - m.Server.SystemInfo = gopsutil.NewSystemInfo() - m.Server.GCNotifier = gcnotify.NewActiveGCNotifier() - m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) - if err != nil { - return err - } - - // Copy configuration flags. - m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest // Setup TLS var TLSConfig *tls.Config @@ -207,27 +197,73 @@ func (m *Command) SetupServer() error { } cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath) if err != nil { - return err + return errors.Wrap(err, "load x509 key pair") } - m.Server.TLS = &tls.Config{ + TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: m.Config.TLS.SkipVerify, } - - TLSConfig = m.Server.TLS } - c := pilosa.GetHTTPClient(TLSConfig) - m.Server.RemoteClient = c - m.Server.Handler.API.RemoteClient = c - m.Server.Cluster.RemoteClient = c - // Statik file system. - m.Server.Handler.FileSystem = &statik.FileSystem{} + diagnosticsInterval := time.Duration(0) + if m.Config.Metric.Diagnostics { + diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval) + } - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) - return nil + statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + if err != nil { + return errors.Wrap(err, "new stats client") + } + + ln, err := getListener(*uri, TLSConfig) + if err != nil { + return errors.Wrap(err, "getting listener") + } + + c := GetHTTPClient(TLSConfig) + handler.API.RemoteClient = c + + m.Server, err = pilosa.NewServer( + pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), + pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), + pilosa.OptServerDataDir(m.Config.DataDir), + pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN), + pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest), + pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)), + pilosa.OptServerDiagnosticsInterval(diagnosticsInterval), + + pilosa.OptServerLogger(m.logger), + pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore), + pilosa.OptServerHandler(handler), + pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), + pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), + pilosa.OptServerStatsClient(statsClient), + pilosa.OptServerListener(ln), + pilosa.OptServerURI(uri), + pilosa.OptServerRemoteClient(c), + ) + + return errors.Wrap(err, "new server") +} + +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} } // SetupNetworking sets up internode communication based on the configuration. @@ -266,7 +302,7 @@ func (m *Command) SetupNetworking() error { if m.GossipTransport != nil { transport = m.GossipTransport } else { - transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger) + transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return err } @@ -277,8 +313,8 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Coordinator = m.Server.NodeID } - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger) - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.logger) + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) if err != nil { return err } @@ -318,3 +354,24 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") } } + +// OpenListener opens a listener for the Server. +func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { + // If bind URI has the https scheme, enable TLS + if uri.Scheme() == "https" && tlsconf != nil { + ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) + if err != nil { + return nil, errors.Wrap(err, "tls.Listener") + } + } else if uri.Scheme() == "http" { + // Open HTTP listener to determine port (if specified as :0). + ln, err = net.Listen("tcp", uri.HostPort()) + if err != nil { + return nil, errors.Wrap(err, "net.Listen") + } + } else { + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + } + + return ln, nil +} diff --git a/server/server_test.go b/server/server_test.go index 532257c13..75e0e2242 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -45,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } @@ -322,11 +322,11 @@ func TestMain_FrameRestore(t *testing.T) { defer m21.Close() // Import from first cluster. - client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal("new client:", err) } - client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal("new client:", err) } diff --git a/test/executor.go b/test/executor.go index afe244ad7..8cd6391c3 100644 --- a/test/executor.go +++ b/test/executor.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" ) // Executor represents a test wrapper for pilosa.Executor. @@ -30,7 +31,7 @@ type Executor struct { var remoteClient *http.Client func init() { - remoteClient = pilosa.GetHTTPClient(nil) + remoteClient = server.GetHTTPClient(nil) } // NewExecutor returns a new instance of Executor. diff --git a/test/pilosa.go b/test/pilosa.go index b00c9841c..366facdd4 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -49,15 +49,13 @@ func NewMain() *Main { } m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} - m.Server.Network = *Network - m.Server.NewAttrStore = NewAttrStore - m.Server.Holder.NewAttrStore = NewAttrStore m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr + m.SetupServer() if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) @@ -101,6 +99,7 @@ func runMainWithCluster(size int) ([]*Main, error) { for i := 0; i < size; i++ { m := NewMainWithCluster(i == 0) + m.Config.Cluster.Disabled = false gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) if err != nil { @@ -136,12 +135,16 @@ func (m *Main) Reopen() error { } // Create new main with the same config. - config := m.Config + config := m.Command.Config m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) - m.Server.Network = *Network + m.Command.Config = config + err := m.SetupServer() + if err != nil { + return errors.Wrap(err, "setting up server") + } + m.Server.NewAttrStore = boltdb.NewAttrStore m.Server.Holder.NewAttrStore = m.Server.NewAttrStore - m.Config = config // Run new program. if err := m.Run(); err != nil { @@ -174,12 +177,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( return seed, err } - // Open server listener. - err = m.Server.OpenListener() - if err != nil { - return seed, err - } - // Open gossip transport to use in SetupServer. transport, err := gossip.NewTransport(host, bindPort, nil) if err != nil { @@ -221,7 +218,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { panic(err) } diff --git a/test/test.go b/test/test.go index c83b657c0..01f980ad2 100644 --- a/test/test.go +++ b/test/test.go @@ -13,10 +13,3 @@ // limitations under the License. package test - -import "flag" - -// Test flags. -var ( - Network = flag.String("network", "tcp", "network name") -) From 9f1720f01d3e73989f7dde42b9c187bf25486b36 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 15:12:23 -0500 Subject: [PATCH 258/311] unexport stuff in pilosa.Server refactor gossip.NewGossipMemberset to not take Server --- diagnostics.go | 16 ++--- gossip/gossip.go | 15 ++-- server.go | 182 +++++++++++++++++++++++------------------------ server/server.go | 5 +- 4 files changed, 107 insertions(+), 111 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 57423d0a5..bfebbb495 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -168,23 +168,23 @@ func (d *DiagnosticsCollector) logErr(err error) bool { // EnrichWithOSInfo adds OS information to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithOSInfo() { - uptime, err := d.server.SystemInfo.Uptime() + uptime, err := d.server.systemInfo.Uptime() if !d.logErr(err) { d.Set("HostUptime", uptime) } - platform, err := d.server.SystemInfo.Platform() + platform, err := d.server.systemInfo.Platform() if !d.logErr(err) { d.Set("OSPlatform", platform) } - family, err := d.server.SystemInfo.Family() + family, err := d.server.systemInfo.Family() if !d.logErr(err) { d.Set("OSFamily", family) } - version, err := d.server.SystemInfo.OSVersion() + version, err := d.server.systemInfo.OSVersion() if !d.logErr(err) { d.Set("OSVersion", version) } - kernelVersion, err := d.server.SystemInfo.KernelVersion() + kernelVersion, err := d.server.systemInfo.KernelVersion() if !d.logErr(err) { d.Set("OSKernelVersion", kernelVersion) } @@ -192,15 +192,15 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() { // EnrichWithMemoryInfo adds memory information to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { - memFree, err := d.server.SystemInfo.MemFree() + memFree, err := d.server.systemInfo.MemFree() if !d.logErr(err) { d.Set("MemFree", memFree) } - memTotal, err := d.server.SystemInfo.MemTotal() + memTotal, err := d.server.systemInfo.MemTotal() if !d.logErr(err) { d.Set("MemTotal", memTotal) } - memUsed, err := d.server.SystemInfo.MemUsed() + memUsed, err := d.server.systemInfo.MemUsed() if !d.logErr(err) { d.Set("MemUsed", memUsed) } diff --git a/gossip/gossip.go b/gossip/gossip.go index 8ad00a8a7..e4856ec7b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -152,7 +152,7 @@ type gossipConfig struct { type GossipMemberSetOption func(*GossipMemberSet) error // WithTransport is a functional option for providing a transport to NewGossipMemberSet. -func WithTransport(transport *Transport) func(*GossipMemberSet) error { +func WithTransport(transport *Transport) GossipMemberSetOption { return func(g *GossipMemberSet) error { g.transport = transport return nil @@ -160,7 +160,7 @@ func WithTransport(transport *Transport) func(*GossipMemberSet) error { } // WithLogger is a functional option for providing a logger to NewGossipMemberSet. -func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { +func WithLogger(logger *log.Logger) GossipMemberSetOption { return func(g *GossipMemberSet) error { g.logger = logger return nil @@ -168,11 +168,8 @@ func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - - g := &GossipMemberSet{ - Logger: server.Logger, - } +func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) { + g := &GossipMemberSet{} // options for _, opt := range options { @@ -227,7 +224,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Ser // conf.Delegate = g conf.SecretKey = gossipKey - conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) + conf.Events = ger conf.Logger = g.logger g.config = &gossipConfig{ @@ -235,7 +232,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Ser gossipSeeds: cfg.Seeds, } - g.statusHandler = server + g.statusHandler = sh return g, nil } diff --git a/server.go b/server.go index f022d562a..d302a7d97 100644 --- a/server.go +++ b/server.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "crypto/tls" "fmt" "net" "net/http" @@ -45,12 +44,45 @@ var _ Broadcaster = &Server{} var _ BroadcastHandler = &Server{} var _ StatusHandler = &Server{} +// Server represents a holder wrapped by a running HTTP server. +type Server struct { + // Close management. + wg sync.WaitGroup + closing chan struct{} + + // Internal + Holder *Holder + Cluster *Cluster + diagnostics *DiagnosticsCollector + + // External + handler *Handler + Broadcaster Broadcaster + BroadcastReceiver BroadcastReceiver + Gossiper Gossiper + remoteClient *http.Client + systemInfo SystemInfo + gcNotifier GCNotifier + NewAttrStore func(string) AttrStore + logger Logger + ln net.Listener + + NodeID string + URI URI + antiEntropyInterval time.Duration + metricInterval time.Duration + diagnosticInterval time.Duration + maxWritesPerRequest int + + defaultClient InternalClient +} + // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error func OptServerLogger(l Logger) ServerOption { return func(s *Server) error { - s.Logger = l + s.logger = l return nil } } @@ -80,7 +112,7 @@ func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { return func(s *Server) error { - s.AntiEntropyInterval = interval + s.antiEntropyInterval = interval return nil } } @@ -94,42 +126,42 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { func OptServerHandler(h *Handler) ServerOption { return func(s *Server) error { - s.Handler = h + s.handler = h return nil } } func OptServerMaxWritesPerRequest(n int) ServerOption { return func(s *Server) error { - s.MaxWritesPerRequest = n + s.maxWritesPerRequest = n return nil } } func OptServerMetricInterval(dur time.Duration) ServerOption { return func(s *Server) error { - s.MetricInterval = dur + s.metricInterval = dur return nil } } func OptServerSystemInfo(si SystemInfo) ServerOption { return func(s *Server) error { - s.SystemInfo = si + s.systemInfo = si return nil } } func OptServerGCNotifier(gcn GCNotifier) ServerOption { return func(s *Server) error { - s.GCNotifier = gcn + s.gcNotifier = gcn return nil } } func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { - s.RemoteClient = c + s.remoteClient = c s.Cluster.RemoteClient = c return nil } @@ -144,7 +176,7 @@ func OptServerStatsClient(sc StatsClient) ServerOption { func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { return func(s *Server) error { - s.DiagnosticInterval = dur + s.diagnosticInterval = dur return nil } } @@ -164,61 +196,27 @@ func OptServerURI(uri *URI) ServerOption { } } -// Server represents a holder wrapped by a running HTTP server. -type Server struct { - // Close management. - wg sync.WaitGroup - closing chan struct{} - - // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - - // External - Handler *Handler - Broadcaster Broadcaster - BroadcastReceiver BroadcastReceiver - Gossiper Gossiper - RemoteClient *http.Client - SystemInfo SystemInfo - GCNotifier GCNotifier - NewAttrStore func(string) AttrStore - Logger Logger - TLS *tls.Config - ln net.Listener - - NodeID string - URI URI - AntiEntropyInterval time.Duration - MetricInterval time.Duration - DiagnosticInterval time.Duration - MaxWritesPerRequest int - - defaultClient InternalClient -} - // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ closing: make(chan struct{}), Cluster: NewCluster(), Holder: NewHolder(), - Handler: NewHandler(), + handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), - SystemInfo: NewNopSystemInfo(), + systemInfo: NewNopSystemInfo(), - GCNotifier: NopGCNotifier, + gcNotifier: NopGCNotifier, NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: time.Minute * 10, - MetricInterval: 0, - DiagnosticInterval: 0, + antiEntropyInterval: time.Minute * 10, + metricInterval: 0, + diagnosticInterval: 0, - Logger: NopLogger, + logger: NopLogger, } for _, opt := range opts { @@ -228,12 +226,12 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } - s.Holder.Logger = s.Logger - s.Holder.Stats.SetLogger(s.Logger) + s.Holder.Logger = s.logger + s.Holder.Stats.SetLogger(s.logger) - s.Cluster.Logger = s.Logger + s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder - s.Cluster.RemoteClient = s.RemoteClient + s.Cluster.RemoteClient = s.remoteClient // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) @@ -243,7 +241,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Open opens and initializes the server. func (s *Server) Open() error { - s.Logger.Printf("open server") + s.logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { return errors.New("Must pass a listener option to NewServer") @@ -269,36 +267,36 @@ func (s *Server) Open() error { s.Holder.Peek() // Create default HTTP client - s.createDefaultClient(s.RemoteClient) + s.createDefaultClient(s.remoteClient) // Create executor for executing queries. - e := NewExecutor(s.RemoteClient) + e := NewExecutor(s.remoteClient) e.Holder = s.Holder e.Node = node e.Cluster = s.Cluster - e.MaxWritesPerRequest = s.MaxWritesPerRequest + e.MaxWritesPerRequest = s.maxWritesPerRequest // Cluster settings. s.Cluster.Broadcaster = s.Broadcaster - s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest + s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest // Initialize HTTP handler. - s.Handler.API.Holder = s.Holder - s.Handler.API.Broadcaster = s.Broadcaster - s.Handler.API.BroadcastHandler = s - s.Handler.API.StatusHandler = s - s.Handler.API.URI = s.URI - s.Handler.API.Cluster = s.Cluster - s.Handler.API.Executor = e + s.handler.API.Holder = s.Holder + s.handler.API.Broadcaster = s.Broadcaster + s.handler.API.BroadcastHandler = s + s.handler.API.StatusHandler = s + s.handler.API.URI = s.URI + s.handler.API.Cluster = s.Cluster + s.handler.API.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster // Serve HTTP. go func() { - err := http.Serve(s.ln, s.Handler) + err := http.Serve(s.ln, s.handler) if err != nil { - s.Logger.Printf("HTTP handler terminated with error: %s\n", err) + s.logger.Printf("HTTP handler terminated with error: %s\n", err) } }() @@ -363,7 +361,7 @@ func (s *Server) LoadNodeID() string { } nodeID, err := s.Holder.loadNodeID() if err != nil { - s.Logger.Printf("loading NodeID: %v", err) + s.logger.Printf("loading NodeID: %v", err) return s.NodeID } return nodeID @@ -378,10 +376,10 @@ func (s *Server) Addr() net.Addr { } func (s *Server) monitorAntiEntropy() { - ticker := time.NewTicker(s.AntiEntropyInterval) + ticker := time.NewTicker(s.antiEntropyInterval) defer ticker.Stop() - s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval) for { // Wait for tick or a close. @@ -392,7 +390,7 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.Logger.Printf("holder sync beginning") + s.logger.Printf("holder sync beginning") // Initialize syncer with local holder and remote client. var syncer HolderSyncer @@ -400,17 +398,17 @@ func (s *Server) monitorAntiEntropy() { syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.RemoteClient = s.RemoteClient + syncer.RemoteClient = s.remoteClient syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. if err := syncer.SyncHolder(); err != nil { - s.Logger.Printf("holder sync error: err=%s", err) + s.logger.Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.Logger.Printf("holder sync complete") + s.logger.Printf("holder sync complete") dif := time.Since(t) s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } @@ -532,7 +530,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { - s.Logger.Printf("SendSync to: %s", node.URI) + s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.URI == node.URI { continue @@ -554,7 +552,7 @@ func (s *Server) SendAsync(pb proto.Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, pb proto.Message) error { - s.Logger.Printf("SendTo: %s", to.URI) + s.logger.Printf("SendTo: %s", to.URI) ctx := context.WithValue(context.Background(), "uri", &to.URI) return s.defaultClient.SendMessage(ctx, pb) } @@ -604,7 +602,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) if err != nil { - s.Logger.Printf("merge remote status: %s", err) + s.logger.Printf("merge remote status: %s", err) } }() @@ -629,7 +627,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger.Printf("Local Index not found: %s", index) + s.logger.Printf("Local Index not found: %s", index) continue } if newMax > oldmaxslices[index] { @@ -645,7 +643,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger.Printf("Local Index not found: %s", index) + s.logger.Printf("Local Index not found: %s", index) continue } if newMaxInverse > oldMaxInverseSlices[index] { @@ -660,14 +658,14 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // monitorDiagnostics periodically polls the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { // Do not send more than once a minute - if s.DiagnosticInterval < time.Minute { - s.Logger.Printf("diagnostics disabled") + if s.diagnosticInterval < time.Minute { + s.logger.Printf("diagnostics disabled") return } else { - s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) + s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) } - s.diagnostics.Logger = s.Logger + s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) @@ -689,11 +687,11 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.CheckVersion() err = s.diagnostics.Flush() if err != nil { - s.Logger.Printf("Diagnostics error: %s", err) + s.logger.Printf("Diagnostics error: %s", err) } } - ticker := time.NewTicker(s.DiagnosticInterval) + ticker := time.NewTicker(s.diagnosticInterval) defer ticker.Stop() flush() for { @@ -710,24 +708,24 @@ func (s *Server) monitorDiagnostics() { // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { // Disable metrics when poll interval is zero. - if s.MetricInterval <= 0 { + if s.metricInterval <= 0 { return } var m runtime.MemStats - ticker := time.NewTicker(s.MetricInterval) + ticker := time.NewTicker(s.metricInterval) defer ticker.Stop() - defer s.GCNotifier.Close() + defer s.gcNotifier.Close() - s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval) + s.logger.Printf("runtime stats initializing (%s interval)", s.metricInterval) for { // Wait for tick or a close. select { case <-s.closing: return - case <-s.GCNotifier.AfterGC(): + case <-s.gcNotifier.AfterGC(): // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: diff --git a/server/server.go b/server/server.go index 829676fd5..3e5bb1899 100644 --- a/server/server.go +++ b/server/server.go @@ -313,8 +313,9 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Coordinator = m.Server.NodeID } - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.logger) - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) + gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) + m.Server.Cluster.EventReceiver = gossipEventReceiver + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) if err != nil { return err } From 1e05a6d62725b0e261f09e32ddd4322b6a6ace34 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 15:15:02 -0500 Subject: [PATCH 259/311] fix getListener comment --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 3e5bb1899..1c3ba577c 100644 --- a/server/server.go +++ b/server/server.go @@ -356,7 +356,7 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { } } -// OpenListener opens a listener for the Server. +// getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS if uri.Scheme() == "https" && tlsconf != nil { From 9deefbaefc00366008b2ef25aa7cb880b3497da3 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 16:25:03 -0500 Subject: [PATCH 260/311] unexport setupLogger and simplify --- server/server.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/server/server.go b/server/server.go index 1c3ba577c..6ff80fe7f 100644 --- a/server/server.go +++ b/server/server.go @@ -142,18 +142,15 @@ func (m *Command) Wait() error { } } -// SetupLogger sets up the logger based on the configuration. -func (m *Command) SetupLogger() (pilosa.Logger, error) { - if m.logger != nil { - return m.logger, nil - } +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { var err error if m.Config.LogPath == "" { m.logOutput = m.Stderr } else { m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { - return nil, errors.Wrap(err, "opening file") + return errors.Wrap(err, "opening file") } } @@ -162,17 +159,14 @@ func (m *Command) SetupLogger() (pilosa.Logger, error) { } else { m.logger = pilosa.NewStandardLogger(m.logOutput) } - return m.logger, nil + return nil } // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - if m.logger == nil { - _, err := m.SetupLogger() - if err != nil { - return errors.Wrap(err, "setting up logger") - } - + err := m.setupLogger() + if err != nil { + return errors.Wrap(err, "setting up logger") } handler := pilosa.NewHandler() From f99479932db488f42c3f61efc37eff5d7c0608fd Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 16:28:24 -0500 Subject: [PATCH 261/311] rename server Run to Start to better reflect functionality --- cmd/server.go | 2 +- server/server.go | 4 ++-- test/pilosa.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 93ecb543b..d906cf189 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -39,7 +39,7 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - if err := Server.Run(); err != nil { + if err := Server.Start(); err != nil { return errors.Wrap(err, "running server") } return errors.Wrap(Server.Wait(), "waiting on Server") diff --git a/server/server.go b/server/server.go index 6ff80fe7f..7f5782211 100644 --- a/server/server.go +++ b/server/server.go @@ -90,8 +90,8 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { } } -// Run executes the pilosa server. -func (m *Command) Run(args ...string) (err error) { // TODO args WTF +// Start starts the pilosa server - it returns once the server is running. +func (m *Command) Start() (err error) { defer close(m.Started) prefix := "~" + string(filepath.Separator) if strings.HasPrefix(m.Config.DataDir, prefix) { diff --git a/test/pilosa.go b/test/pilosa.go index 366facdd4..a390cd462 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -116,7 +116,7 @@ func runMainWithCluster(size int) ([]*Main, error) { func MustRunMain() *Main { m := NewMain() m.Config.Metric.Diagnostics = false // Disable diagnostics. - if err := m.Run(); err != nil { + if err := m.Start(); err != nil { panic(err) } return m @@ -147,7 +147,7 @@ func (m *Main) Reopen() error { m.Server.Holder.NewAttrStore = m.Server.NewAttrStore // Run new program. - if err := m.Run(); err != nil { + if err := m.Start(); err != nil { return err } return nil From e4a7e2edbd1e22b050f8a584e8936b8044cbfb93 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 16:47:22 -0500 Subject: [PATCH 262/311] remove references to row and column labels from the docs --- docs/api-reference.md | 4 +-- docs/data-model.md | 18 ++++++------- docs/pdk.md | 2 +- docs/query-language.md | 58 +++++++++++++++++++++--------------------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index a2697f311..9fc232535 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -68,7 +68,7 @@ Sends a query to the Pilosa server with the given index. The request body is UTF ``` request curl localhost:10101/index/user/query \ -X POST \ - -d 'Bitmap(frame="language", rowID=5)' + -d 'Bitmap(frame="language", row=5)' ``` ``` response {"results":[{"attrs":{},"bits":[100]}]} @@ -83,7 +83,7 @@ The query is executed for all [slices](../data-model/#slice) by default. To use ``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ - -d 'Bitmap(frame="language", rowID=5)' + -d 'Bitmap(frame="language", row=5)' ``` ``` response { diff --git a/docs/data-model.md b/docs/data-model.md index dc8646a85..693b4c97d 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -88,8 +88,8 @@ The standard View contains the same Row/Column format as the input data. If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: ``` -SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00") -SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") +SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00") +SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") ``` ![time quantum frame diagram](/img/docs/frame-time-quantum.svg) @@ -100,17 +100,17 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rows' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `SetFieldValue()` queries will result in the data described in the illustration below: ``` -SetFieldValue(columnID=1, frame="A", field0=1) -SetFieldValue(columnID=2, frame="A", field0=2) -SetFieldValue(columnID=3, frame="A", field0=3) -SetFieldValue(columnID=4, frame="A", field0=7) -SetFieldValue(columnID=2, frame="A", field1=1) -SetFieldValue(columnID=3, frame="A", field1=6) +SetFieldValue(col=1, frame="A", field0=1) +SetFieldValue(col=2, frame="A", field0=2) +SetFieldValue(col=3, frame="A", field0=3) +SetFieldValue(col=4, frame="A", field0=7) +SetFieldValue(col=2, frame="A", field1=1) +SetFieldValue(col=3, frame="A", field1=6) ``` ![BSI frame diagram](/img/docs/frame-bsi.svg) diff --git a/docs/pdk.md b/docs/pdk.md index 8c02dc279..9be1d70be 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -34,7 +34,7 @@ With this definition available, the PDK tool can run the import, which consists - for each CSV record: - generate a columnID - apply all ParserMappers, generating a list of (frame, ID) pairs - - set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID) + - set the appropriate bit. schematically: SetBit(row=rowID, frame=frame, col=columnID) The process is summarized in this flowchart: diff --git a/docs/query-language.md b/docs/query-language.md index f26cf021d..83a10ab36 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -25,17 +25,17 @@ There will be one item in the `results` array for each PQL query in the request. * Angle Brackets `<>` denote required arguments * Square Brackets `[]` denote optional arguments -* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`) +* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`) ##### Examples Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data. -The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: +The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", columnID=10, rowID=1)' + -d 'SetBit(frame="stargazer", col=10, row=1)' ``` ``` response {"results":[true]} @@ -58,7 +58,7 @@ curl localhost:10101/index/repository/query \ **Spec:** ``` -SetBit(, , , +SetBit(, , , [timestamp=TIMESTAMP]) ``` @@ -76,26 +76,26 @@ A return value of `false` indicates that the bit was already set to 1 and nothin **Examples:** ``` -SetBit(frame="stargazer", columnID=10, rowID=1) +SetBit(frame="stargazer", col=10, row=1) ``` This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10. SetBit also supports providing a timestamp. To write the date that a user starred a repository. ``` -SetBit(frame="stargazer", columnID=10, rowID=1, timestamp="2016-01-01T00:00") +SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") ``` Setting multiple bits in a single request: ``` -SetBit(frame="stargazer", columnID=10, rowID=1) SetBit(frame="stargazer", columnID=10, rowID=2) SetBit(frame="stargazer", columnID=20, rowID=1) SetBit(frame="stargazer", columnID=30, rowID=2) +SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2) ``` #### SetRowAttrs **Spec:** ``` -SetRowAttrs(, , +SetRowAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -111,13 +111,13 @@ SetRowAttrs queries always return `null` upon success. **Examples:** ``` -SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true) +SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true) ``` Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. ``` -SetRowAttrs(frame="stargazer", rowID=10, username=null) +SetRowAttrs(frame="stargazer", row=10, username=null) ``` Delete username value for user 10. @@ -127,7 +127,7 @@ Delete username value for user 10. **Spec:** ``` -SetColumnAttrs(, , +SetColumnAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -143,13 +143,13 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu **Examples:** ``` -SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true) +SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` -Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`. +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", col=10)`. ``` -SetColumnAttrs(columnID=10, url=null) +SetColumnAttrs(col=10, url=null) ``` Delete url value for repo 10. @@ -160,7 +160,7 @@ Delete url value for repo 10. **Spec:** ``` -SetBit(, , , +SetBit(, , , [timestamp=TIMESTAMP]) ``` @@ -177,7 +177,7 @@ A return value of `false` indicates that the bit was already set to 0 and nothin **Examples:** ``` -ClearBit(frame="stargazer", columnID=10, rowID=1) +ClearBit(frame="stargazer", col=10, row=1) ``` Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame. @@ -188,12 +188,12 @@ Remove relationship between the stargazer in row 1 and the repository in column **Spec:** ``` -SetFieldValue(, , ) +SetFieldValue(, , ) ``` **Description:** -`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`. +`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`. **Result Type:** null @@ -203,7 +203,7 @@ SetFieldValue returns `null` upon success. Set the number of pull requests of repository 10. ``` -SetFieldValue(columnID=10, frame="stats", pullrequests=2) +SetFieldValue(col=10, frame="stats", pullrequests=2) ``` @@ -214,7 +214,7 @@ SetFieldValue(columnID=10, frame="stats", pullrequests=2) **Spec:** ``` -Bitmap(, ( | =UINT)) +Bitmap(, ( | =UINT)) ``` **Description:** @@ -229,7 +229,7 @@ e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` Query all repositories that user 1 has starred. ``` -Bitmap(frame="stargazer", rowID=1) +Bitmap(frame="stargazer", row=1) ``` Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` @@ -286,7 +286,7 @@ attrs will always be empty Query repositories which have been starred by two users. ``` -Intersect(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2)) +Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) ``` Returns `{"attrs":{},"bits":[10]}`. @@ -313,7 +313,7 @@ attrs will always be empty Query repositories which have been starred by one user and not another. ``` -Difference(Bitmap(frame="stargazer", rowID=1), Bitmap( frame="stargazer", rowID=2)) +Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2)) ``` Return `{"results":[{"attrs":{},"bits":[20]}]}` @@ -321,7 +321,7 @@ Return `{"results":[{"attrs":{},"bits":[20]}]}` * bits are repositories that were starred by user 1 BUT NOT user 2 ``` -Difference(Bitmap(frame="stargazer", rowID=2), Bitmap( frame="stargazer", rowID=1)) +Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1)) ``` Return `{"attrs":{},"bits":[30]}` @@ -349,7 +349,7 @@ attrs will always be empty Query repositories which have been starred by two users. ``` -Xor(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2)) +Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) ``` Returns `{"attrs":{},"bits":[30]}`. @@ -373,7 +373,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in. Query the number of repositories to which a user has contributed. ``` -Count(Bitmap(frame="stargazer", rowID=1)) +Count(Bitmap(frame="stargazer", row=1)) ``` Return `2` @@ -427,7 +427,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]` * Results are the top two users sorted by number of repositories they've starred in descending order. ``` -TopN(Bitmap(frame="language", rowID=1), frame="stargazer", n=2) +TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2) ``` Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` @@ -439,7 +439,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` **Spec:** ``` -Range(, , +Range(, , , ) ``` @@ -455,7 +455,7 @@ between the given `start` and `end` timestamps. When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range. ``` -Range(frame="stargazer", rowID=1, start="2010-01-01T00:00", end="2017-03-02T03:00") +Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00") ``` Returns `{{"attrs":{},"bits":[10]}` From 7ecd2ec79468ecd89ce5f76aa3378494c888dce0 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 17:10:48 -0500 Subject: [PATCH 263/311] adjust references to RowID and ColumnID in the docs --- docs/administration.md | 6 +++--- docs/data-model.md | 2 +- docs/glossary.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index c9d932df0..190b5ca1f 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -48,9 +48,9 @@ On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog #### Importing -The import API expects a csv of RowID,ColumnID's. +The import API expects a csv of rowID,columnID's. -When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. +When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. ``` pilosa import --sort -i project -f stargazer project-stargazer.csv @@ -70,7 +70,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count #### Exporting -Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. +Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format rowID,columnID and sorted by columnID. ``` curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \ --header "Accept: text/csv" diff --git a/docs/data-model.md b/docs/data-model.md index 693b4c97d..98750eda8 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -100,7 +100,7 @@ SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rows' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `SetFieldValue()` queries will result in the data described in the illustration below: diff --git a/docs/glossary.md b/docs/glossary.md index d222cb511..bd9293c3d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -24,7 +24,7 @@ nav = [] Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). -[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. +[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. [Index](../data-model/#index): An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes. @@ -62,6 +62,6 @@ nav = [] [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). -[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). [View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. From ff8ab6930a66bfe2a55ce97484c7aea808db2189 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 24 Apr 2018 16:13:36 +0300 Subject: [PATCH 264/311] Run 32bits on CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 15c511e78..85e721329 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: install: - make install-dep install-statik vendor generate-statik script: + - GOARCH=386 make test - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 after_success: From f9ff20689a29b631f131c4eefc018d5d5f177f7a Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 24 Apr 2018 08:44:58 -0500 Subject: [PATCH 265/311] remove holder.Peek, combine with HasData, move server logic Server initializing happens more in NewServer than Open now - expecting to continue this trend. goal was to remove remoteClient from Server (since it has a defaultClient) as well, but we'll have to refactor the client usage in fragment and frame first. --- cluster.go | 19 ++++++++++---- holder.go | 68 +++++++++++++++++++++++------------------------- holder_test.go | 29 ++++++++------------- server.go | 49 +++++++++++++++------------------- server/server.go | 1 + 5 files changed, 79 insertions(+), 87 deletions(-) diff --git a/cluster.go b/cluster.go index be668a6bc..bbf6f799e 100644 --- a/cluster.go +++ b/cluster.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "encoding/binary" - "errors" "fmt" "hash/fnv" "io/ioutil" @@ -33,6 +32,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -899,7 +899,10 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. - c.AddNode(c.Node) + err := c.AddNode(c.Node) + if err != nil { + return errors.Wrap(err, "adding local node") + } // Start the EventReceiver. if err := c.EventReceiver.Start(c); err != nil { @@ -1684,13 +1687,15 @@ func (c *Cluster) nodeJoin(node *Node) error { // Only change to normal if there is no existing data. Otherwise, // the coordinator needs to wait to receive READY messages (nodeStates) // from remote nodes before setting the cluster to state NORMAL. - if !c.Holder.HasData() { + if ok, err := c.Holder.HasData(); !ok && err == nil { // If the result of the previous AddNode completed the joining of nodes // in the topology, then change the state to NORMAL. if c.haveTopologyAgreement() { return c.setStateAndBroadcast(ClusterStateNormal) } return nil + } else if err != nil { + return errors.Wrap(err, "checking if holder has data") } if c.haveTopologyAgreement() && c.allNodesReady() { @@ -1712,11 +1717,13 @@ func (c *Cluster) nodeJoin(node *Node) error { } // If the holder does not yet contain data, go ahead and add the node. - if !c.Holder.HasData() { + if ok, err := c.Holder.HasData(); !ok && err == nil { if err := c.AddNode(node); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) + } else if err != nil { + return errors.Wrap(err, "checking if holder has data2") } // If the cluster has data, we need to change to RESIZING and @@ -1770,11 +1777,13 @@ func (c *Cluster) nodeLeave(node *Node) error { } // If the holder does not yet contain data, go ahead and remove the node. - if !c.Holder.HasData() { + if ok, err := c.Holder.HasData(); !ok && err == nil { if err := c.RemoveNode(n); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) + } else if err != nil { + return errors.Wrap(err, "checking if holder has data") } // If the cluster has data then change state to RESIZING and diff --git a/holder.go b/holder.go index 2279a3e19..7649caeba 100644 --- a/holder.go +++ b/holder.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "errors" "fmt" "io/ioutil" "net/http" @@ -30,6 +29,7 @@ import ( "time" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -47,7 +47,6 @@ type Holder struct { // Indexes by name. indexes map[string]*Index - hasData bool // opened channel is closed once Open() completes. opened chan struct{} @@ -91,36 +90,6 @@ func NewHolder() *Holder { } } -// Peek reads the root data directory for the holder -// without actually loading any data into memory. -// HasData is returned, and h.hasData is set. -func (h *Holder) Peek() bool { - h.Logger.Printf("peek at holder path: %s", h.Path) - h.hasData = false - - // Open path to read all index directories. - f, err := os.Open(h.Path) - if err != nil { - return false - } - defer f.Close() - - fis, err := f.Readdir(0) - if err != nil { - return false - } - - for _, fi := range fis { - if !fi.IsDir() { - continue - } - h.hasData = true - break - } - - return h.hasData -} - // Open initializes the root data directory for the holder. func (h *Holder) Open() error { h.setFileLimit() @@ -198,10 +167,37 @@ func (h *Holder) Close() error { // HasData returns true if Holder contains at least one index. // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. -func (h *Holder) HasData() bool { - h.mu.RLock() - defer h.mu.RUnlock() - return h.hasData || len(h.indexes) > 0 +func (h *Holder) HasData() (bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.indexes) > 0 { + return true, nil + } + // Open path to read all index directories. + if _, err := os.Stat(h.Path); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, errors.Wrap(err, "statting data dir") + } + + f, err := os.Open(h.Path) + if err != nil { + return false, errors.Wrap(err, "opening data dir") + } + defer f.Close() + + fis, err := f.Readdir(0) + if err != nil { + return false, errors.Wrap(err, "reading data dir") + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + return true, nil + } + return false, nil } // MaxSlices returns MaxSlice map for all indexes. diff --git a/holder_test.go b/holder_test.go index 7053fa1f2..e819c6e94 100644 --- a/holder_test.go +++ b/holder_test.go @@ -283,26 +283,24 @@ func TestHolder_HasData(t *testing.T) { h := test.MustOpenHolder() defer h.Close() - if h.HasData() { - t.Fatal("expected HasData to return false") + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) } if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } - if !h.HasData() { - t.Fatal("expected HasData to return true") + if ok, err := h.HasData(); !ok || err != nil { + t.Fatal("expected HasData to return true, but ", ok, err) } }) t.Run("Peek", func(t *testing.T) { h := test.NewHolder() - if hasData := h.Peek(); hasData != false { - t.Fatal("expected Peek to return false") - } else if h.HasData() { - t.Fatal("expected HasData to return false") + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) } // Create an index directory to indicate data exists. @@ -310,24 +308,19 @@ func TestHolder_HasData(t *testing.T) { t.Fatal(err) } - if hasData := h.Peek(); hasData != true { - t.Fatal("expected Peek to return true") - } else if !h.HasData() { - t.Fatal("expected HasData to return true") + if ok, err := h.HasData(); !ok || err != nil { + t.Fatal("expected HasData to return true, no err, but", ok, err) } }) t.Run("Peek at missing directory", func(t *testing.T) { h := test.NewHolder() - // Ensure that hasData is false when trying to peek into - // a directory that doesn't exist. + // Ensure that hasData is false when dir doesn't exist. h.Path = "bad-path" - if hasData := h.Peek(); hasData != false { - t.Fatal("expected Peek to return false") - } else if h.HasData() { - t.Fatal("expected HasData to return false") + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) } }) } diff --git a/server.go b/server.go index d302a7d97..2b7903fb6 100644 --- a/server.go +++ b/server.go @@ -54,6 +54,7 @@ type Server struct { Holder *Holder Cluster *Cluster diagnostics *DiagnosticsCollector + executor *Executor // External handler *Handler @@ -161,7 +162,8 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { - s.remoteClient = c + s.executor = NewExecutor(c) + s.defaultClient = NewInternalHTTPClientFromURI(nil, c) s.Cluster.RemoteClient = c return nil } @@ -231,11 +233,28 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder - s.Cluster.RemoteClient = s.remoteClient + // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) } + + s.NodeID = s.LoadNodeID() + // Set Cluster Node. + node := &Node{ + ID: s.NodeID, + URI: s.URI, + IsCoordinator: s.Cluster.Coordinator == s.NodeID, + } + s.Cluster.Node = node + s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) + + s.executor.Holder = s.Holder + s.executor.Node = node + s.executor.Cluster = s.Cluster + s.executor.MaxWritesPerRequest = s.maxWritesPerRequest + s.handler.API.Executor = s.executor + return s, nil } @@ -248,33 +267,12 @@ func (s *Server) Open() error { } // Get or create NodeID. - s.NodeID = s.LoadNodeID() - - // Set Cluster Node. - node := &Node{ - ID: s.NodeID, - URI: s.URI, - IsCoordinator: s.Cluster.Coordinator == s.NodeID, - } - s.Cluster.Node = node // Append the NodeID tag to stats. - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) - - // Peek at the holder to determine if there is data on disk. - // Don't actually load the data until after the Cluster - // management starts. - s.Holder.Peek() // Create default HTTP client - s.createDefaultClient(s.remoteClient) // Create executor for executing queries. - e := NewExecutor(s.remoteClient) - e.Holder = s.Holder - e.Node = node - e.Cluster = s.Cluster - e.MaxWritesPerRequest = s.maxWritesPerRequest // Cluster settings. s.Cluster.Broadcaster = s.Broadcaster @@ -287,7 +285,6 @@ func (s *Server) Open() error { s.handler.API.StatusHandler = s s.handler.API.URI = s.URI s.handler.API.Cluster = s.Cluster - s.handler.API.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster @@ -750,10 +747,6 @@ func (s *Server) monitorRuntime() { } } -func (s *Server) createDefaultClient(remoteClient *http.Client) { - s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient) -} - // CountOpenFiles on operating systems that support lsof. func CountOpenFiles() (int, error) { switch runtime.GOOS { diff --git a/server/server.go b/server/server.go index 7f5782211..8fd699ce5 100644 --- a/server/server.go +++ b/server/server.go @@ -305,6 +305,7 @@ func (m *Command) SetupNetworking() error { // Set Coordinator. if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { m.Server.Cluster.Coordinator = m.Server.NodeID + m.Server.Cluster.Node.IsCoordinator = true } gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) From 75bd93d89570608d49097b2db8ed6d86bf94fa51 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 27 Apr 2018 15:11:15 +0300 Subject: [PATCH 266/311] Fixes crash due to server.diagnostics.server not set --- server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server.go b/server.go index 2b7903fb6..7186e55cc 100644 --- a/server.go +++ b/server.go @@ -220,6 +220,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { logger: NopLogger, } + s.diagnostics.server = s for _, opt := range opts { err := opt(s) From 210c78a20a7bb4a9388494f7bc7f03f5fa1d528b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 27 Apr 2018 11:26:05 -0500 Subject: [PATCH 267/311] Use build matrix instead of separate commands to test different architectures --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85e721329..46a25254c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ env: install: - make install-dep install-statik vendor generate-statik script: - - GOARCH=386 make test - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 after_success: @@ -28,6 +27,9 @@ deploy: matrix: allow_failures: - go: master + env: + - GOARCH=386 + - GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" From 0f8b630ab2bb8e6a8a652310d4202f932b22bd6f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 27 Apr 2018 11:43:31 -0500 Subject: [PATCH 268/311] Only run "deployment" (binary prerelease upload) for one build condition. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 46a25254c..5c7e8c70a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ deploy: on: all_branches: true go: "1.10" + condition: $GOARCH = amd64 matrix: allow_failures: - go: master From 52002f54556f656bc78c912614473f4cba44f5f0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 27 Apr 2018 14:51:54 -0500 Subject: [PATCH 269/311] Correct travis.ci matrix build config --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c7e8c70a..3a9abe355 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" + matrix: + - GOARCH=386 + - GOARCH=amd64 install: - make install-dep install-statik vendor generate-statik script: @@ -28,9 +31,6 @@ deploy: matrix: allow_failures: - go: master - env: - - GOARCH=386 - - GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" From 57ac2f40aaf1cd67467915e77cf35ff3a9010862 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Sat, 28 Apr 2018 09:09:03 -0500 Subject: [PATCH 270/311] fix nil client bug in monitorAntiEntropy (and test) also fix bug where broadcast_test tried to listen on localhost:10101 --- broadcast_test.go | 1 + server.go | 1 + server_test.go | 35 +++++++++++++++++++++++++++++++++++ test/pilosa.go | 34 +++++++++++++++++++++++++++------- 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 server_test.go diff --git a/broadcast_test.go b/broadcast_test.go index 187e50b4e..5654f7c28 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -57,6 +57,7 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) + com.Config.Bind = "localhost:0" err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO if err != nil { t.Fatalf("setting up server: %v", err) diff --git a/server.go b/server.go index 7186e55cc..06260ae5d 100644 --- a/server.go +++ b/server.go @@ -163,6 +163,7 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { s.executor = NewExecutor(c) + s.remoteClient = c s.defaultClient = NewInternalHTTPClientFromURI(nil, c) s.Cluster.RemoteClient = c return nil diff --git a/server_test.go b/server_test.go new file mode 100644 index 000000000..9e10374d7 --- /dev/null +++ b/server_test.go @@ -0,0 +1,35 @@ +package pilosa_test + +import ( + "context" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +// TestMonitorAntiEntropy is a regression test which which caught a bug where +// pilosa.Server was not having its remoteClient field set by an option and so +// it was using a nil client in monitorAntiEntropy. +func TestMonitorAntiEntropy(t *testing.T) { + cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1)) + client := cluster[1].Client() + err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{}) + if err != nil { + t.Fatalf("creating frame: %v", err) + } + + time.Sleep(time.Millisecond * 2) + for _, m := range cluster { + err := m.Close() + if err != nil { + t.Fatal(err) + } + } + +} diff --git a/test/pilosa.go b/test/pilosa.go index a390cd462..24838d5de 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -23,11 +23,13 @@ import ( "os" "strings" "testing" + "time" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -41,8 +43,17 @@ type Main struct { Stderr bytes.Buffer } +type MainOpt func(m *Main) error + +func OptAntiEntropyInterval(dur time.Duration) MainOpt { + return func(m *Main) error { + m.Command.Config.AntiEntropy.Interval = toml.Duration(dur) + return nil + } +} + // NewMain returns a new instance of Main with a temporary data directory and random port. -func NewMain() *Main { +func NewMain(opts ...MainOpt) *Main { path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) @@ -55,6 +66,15 @@ func NewMain() *Main { m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr + + for _, opt := range opts { + err := opt(m) + if err != nil { + panic(err) + } + + } + m.SetupServer() if testing.Verbose() { @@ -66,8 +86,8 @@ func NewMain() *Main { } // NewMainWithCluster returns a new instance of Main with clustering enabled. -func NewMainWithCluster(isCoordinator bool) *Main { - m := NewMain() +func NewMainWithCluster(isCoordinator bool, opts ...MainOpt) *Main { + m := NewMain(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator return m @@ -75,8 +95,8 @@ func NewMainWithCluster(isCoordinator bool) *Main { // MustRunMainWithCluster ruturns a running array of *Main where // all nodes are joined via memberlist (i.e. clustering enabled). -func MustRunMainWithCluster(t *testing.T, size int) []*Main { - ma, err := runMainWithCluster(size) +func MustRunMainWithCluster(t *testing.T, size int, opts ...MainOpt) []*Main { + ma, err := runMainWithCluster(size, opts...) if err != nil { t.Fatalf("new main array with cluster: %v", err) } @@ -85,7 +105,7 @@ func MustRunMainWithCluster(t *testing.T, size int) []*Main { // runMainWithCluster runs an array of *Main where all nodes are // joined via memberlist (i.e. clustering enabled). -func runMainWithCluster(size int) ([]*Main, error) { +func runMainWithCluster(size int, opts ...MainOpt) ([]*Main, error) { if size == 0 { return nil, errors.New("cluster must contain at least one node") } @@ -98,7 +118,7 @@ func runMainWithCluster(size int) ([]*Main, error) { var gossipSeeds = make([]string, size) for i := 0; i < size; i++ { - m := NewMainWithCluster(i == 0) + m := NewMainWithCluster(i == 0, opts...) m.Config.Cluster.Disabled = false gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) From 7f41c0256ca49ed607608eb01237fda80c359bb9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Sun, 29 Apr 2018 16:40:32 -0500 Subject: [PATCH 271/311] avoid creating a slice of nil timestamps on Import() --- frame.go | 6 +++++- index.go | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/frame.go b/frame.go index 0c87210bf..3de3f3613 100644 --- a/frame.go +++ b/frame.go @@ -834,7 +834,11 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Split import data by fragment. dataByFragment := make(map[importKey]importData) for i := range rowIDs { - rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i] + rowID, columnID := rowIDs[i], columnIDs[i] + var timestamp *time.Time + if len(timestamps) > i { + timestamp = timestamps[i] + } var standard, inverse []string if timestamp == nil { diff --git a/index.go b/index.go index 00ee98f11..dc676c304 100644 --- a/index.go +++ b/index.go @@ -642,7 +642,8 @@ func (i *Index) openInputDefinitions() error { // InputBits Process the []Bit though the Frame import process func (i *Index) InputBits(frame string, bits []*Bit) error { var rowIDs, columnIDs []uint64 - timestamps := make([]*time.Time, len(bits)) + var timestamps []*time.Time + f := i.Frame(frame) if f == nil { return fmt.Errorf("Frame not found: %s", frame) @@ -657,6 +658,11 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { // Convert timestamps to time.Time. if bit.Timestamp > 0 { + // Don't create a full timestamps slice unless + // at least one bit contains a timestamp. + if len(timestamps) == 0 { + timestamps = make([]*time.Time, len(bits)) + } t := time.Unix(bit.Timestamp, 0) timestamps[i] = &t } From 15f997c1379c0c3290fb7c6904a046a97633889f Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 30 Apr 2018 16:45:31 +0300 Subject: [PATCH 272/311] Added /info endpoint. Fixes #1232 --- api.go | 11 +++++++++++ handler.go | 8 ++++++++ handler_test.go | 15 +++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/api.go b/api.go index 700b21d25..1f33a6cde 100644 --- a/api.go +++ b/api.go @@ -1121,6 +1121,17 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } +// Info returns information about this server instance +func (api *API) Info() ServerInfo { + return ServerInfo{ + SliceWidth: SliceWidth, + } +} + +type ServerInfo struct { + SliceWidth uint64 `json:"sliceWidth"` +} + type apiMethod int // API validation constants. diff --git a/handler.go b/handler.go index 7bf11af1b..871ba885a 100644 --- a/handler.go +++ b/handler.go @@ -126,6 +126,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") @@ -252,6 +253,13 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { } } +func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { + info := h.API.Info() + if err := json.NewEncoder(w).Encode(info); err != nil { + h.Logger.Printf("write info response error: %s", err) + } +} + type getSchemaResponse struct { Indexes []*IndexInfo `json:"indexes"` } diff --git a/handler_test.go b/handler_test.go index 4187b3303..963c0cccc 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -154,6 +155,20 @@ func TestHandler_Status(t *testing.T) { } } +func TestHandler_Info(t *testing.T) { + s := test.NewServer() + defer s.Close() + h := test.NewHandler() + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) { + t.Fatalf("unexpected body: %s", body) + } +} + // Ensure the handler can abort a cluster resize. func TestHandler_ClusterResizeAbort(t *testing.T) { From 5a56b33826691a027efe266dfd9ee69a32936374 Mon Sep 17 00:00:00 2001 From: Will Fitzgerald Date: Tue, 1 May 2018 05:45:19 -0400 Subject: [PATCH 273/311] fix minor spelling errors --- api.go | 2 +- gossip/gossip.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 700b21d25..02ffa91dc 100644 --- a/api.go +++ b/api.go @@ -788,7 +788,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ignore this error becuase views do not exist on all nodes due to slice distribution. + // Ignore this error because views do not exist on all nodes due to slice distribution. if err != ErrInvalidView { return err } diff --git a/gossip/gossip.go b/gossip/gossip.go index e4856ec7b..25dd40cc4 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -443,7 +443,7 @@ type Transport struct { // NewTransport returns a NetTransport based on the given host and port. // It will dynamically bind to a port if port is 0. -// This is useful for test cases where specifiying a port is not reasonable. +// This is useful for test cases where specifying a port is not reasonable. //func NewTransport(host string, port int) (*memberlist.NetTransport, error) { func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) { // memberlist config From 806437bd23a63183d345b7226477f7ddbd364a09 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 2 May 2018 17:06:58 +0300 Subject: [PATCH 274/311] Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path --- broadcast_test.go | 7 ++++++- server.go | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/broadcast_test.go b/broadcast_test.go index 5654f7c28..970a249cb 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -56,9 +56,14 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-") + if err != nil { + panic(err) + } com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) com.Config.Bind = "localhost:0" - err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO + com.Config.DataDir = path + err = com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO if err != nil { t.Fatalf("setting up server: %v", err) } diff --git a/server.go b/server.go index 06260ae5d..46f8a0dda 100644 --- a/server.go +++ b/server.go @@ -21,6 +21,7 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "runtime" "strconv" "strings" @@ -230,6 +231,11 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } + err := s.expandDataDirName() + if err != nil { + return nil, err + } + s.Holder.Logger = s.logger s.Holder.Stats.SetLogger(s.logger) @@ -260,6 +266,18 @@ func NewServer(opts ...ServerOption) (*Server, error) { return s, nil } +func (s *Server) expandDataDirName() error { + prefix := "~" + string(filepath.Separator) + if strings.HasPrefix(s.Holder.Path, prefix) { + HomeDir := os.Getenv("HOME") + if HomeDir == "" { + return errors.New("data directory not specified and no home dir available") + } + s.Holder.Path = filepath.Join(HomeDir, strings.TrimPrefix(s.Holder.Path, prefix)) + } + return nil +} + // Open opens and initializes the server. func (s *Server) Open() error { s.logger.Printf("open server") From 244c4e894e875ee048c56fc04c7d525a81b0e72a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 2 May 2018 17:33:51 +0300 Subject: [PATCH 275/311] Remove ~ expanding code from Command.Start --- server/server.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/server.go b/server/server.go index 8fd699ce5..3098cddeb 100644 --- a/server/server.go +++ b/server/server.go @@ -28,9 +28,7 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "strconv" - "strings" "syscall" "time" @@ -93,14 +91,6 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { // Start starts the pilosa server - it returns once the server is running. func (m *Command) Start() (err error) { defer close(m.Started) - prefix := "~" + string(filepath.Separator) - if strings.HasPrefix(m.Config.DataDir, prefix) { - HomeDir := os.Getenv("HOME") - if HomeDir == "" { - return errors.New("data directory not specified and no home dir available") - } - m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) - } // SetupServer err = m.SetupServer() From 331a4fc5b8d2818acbfa92bd23e7c673fb8783ad Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 2 May 2018 12:21:03 -0500 Subject: [PATCH 276/311] add NOTICE to binary distributions --- Dockerfile | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf076e526..404be937f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,8 @@ LABEL maintainer "dev@pilosa.com" COPY --from=builder /go/bin/pilosa /pilosa +COPY NOTICE /NOTICE + EXPOSE 10101 VOLUME /data diff --git a/Makefile b/Makefile index cab932fba..e4d120659 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ build: vendor # Create a single release build under the build directory release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" - cp LICENSE README.md build/pilosa-$(VERSION_ID) + cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz From 18d7664fc43cf2148a6c6c15557ade6574a289b7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 2 May 2018 15:04:12 -0500 Subject: [PATCH 277/311] Rename ID file to ".id" for consistency with existing data files. --- holder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holder.go b/holder.go index 7649caeba..d0a242ca5 100644 --- a/holder.go +++ b/holder.go @@ -524,7 +524,7 @@ func (h *Holder) setFileLimit() { } func (h *Holder) loadNodeID() (string, error) { - idPath := path.Join(h.Path, "ID") + idPath := path.Join(h.Path, ".id") nodeID := "" h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { From beaa69a7bd6eaa38e3eb0e6ed4076eb8768fb165 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 3 May 2018 00:07:56 +0300 Subject: [PATCH 278/311] Made expandDirName generic. --- server.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/server.go b/server.go index 46f8a0dda..86f14c8b3 100644 --- a/server.go +++ b/server.go @@ -231,14 +231,21 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } - err := s.expandDataDirName() + path, err := expandDirName(s.Holder.Path) if err != nil { return nil, err } + s.Holder.Path = path s.Holder.Logger = s.logger s.Holder.Stats.SetLogger(s.logger) + path, err = expandDirName(s.Cluster.Path) + if err != nil { + return nil, err + } + s.Cluster.Path = path + s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder @@ -266,18 +273,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { return s, nil } -func (s *Server) expandDataDirName() error { - prefix := "~" + string(filepath.Separator) - if strings.HasPrefix(s.Holder.Path, prefix) { - HomeDir := os.Getenv("HOME") - if HomeDir == "" { - return errors.New("data directory not specified and no home dir available") - } - s.Holder.Path = filepath.Join(HomeDir, strings.TrimPrefix(s.Holder.Path, prefix)) - } - return nil -} - // Open opens and initializes the server. func (s *Server) Open() error { s.logger.Printf("open server") @@ -796,3 +791,15 @@ type StatusHandler interface { ClusterStatus() (proto.Message, error) HandleRemoteStatus(proto.Message) error } + +func expandDirName(path string) (string, error) { + prefix := "~" + string(filepath.Separator) + if strings.HasPrefix(path, prefix) { + HomeDir := os.Getenv("HOME") + if HomeDir == "" { + return "", errors.New("data directory not specified and no home dir available") + } + return filepath.Join(HomeDir, strings.TrimPrefix(path, prefix)), nil + } + return path, nil +} From 85f90ac8fcf42a631c5490dba6500e2c37df419e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 3 May 2018 16:11:43 -0500 Subject: [PATCH 279/311] Log time/version to startup log (Fixes #391) --- holder.go | 22 ++++++++++++++++++++++ server.go | 7 +++++++ 2 files changed, 29 insertions(+) diff --git a/holder.go b/holder.go index 7649caeba..49037d09f 100644 --- a/holder.go +++ b/holder.go @@ -547,6 +547,28 @@ func (h *Holder) loadNodeID() (string, error) { return nodeID, nil } +// Log startup time and version to $DATA_DIR/.startup.log +func (h *Holder) logStartup() error { + time, err := time.Now().MarshalText() + if err != nil { + return errors.Wrap(err, "creating timestamp") + } + logLine := fmt.Sprintf("%s\t%s\n", time, Version) + + f, err := os.OpenFile(h.Path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return errors.Wrap(err, "opening startup log") + } + + defer f.Close() + + if _, err = f.WriteString(logLine); err != nil { + return errors.Wrap(err, "writing startup log") + } + + return nil +} + // HolderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. type HolderSyncer struct { diff --git a/server.go b/server.go index 06260ae5d..7617d7e69 100644 --- a/server.go +++ b/server.go @@ -17,6 +17,7 @@ package pilosa import ( "context" "fmt" + "log" "net" "net/http" "os" @@ -268,6 +269,12 @@ func (s *Server) Open() error { return errors.New("Must pass a listener option to NewServer") } + // Log startup + err := s.Holder.logStartup() + if err != nil { + log.Println(errors.Wrap(err, "logging startup")) + } + // Get or create NodeID. // Append the NodeID tag to stats. From e751945bf142cd53c80d2bfa930d00b945647e5f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 11:33:59 -0500 Subject: [PATCH 280/311] Add docs for upgrading Pilosa --- docs/administration.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 190b5ca1f..eee32e419 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -92,7 +92,41 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ #### Upgrading -When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes. +To upgrade Pilosa: + +1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`. +2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases). +3. Shut down the Pilosa cluster. A graceful shutdown should ensure that all data files are flushed to disk. +4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node. +5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make. +6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes. + +##### Version 0.9 + +Pilosa v0.9 introduces a few compatibility changes that need to be addressed. + +**Configuration changes**: These changes need to occur during step 5 above: + +1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster. +2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set. +3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds. + +**Data directory changes**: These changes also need to occur during step 5 above (while the cluster is shut down): + +Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`: + +1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. +2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`. +3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`. +4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position. +5. Copy the file `topology` into the data directories of every node in the cluster, naming it `.topology` (note the dot), e.g. `cp topology ~/.pilosa/.topology` or `scp topology node1:.pilosa/.topology`. +6. Copy the node ID files into the respective node data directories. For example, `node0.id` will be placed on the first node in the `cluster.hosts` list, with the name `.id`. For example: `scp node0.id node0:.pilosa/.id`. Again, it is very important that the ordering you give the nodes with these IDs matches the ordering you had in your existing `cluster.hosts` setting. + +**Application changes**: + +1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported. +2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed. +3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release. ### Resizing the Cluster From 0afb95c1711b199501c282e731e82cbe9234fd66 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 12:33:20 -0500 Subject: [PATCH 281/311] Minor upgrading docs changes --- docs/administration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index eee32e419..0ecbbed94 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -96,7 +96,7 @@ To upgrade Pilosa: 1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`. 2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases). -3. Shut down the Pilosa cluster. A graceful shutdown should ensure that all data files are flushed to disk. +3. Shut down the Pilosa cluster. 4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node. 5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make. 6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes. @@ -105,13 +105,13 @@ To upgrade Pilosa: Pilosa v0.9 introduces a few compatibility changes that need to be addressed. -**Configuration changes**: These changes need to occur during step 5 above: +**Configuration changes**: These changes need to occur before starting Pilosa v0.9: 1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster. 2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set. 3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds. -**Data directory changes**: These changes also need to occur during step 5 above (while the cluster is shut down): +**Data directory changes**: These changes need to occur while the cluster is shut down, before starting Pilosa v0.9: Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`: From 992c3e8bdda48572b0cc3eb67ac56c6a6a7e714b Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 26 Apr 2018 08:39:56 -0500 Subject: [PATCH 282/311] WIP - adding more locking to Cluster --- api.go | 7 +- cluster.go | 23 ++- cluster_test.go | 170 ++++++++--------- server/cluster_test.go | 4 +- test/pilosa.go | 7 +- utils_test.go | 425 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 538 insertions(+), 98 deletions(-) create mode 100644 utils_test.go diff --git a/api.go b/api.go index 02ffa91dc..9f7356cb8 100644 --- a/api.go +++ b/api.go @@ -1053,8 +1053,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return nil, nil, errors.Wrap(err, "validate api method") } - oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) - newNode = api.Cluster.nodeByID(id) + oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.NodeByID(id) if newNode == nil { return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") } @@ -1102,9 +1102,6 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "validate api method") } - if !api.Cluster.IsCoordinator() { - return ErrNodeNotCoordinator - } err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) return errors.Wrap(err, "complete current job") } diff --git a/cluster.go b/cluster.go index bbf6f799e..c9f5fd9a8 100644 --- a/cluster.go +++ b/cluster.go @@ -292,6 +292,12 @@ func (c *Cluster) CoordinatorNode() *Node { // IsCoordinator is true if this node is the coordinator. func (c *Cluster) IsCoordinator() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.isCoordinator() +} + +func (c *Cluster) isCoordinator() bool { return c.Coordinator == c.Node.ID } @@ -300,6 +306,8 @@ func (c *Cluster) IsCoordinator() bool { // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. func (c *Cluster) SetCoordinator(n *Node) error { + c.mu.Lock() + defer c.mu.Unlock() // Verify that the new Coordinator value matches // this node. if c.Node.ID != n.ID { @@ -511,6 +519,12 @@ func (c *Cluster) Status() *internal.ClusterStatus { } } +func (c *Cluster) NodeByID(id string) *Node { + c.mu.RLock() + defer c.mu.RUnlock() + return c.nodeByID(id) +} + // nodeByID returns a node reference by ID. func (c *Cluster) nodeByID(id string) *Node { for _, n := range c.Nodes { @@ -1186,6 +1200,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, func (c *Cluster) CompleteCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() + if !c.isCoordinator() { + return ErrNodeNotCoordinator + } if c.currentJob == nil { return ErrResizeNotRunning } @@ -1797,9 +1814,11 @@ func (c *Cluster) nodeLeave(node *Node) error { } func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { + c.mu.Lock() + defer c.mu.Unlock() c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). - if c.IsCoordinator() { + if c.isCoordinator() { return nil } @@ -1836,7 +1855,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } } - c.SetState(cs.State) + c.setState(cs.State) c.markAsJoined() diff --git a/cluster_test.go b/cluster_test.go index 6c5d41829..b09820ac8 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "bytes" @@ -22,29 +22,27 @@ import ( "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" ) // Ensure the cluster can fairly distribute partitions across the nodes. func TestCluster_Owners(t *testing.T) { - c := pilosa.Cluster{ - Nodes: []*pilosa.Node{ - {URI: test.NewURIFromHostPort("serverA", 1000)}, - {URI: test.NewURIFromHostPort("serverB", 1000)}, - {URI: test.NewURIFromHostPort("serverC", 1000)}, + c := Cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, }, - Hasher: test.NewModHasher(), + Hasher: NewTestModHasher(), ReplicaN: 2, } // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) { + if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) { + if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } } @@ -52,7 +50,7 @@ func TestCluster_Owners(t *testing.T) { // Ensure the partitioner can assign a fragment to a partition. func TestCluster_Partition(t *testing.T) { if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := pilosa.NewCluster() + c := NewCluster() c.PartitionN = partitionN partitionID := c.Partition(index, slice) @@ -85,7 +83,7 @@ func TestHasher(t *testing.T) { {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, } { for i, v := range tt.bucket { - if got := pilosa.NewHasher().Hash(tt.key, i+1); got != v { + if got := NewHasher().Hash(tt.key, i+1); got != v { t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) } } @@ -94,8 +92,8 @@ func TestHasher(t *testing.T) { // Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { - c := test.NewCluster(5) - slices := c.OwnsSlices("test", 10, test.NewURIFromHostPort("host2", 0)) + c := NewTestCluster(5) + slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { t.Fatalf("unexpected slices for node's index: %v", slices) @@ -104,7 +102,7 @@ func TestCluster_OwnsSlices(t *testing.T) { // Ensure ContainsSlices can find the actual slice list for node and index. func TestCluster_ContainsSlices(t *testing.T) { - c := test.NewCluster(5) + c := NewTestCluster(5) c.ReplicaN = 3 slices := c.ContainsSlices("test", 10, c.Nodes[2]) @@ -114,20 +112,20 @@ func TestCluster_ContainsSlices(t *testing.T) { } func TestCluster_Nodes(t *testing.T) { - uri0 := test.NewURIFromHostPort("node0", 0) - uri1 := test.NewURIFromHostPort("node1", 0) - uri2 := test.NewURIFromHostPort("node2", 0) - uri3 := test.NewURIFromHostPort("node3", 0) + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) - node0 := &pilosa.Node{ID: "node0", URI: uri0} - node1 := &pilosa.Node{ID: "node1", URI: uri1} - node2 := &pilosa.Node{ID: "node2", URI: uri2} - node3 := &pilosa.Node{ID: "node3", URI: uri3} + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} - nodes := []*pilosa.Node{node0, node1, node2} + nodes := []*Node{node0, node1, node2} t.Run("NodeIDs", func(t *testing.T) { - actual := pilosa.Nodes(nodes).IDs() + actual := Nodes(nodes).IDs() expected := []string{node0.ID, node1.ID, node2.ID} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) @@ -135,24 +133,24 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Filter", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs() - expected := []pilosa.URI{uri0, uri2} + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("FilterURI", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs() - expected := []pilosa.URI{uri0, uri2} + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("Contains", func(t *testing.T) { - actualTrue := pilosa.Nodes(nodes).Contains(node1) - actualFalse := pilosa.Nodes(nodes).Contains(node3) + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) if !reflect.DeepEqual(actualTrue, true) { t.Errorf("expected: %v, but got: %v", true, actualTrue) } @@ -162,9 +160,9 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Clone", func(t *testing.T) { - clone := pilosa.Nodes(nodes).Clone() - actual := pilosa.Nodes(clone).URIs() - expected := []pilosa.URI{uri0, uri1, uri2} + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -172,16 +170,16 @@ func TestCluster_Nodes(t *testing.T) { } func TestCluster_Coordinator(t *testing.T) { - uri1 := test.NewURIFromHostPort("node1", 0) - uri2 := test.NewURIFromHostPort("node2", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) - node1 := &pilosa.Node{ID: "node1", URI: uri1} - node2 := &pilosa.Node{ID: "node2", URI: uri2} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} - c1 := *pilosa.NewCluster() + c1 := *NewCluster() c1.Node = node1 c1.Coordinator = node1.ID - c2 := *pilosa.NewCluster() + c2 := *NewCluster() c2.Node = node2 c2.Coordinator = node1.ID @@ -195,17 +193,17 @@ func TestCluster_Coordinator(t *testing.T) { } func TestCluster_Topology(t *testing.T) { - c1 := test.NewCluster(1) // automatically creates Node{ID: "node0"} + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} - uri0 := test.NewURIFromHostPort("host0", 0) - uri1 := test.NewURIFromHostPort("host1", 0) - uri2 := test.NewURIFromHostPort("host2", 0) - invalid := test.NewURIFromHostPort("invalid", 0) + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) - node0 := &pilosa.Node{ID: "node0", URI: uri0} - node1 := &pilosa.Node{ID: "node1", URI: uri1} - node2 := &pilosa.Node{ID: "node2", URI: uri2} - nodeinvalid := &pilosa.Node{ID: "nodeinvalid", URI: invalid} + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} t.Run("AddNode", func(t *testing.T) { err := c1.AddNode(node1) @@ -243,7 +241,7 @@ func TestCluster_Topology(t *testing.T) { func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, no data", func(t *testing.T) { - tc := test.NewTestCluster(1) + tc := NewClusterCluster(1) // Open TestCluster. if err := tc.Open(); err != nil { @@ -253,11 +251,11 @@ func TestCluster_ResizeStates(t *testing.T) { node := tc.Clusters[0] // Ensure that node comes up in state NORMAL. - if node.State() != pilosa.ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State()) + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) } - expectedTop := &pilosa.Topology{ + expectedTop := &Topology{ NodeIDs: []string{node.Node.ID}, } @@ -273,13 +271,13 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Single node, in topology", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node := tc.Clusters[0] // write topology to data file - top := &pilosa.Topology{ + top := &Topology{ NodeIDs: []string{node.Node.ID}, } tc.WriteTopology(node.Path, top) @@ -290,8 +288,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensure that node comes up in state NORMAL. - if node.State() != pilosa.ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State()) + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) } // Close TestCluster. @@ -301,13 +299,13 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Single node, not in topology", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node := tc.Clusters[0] // write topology to data file - top := &pilosa.Topology{ + top := &Topology{ NodeIDs: []string{"some-other-host"}, } tc.WriteTopology(node.Path, top) @@ -326,7 +324,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) // Open TestCluster. @@ -340,13 +338,13 @@ func TestCluster_ResizeStates(t *testing.T) { node1 := tc.Clusters[1] // Ensure that nodes comes up in state NORMAL. - if node0.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) - } else if node1.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State()) + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) } - expectedTop := &pilosa.Topology{ + expectedTop := &Topology{ NodeIDs: []string{node0.Node.ID, node1.Node.ID}, } @@ -364,12 +362,12 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node0 := tc.Clusters[0] // write topology to data file - top := &pilosa.Topology{ + top := &Topology{ NodeIDs: []string{"node0", "node2"}, } tc.WriteTopology(node0.Path, top) @@ -380,8 +378,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensure that node is in state STARTING before the other node joins. - if node0.State() != pilosa.ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State()) + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) } // Expect an error by adding a node not in the topology. @@ -395,10 +393,10 @@ func TestCluster_ResizeStates(t *testing.T) { node2 := tc.Clusters[2] // Ensure that node comes up in state NORMAL. - if node0.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) - } else if node2.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State()) + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) } // Close TestCluster. @@ -408,7 +406,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node0 := tc.Clusters[0] @@ -418,20 +416,20 @@ func TestCluster_ResizeStates(t *testing.T) { } // Add Bit Data to node0. - if err := tc.CreateFrame("i", "f", pilosa.FrameOptions{}); err != nil { + if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil { t.Fatal(err) } tc.SetBit("i", "f", "standard", 1, 101, nil) tc.SetBit("i", "f", "standard", 1, 1300000, nil) // Add Field Data to node0. - if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{ + if err := tc.CreateFrame("i", "fields", FrameOptions{ InverseEnabled: false, - //CacheType: pilosa.CacheTypeNone, - Fields: []*pilosa.Field{ + //CacheType: CacheTypeNone, + Fields: []*Field{ { Name: "fld0", - Type: pilosa.FieldTypeInt, + Type: FieldTypeInt, Min: -100, Max: 100, }, @@ -461,13 +459,13 @@ func TestCluster_ResizeStates(t *testing.T) { node1 := tc.Clusters[1] // Ensure that nodes come up in state NORMAL. - if node0.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) - } else if node1.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State()) + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) } - expectedTop := &pilosa.Topology{ + expectedTop := &Topology{ NodeIDs: []string{node0.Node.ID, node1.Node.ID}, } @@ -510,7 +508,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Ensures that coordinator can be changed. func TestCluster_UpdateCoordinator(t *testing.T) { t.Run("UpdateCoordinator", func(t *testing.T) { - c := test.NewCluster(2) + c := NewTestCluster(2) oldNode := c.Nodes[0] newNode := c.Nodes[1] diff --git a/server/cluster_test.go b/server/cluster_test.go index 1a64ce305..326753a3d 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -396,7 +396,7 @@ func TestCluster_GossipMembership(t *testing.T) { var eg errgroup.Group eg.Go(func() error { // Pass invalid seed as first in list - _, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) + _, err := m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) if err != nil { return err } @@ -409,7 +409,7 @@ func TestCluster_GossipMembership(t *testing.T) { eg.Go(func() error { // Pass invalid seed as last in list - _, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) + _, err := m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) if err != nil { return err } diff --git a/test/pilosa.go b/test/pilosa.go index 24838d5de..f6e3a3457 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -66,7 +66,6 @@ func NewMain(opts ...MainOpt) *Main { m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr - for _, opt := range opts { err := opt(m) if err != nil { @@ -74,8 +73,10 @@ func NewMain(opts ...MainOpt) *Main { } } - - m.SetupServer() + err = m.SetupServer() + if err != nil { + panic(err) + } if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 000000000..7bc17d424 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,425 @@ +package pilosa + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" +) + +// NewCluster returns a cluster with n nodes and uses a mod-based hasher. +func NewTestCluster(n int) *Cluster { + path, err := ioutil.TempDir("", "pilosa-cluster-") + if err != nil { + panic(err) + } + + c := NewCluster() + c.ReplicaN = 1 + c.Hasher = NewTestModHasher() + c.Path = path + c.Topology = NewTopology() + + for i := 0; i < n; i++ { + c.Nodes = append(c.Nodes, &Node{ + ID: fmt.Sprintf("node%d", i), + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + } + + c.Node = c.Nodes[0] + c.Coordinator = c.Nodes[0].ID + c.SetState(ClusterStateNormal) + + return c +} + +// NewURI is a test URI creator that intentionally swallows errors. +func NewTestURI(scheme, host string, port uint16) URI { + uri := DefaultURI() + uri.SetScheme(scheme) + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +func NewTestURIFromHostPort(host string, port uint16) URI { + uri := DefaultURI() + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +// ModHasher represents a simple, mod-based hashing. +type TestModHasher struct{} + +// NewModHasher returns a new instance of ModHasher with n buckets. +func NewTestModHasher() *TestModHasher { return &TestModHasher{} } + +func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n } + +// ClusterCluster represents a cluster of test nodes, each of which +// has a Cluster. +type ClusterCluster struct { + Clusters []*Cluster + + common *commonClusterSettings + + mu sync.RWMutex + resizing bool + resizeDone chan struct{} +} + +type commonClusterSettings struct { + Nodes []*Node +} + +func (t *ClusterCluster) CreateIndex(name string) error { + for _, c := range t.Clusters { + if _, err := c.Holder.CreateIndexIfNotExists(name, IndexOptions{}); err != nil { + return err + } + } + return nil +} + +func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error { + for _, c := range t.Clusters { + idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) + if err != nil { + return err + } + if _, err := idx.CreateFrame(frame, opt); err != nil { + return err + } + } + return nil +} +func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { + // Determine which node should receive the SetBit. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := colID / SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetBit(view, rowID, colID, x) + if err != nil { + return err + } + } + + return nil +} + +func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { + // Determine which node should receive the SetFieldValue. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := columnID / SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetFieldValue(columnID, name, value) + if err != nil { + return err + } + } + + return nil +} + +func (t *ClusterCluster) clusterByID(id string) *Cluster { + for _, c := range t.Clusters { + if c.Node.ID == id { + return c + } + } + return nil +} + +// AddNode adds a node to the cluster and (potentially) starts a resize job. +func (t *ClusterCluster) AddNode(saveTopology bool) error { + id := len(t.Clusters) + + c, err := t.addCluster(id, saveTopology) + if err != nil { + return err + } + + // Send NodeJoin event to coordinator. + if id > 0 { + coord := t.Clusters[0] + ev := &NodeEvent{ + Event: NodeJoin, + Node: c.Node, + } + + if err := coord.ReceiveEvent(ev); err != nil { + return err + } + + // Wait for the AddNode job to finish. + if c.State() != ClusterStateNormal { + t.resizeDone = make(chan struct{}) + t.mu.Lock() + t.resizing = true + t.mu.Unlock() + <-t.resizeDone + } + } + + return nil +} + +// WriteTopology writes the given topology to disk. +func (t *ClusterCluster) WriteTopology(path string, top *Topology) error { + if buf, err := proto.Marshal(top.Encode()); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { + return err + } + return nil +} + +func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) { + + id := fmt.Sprintf("node%d", i) + uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)) + + node := &Node{ + ID: id, + URI: uri, + } + + // add URI to common + //t.common.NodeIDs = append(t.common.NodeIDs, id) + //sort.Sort(t.common.NodeIDs) + + // add node to common + t.common.Nodes = append(t.common.Nodes, node) + + // create node-specific temp directory + path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + if err != nil { + return nil, err + } + + // holder + h := NewHolder() + h.Path = path + + // cluster + c := NewCluster() + c.ReplicaN = 1 + c.Hasher = NewTestModHasher() + c.Path = path + c.Topology = NewTopology() + c.Holder = h + c.MemberSet = NewStaticMemberSet(c.Nodes) + c.Node = node + c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator + c.Broadcaster = t + + // add nodes + if saveTopology { + for _, n := range t.common.Nodes { + c.AddNode(n) + } + } + + // Add this node to the ClusterCluster. + t.Clusters = append(t.Clusters, c) + + return c, nil +} + +// NewClusterCluster returns a new instance of test.Cluster. +func NewClusterCluster(n int) *ClusterCluster { + + tc := &ClusterCluster{ + common: &commonClusterSettings{}, + } + + // add clusters + for i := 0; i < n; i++ { + _, err := tc.addCluster(i, true) + if err != nil { + panic(err) + } + } + return tc +} + +// SetState sets the state of the cluster on each node. +func (t *ClusterCluster) SetState(state string) { + for _, c := range t.Clusters { + c.SetState(state) + } +} + +// Open opens all clusters in the test cluster. +func (t *ClusterCluster) Open() error { + for _, c := range t.Clusters { + if err := c.Open(); err != nil { + return err + } + if err := c.Holder.Open(); err != nil { + return err + } + if err := c.SetNodeState(NodeStateReady); err != nil { + return err + } + } + + // Start the listener on the coordinator. + if len(t.Clusters) == 0 { + return nil + } + t.Clusters[0].ListenForJoins() + + return nil +} + +// Close closes all clusters in the test cluster. +func (t *ClusterCluster) Close() error { + for _, c := range t.Clusters { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + +// ClusterCluster implements Broadcaster interface. + +// SendSync is a test implemenetation of Broadcaster SendSync method. +func (t *ClusterCluster) SendSync(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ClusterStatus: + // Apply the send message to all nodes (except the coordinator). + for _, c := range t.Clusters { + c.MergeClusterStatus(obj) + } + t.mu.RLock() + if obj.State == ClusterStateNormal && t.resizing { + close(t.resizeDone) + } + t.mu.RUnlock() + } + + return nil +} + +// SendAsync is a test implemenetation of Broadcaster SendAsync method. +func (t *ClusterCluster) SendAsync(pb proto.Message) error { + return nil +} + +// SendTo is a test implemenetation of Broadcaster SendTo method. +func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ResizeInstruction: + err := t.FollowResizeInstruction(obj) + if err != nil { + return err + } + case *internal.ResizeInstructionComplete: + coord := t.clusterByID(to.ID) + go coord.MarkResizeInstructionComplete(obj) + } + return nil +} + +// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. +func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + + // Prepare the return message. + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + Node: instr.Node, + Error: "", + } + + // Stop processing on any error. + if err := func() error { + + // figure out which node it was meant for, then call the operation on that cluster + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + instrNode := DecodeNode(instr.Node) + destCluster := t.clusterByID(instrNode.ID) + + // Sync the schema received in the resize instruction. + if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + return err + } + + for _, src := range instr.Sources { + srcNode := DecodeNode(src.Node) + srcCluster := t.clusterByID(srcNode.ID) + + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + if destFragment == nil { + // Create fragment on destination if it doesn't exist. + f := destCluster.Holder.Frame(src.Index, src.Frame) + v := f.View(src.View) + var err error + destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + } + + buf := bytes.NewBuffer(nil) + + bw := bufio.NewWriter(buf) + br := bufio.NewReader(buf) + + // Get the fragment from source. + if _, err := srcFragment.WriteTo(bw); err != nil { + return err + } + + // Flush the bufio.buf to the io.Writer (buf). + bw.Flush() + + // Write data to destination. + if _, err := destFragment.ReadFrom(br); err != nil { + return err + } + } + + return nil + }(); err != nil { + complete.Error = err.Error() + } + + node := DecodeNode(instr.Coordinator) + if err := t.SendTo(node, complete); err != nil { + return err + } + + return nil +} From 2ef78af0ca86af9336cf61d1b7d88ab7eec692ce Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 26 Apr 2018 10:42:50 -0500 Subject: [PATCH 283/311] adjusted protection around cluster coordinator mutations --- cluster.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index c9f5fd9a8..c7285d17a 100644 --- a/cluster.go +++ b/cluster.go @@ -315,7 +315,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { } // Update IsCoordinator on all nodes (locally). - _ = c.UpdateCoordinator(n) + _ = c.updateCoordinator(n) // Send the update coordinator message to all nodes. err := c.Broadcaster.SendSync( @@ -335,6 +335,12 @@ func (c *Cluster) SetCoordinator(n *Node) error { // to true, and sets all other nodes to false. Returns true if the value // changed. func (c *Cluster) UpdateCoordinator(n *Node) bool { + c.mu.Lock() + defer c.mu.Unlock() + _ = c.updateCoordinator(n) +} + +func (c *Cluster) updateCoordinator(n *Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID From 4a1d15ec9dd235d39628488947ec1d337f23aca2 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 26 Apr 2018 11:43:39 -0500 Subject: [PATCH 284/311] missed a file on last commit --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index c7285d17a..b02bd50d2 100644 --- a/cluster.go +++ b/cluster.go @@ -337,7 +337,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { func (c *Cluster) UpdateCoordinator(n *Node) bool { c.mu.Lock() defer c.mu.Unlock() - _ = c.updateCoordinator(n) + return c.updateCoordinator(n) } func (c *Cluster) updateCoordinator(n *Node) bool { From 985b56f50b6f0f660b1f86d6810fb967d19c2d03 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 27 Apr 2018 10:20:24 -0500 Subject: [PATCH 285/311] fix utils_test comments --- utils_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils_test.go b/utils_test.go index 7bc17d424..05afeac19 100644 --- a/utils_test.go +++ b/utils_test.go @@ -13,7 +13,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -// NewCluster returns a cluster with n nodes and uses a mod-based hasher. +// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. func NewTestCluster(n int) *Cluster { path, err := ioutil.TempDir("", "pilosa-cluster-") if err != nil { @@ -40,7 +40,7 @@ func NewTestCluster(n int) *Cluster { return c } -// NewURI is a test URI creator that intentionally swallows errors. +// NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { uri := DefaultURI() uri.SetScheme(scheme) @@ -59,13 +59,14 @@ func NewTestURIFromHostPort(host string, port uint16) URI { // ModHasher represents a simple, mod-based hashing. type TestModHasher struct{} -// NewModHasher returns a new instance of ModHasher with n buckets. +// NewTestModHasher returns a new instance of ModHasher with n buckets. func NewTestModHasher() *TestModHasher { return &TestModHasher{} } func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n } // ClusterCluster represents a cluster of test nodes, each of which // has a Cluster. +// ClusterCluster implements Broadcaster interface. type ClusterCluster struct { Clusters []*Cluster @@ -101,6 +102,7 @@ func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) erro } return nil } + func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. @@ -312,8 +314,6 @@ func (t *ClusterCluster) Close() error { return nil } -// ClusterCluster implements Broadcaster interface. - // SendSync is a test implemenetation of Broadcaster SendSync method. func (t *ClusterCluster) SendSync(pb proto.Message) error { switch obj := pb.(type) { From 84ed1953bd60a8cdb5eb9324ac79d3ab68746eb7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 2 May 2018 11:52:35 -0500 Subject: [PATCH 286/311] Update changelog --- CHANGELOG.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09562dc06..ed2baf7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## Unreleased +## [v0.9.0] - 2018-05-02 + +This version contains 183 contribution from 12 contributors. There are 138 files changed, 17,084 insertions, and 7,482 deletions. ### Added @@ -23,7 +25,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)) +- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)), ([#1215](https://github.com/pilosa/pilosa/pull/1215)), ([#1213](https://github.com/pilosa/pilosa/pull/1213)), ([#1224](https://github.com/pilosa/pilosa/pull/1224)) ### Changed @@ -42,12 +45,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) - Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) +- Refactoring pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) +- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) +- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) ### Removed - Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) - Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) - Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) +- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) +- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) +- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) +- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) +- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) +- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) ### Fixed @@ -60,6 +72,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) - Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) - Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) +- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) +- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) ### Performance From 46a929e1d64701e42d47088f2fe60e05911a1d94 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 15:49:03 -0500 Subject: [PATCH 287/311] More updates for v0.9 branch --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed2baf7af..f9671e748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [v0.9.0] - 2018-05-02 +## [v0.9.0] - 2018-05-04 This version contains 183 contribution from 12 contributors. There are 138 files changed, 17,084 insertions, and 7,482 deletions. @@ -22,11 +22,12 @@ This version contains 183 contribution from 12 contributors. There are 138 files - Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) - Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) - Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013)) +- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) - Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)), ([#1215](https://github.com/pilosa/pilosa/pull/1215)), ([#1213](https://github.com/pilosa/pilosa/pull/1213)), ([#1224](https://github.com/pilosa/pilosa/pull/1224)) +- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) ### Changed @@ -48,6 +49,7 @@ This version contains 183 contribution from 12 contributors. There are 138 files - Refactoring pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) - Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) - Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) +- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) ### Removed @@ -74,6 +76,7 @@ This version contains 183 contribution from 12 contributors. There are 138 files - Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) - Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) - Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) +- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) ### Performance From 995a90ac9d3b1f6578f19c77b99d917cd5357c6d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 16:22:00 -0500 Subject: [PATCH 288/311] Update pilosa version in installation docs --- docs/installation.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index db0828b2f..a0632c34c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz + tar xfz pilosa-v0.9.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.9.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -234,19 +234,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.3-linux-amd64.tar.gz + tar xfz pilosa-v0.9.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.9.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: From 3529043fbcc4df1c29b5026138f055e9fc05bf57 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 16:22:08 -0500 Subject: [PATCH 289/311] Update Go version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 404be937f..0392dbe87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10 as builder +FROM golang:1.10.2 as builder COPY . /go/src/github.com/pilosa/pilosa/ From 4454bedb3d14f1df9bc7a1c197a855f557f5e451 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 16:28:19 -0500 Subject: [PATCH 290/311] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9671e748..2a0dc777c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [v0.9.0] - 2018-05-04 -This version contains 183 contribution from 12 contributors. There are 138 files changed, 17,084 insertions, and 7,482 deletions. +This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions. ### Added From 76ee46bd0d265eb4cd87e24797d9860a9b7bebf3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 17:03:43 -0500 Subject: [PATCH 291/311] Add note about upgrading to v0.9 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a0dc777c..32c84b835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions. +*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* + ### Added - Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) From 7cecfd30cf7c5354cc926f150c15ef2e9aa7f650 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 17:07:39 -0500 Subject: [PATCH 292/311] Changelog fixes --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c84b835..5ab4ad1bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ This version contains 188 contribution from 12 contributors. There are 141 files - Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) - Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) - Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) +- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) - Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) @@ -45,10 +45,9 @@ This version contains 188 contribution from 12 contributors. There are 141 files - Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) - Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) +- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) -- Refactoring pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) +- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) - Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) - Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) - Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) @@ -80,6 +79,10 @@ This version contains 188 contribution from 12 contributors. There are 141 files - Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) - Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) +### Deprecated + +- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) + ### Performance - Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) From b3f529cb1b08d8455b7a023c10e6b4db1a3de9ab Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sun, 6 May 2018 19:44:36 -0700 Subject: [PATCH 293/311] remove unused NodeID method on InternalClient --- client.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/client.go b/client.go index c2e88c77b..eee1a3a9c 100644 --- a/client.go +++ b/client.go @@ -1142,30 +1142,6 @@ func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI { return clientURI } -func (c *InternalHTTPClient) NodeID(uri *URI) (string, error) { - u := uriPathToURL(uri, "/id") - req, err := http.NewRequest("GET", u.String(), nil) - resp, err := c.HTTPClient.Do(req) - if err != nil { - return "", fmt.Errorf("executing http request: %v", err) - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading response body: %v", err) - } - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return "", fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) - } - return string(body), nil -} - // Bit represents the location of a single bit. type Bit struct { RowID uint64 @@ -1362,5 +1338,4 @@ type InternalClient interface { ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, pb proto.Message) error - NodeID(uri *URI) (string, error) } From 0d3df71e37b3a34cd31c1ff6c7723c5ac7e4d586 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sun, 6 May 2018 20:12:23 -0700 Subject: [PATCH 294/311] add URI argument to InternalClient.SendMessage passing values through context is error prone and usually bad practice. --- client.go | 6 +++--- server.go | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/client.go b/client.go index eee1a3a9c..ecfae5801 100644 --- a/client.go +++ b/client.go @@ -1098,13 +1098,13 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin } // SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, pb proto.Message) error { +func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return fmt.Errorf("marshaling message: %v", err) } - u := uriPathToURL(ctx.Value("uri").(*URI), "/cluster/message") + u := uriPathToURL(uri, "/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -1337,5 +1337,5 @@ type InternalClient interface { BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, pb proto.Message) error + SendMessage(ctx context.Context, uri *URI, pb proto.Message) error } diff --git a/server.go b/server.go index 7617d7e69..52999b1c5 100644 --- a/server.go +++ b/server.go @@ -536,15 +536,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { + node := node s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.URI == node.URI { continue } - ctx := context.WithValue(context.Background(), "uri", &node.URI) eg.Go(func() error { - return s.defaultClient.SendMessage(ctx, pb) + return s.defaultClient.SendMessage(context.Background(), &node.URI, pb) }) } @@ -559,8 +559,7 @@ func (s *Server) SendAsync(pb proto.Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, pb proto.Message) error { s.logger.Printf("SendTo: %s", to.URI) - ctx := context.WithValue(context.Background(), "uri", &to.URI) - return s.defaultClient.SendMessage(ctx, pb) + return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) } // Server implements StatusHandler. From 2738c922864edabfc3237511eb0364640f1cf10d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sun, 6 May 2018 20:30:58 -0700 Subject: [PATCH 295/311] stop passing uri via context to InternalClient.ExecuteQuery --- client.go | 14 ++++++++++---- client_test.go | 8 ++++---- ctl/bench.go | 2 +- executor.go | 3 +-- fragment.go | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/client.go b/client.go index ecfae5801..8950e8406 100644 --- a/client.go +++ b/client.go @@ -223,8 +223,13 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl return a, nil } -// ExecuteQuery executes query against index on the server. -func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +// QueryNode executes query against the index. +func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return c.QueryNode(ctx, c.defaultURI, index, queryRequest) +} + +// QueryNode executes query against the index, sending the request to the node specified. +func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { if index == "" { return nil, ErrIndexRequired } else if queryRequest.Query == "" { @@ -238,7 +243,7 @@ func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, que } // Create HTTP request. - u := c.clientURI(ctx).Path(fmt.Sprintf("/index/%s/query", index)) + u := uri.Path(fmt.Sprintf("/index/%s/query", index)) req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) if err != nil { return nil, err @@ -1320,7 +1325,8 @@ type InternalClient interface { Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) - ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error ImportK(ctx context.Context, index, frame string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error diff --git a/client_test.go b/client_test.go index be3505acd..88dc3accb 100644 --- a/client_test.go +++ b/client_test.go @@ -150,7 +150,7 @@ func TestClient_MultiNode(t *testing.T) { Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN), Remote: false, } - result, err := client[0].ExecuteQuery(context.Background(), "i", queryRequest) + result, err := client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestClient_MultiNode(t *testing.T) { hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) - result, err = client[0].ExecuteQuery(context.Background(), "i", queryRequest) + result, err = client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } @@ -188,11 +188,11 @@ func TestClient_MultiNode(t *testing.T) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } - result1, err := client[1].ExecuteQuery(context.Background(), "i", queryRequest) + result1, err := client[1].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } - result2, err := client[2].ExecuteQuery(context.Background(), "i", queryRequest) + result2, err := client[2].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } diff --git a/ctl/bench.go b/ctl/bench.go index 9e37fb704..b9743bb5e 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -95,7 +95,7 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalCl Query: fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID), Remote: false, } - if _, err := client.ExecuteQuery(ctx, cmd.Index, queryRequest); err != nil { + if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil { return err } } diff --git a/executor.go b/executor.go index 62bc0e8c8..59e40511d 100644 --- a/executor.go +++ b/executor.go @@ -1490,8 +1490,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * Remote: true, } - ctx = context.WithValue(ctx, "uri", node.URI) - pb, err := e.client.ExecuteQuery(ctx, index, pbreq) + pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq) if err != nil { return nil, err } diff --git a/fragment.go b/fragment.go index 448ba3202..7168f4443 100644 --- a/fragment.go +++ b/fragment.go @@ -1924,7 +1924,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := clients[i].ExecuteQuery(context.Background(), f.Index(), queryRequest) + _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) if err != nil { return err } From 420631e748925d8f3542fc734ab6b5f809629d83 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sun, 6 May 2018 20:36:52 -0700 Subject: [PATCH 296/311] remove last vestiges of passing URI via context this chould be safe as context.WithValue doesn't seem to appear anywhere else in Pilosa --- client.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/client.go b/client.go index 8950e8406..314f7d4e2 100644 --- a/client.go +++ b/client.go @@ -88,7 +88,7 @@ func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[st // maxSliceByIndex returns the number of slices on a server by index. func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.clientURI(ctx), "/slices/max") + u := uriPathToURL(c.defaultURI, "/slices/max") // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -1137,16 +1137,6 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto return nil } -func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI { - clientURI := c.defaultURI - if contextURI, ok := ctx.Value("uri").(*URI); ok { - clientURI = contextURI - } else if contextURI, ok := ctx.Value("uri").(URI); ok { - clientURI = &contextURI - } - return clientURI -} - // Bit represents the location of a single bit. type Bit struct { RowID uint64 From b2eb8f02ee2395cfe6a5f3712ca2ea91bc0e7ee1 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 7 May 2018 12:42:39 -0700 Subject: [PATCH 297/311] fix QueryNode comment --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 314f7d4e2..5997cf4ef 100644 --- a/client.go +++ b/client.go @@ -223,7 +223,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl return a, nil } -// QueryNode executes query against the index. +// Query executes query against the index. func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } From f5a4fd82b1d417ca05b3769b934002554be1b90a Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 8 May 2018 17:10:36 -0500 Subject: [PATCH 298/311] Wrap errors in ctl/*.go --- ctl/backup.go | 12 ++++++------ ctl/bench.go | 4 ++-- ctl/bench_test.go | 3 ++- ctl/check.go | 15 ++++++++------- ctl/check_test.go | 2 +- ctl/common.go | 5 +++-- ctl/export.go | 11 ++++++----- ctl/import.go | 24 ++++++++++++------------ ctl/inspect.go | 9 +++++---- ctl/restore.go | 8 ++++---- 10 files changed, 49 insertions(+), 44 deletions(-) diff --git a/ctl/backup.go b/ctl/backup.go index 1b16391e0..54a760695 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -16,12 +16,12 @@ package ctl import ( "context" - "errors" "io" "os" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // BackupCommand represents a command for backing up a view. @@ -60,26 +60,26 @@ func (cmd *BackupCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } // Open output file. f, err := os.Create(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "creating file") } defer f.Close() // Begin streaming backup. if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return err + return errors.Wrap(err, "backing up") } // Sync & close file to ensure durability. if err := f.Sync(); err != nil { - return err + return errors.Wrap(err, "syncing") } else if err = f.Close(); err != nil { - return err + return errors.Wrap(err, "closing file") } return nil diff --git a/ctl/bench.go b/ctl/bench.go index b9743bb5e..425894738 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -16,7 +16,6 @@ package ctl import ( "context" - "errors" "fmt" "io" "math/rand" @@ -25,6 +24,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // BenchCommand represents a command for benchmarking index operations. @@ -58,7 +58,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } switch cmd.Op { diff --git a/ctl/bench_test.go b/ctl/bench_test.go index c3014e6ca..4790ccb44 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pkg/errors" ) func TestBenchCommand_InvalidOption(t *testing.T) { @@ -31,7 +32,7 @@ func TestBenchCommand_InvalidOption(t *testing.T) { cm := NewBenchCommand(stdin, stdout, stderr) err := cm.Run(context.Background()) - if err != pilosa.ErrHostRequired { + if errors.Cause(err) != pilosa.ErrHostRequired { t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err) } diff --git a/ctl/check.go b/ctl/check.go index 34b7033fc..04f947eb4 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -24,6 +24,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // CheckCommand represents a command for performing consistency checks on data files. @@ -48,17 +49,17 @@ func (cmd *CheckCommand) Run(ctx context.Context) error { switch filepath.Ext(path) { case "": if err := cmd.checkBitmapFile(path); err != nil { - return err + return errors.Wrap(err, "checking bitmap") } case ".cache": if err := cmd.checkCacheFile(path); err != nil { - return err + return errors.Wrap(err, "checking cache") } case ".snapshotting": if err := cmd.checkSnapshotFile(path); err != nil { - return err + return errors.Wrap(err, "checking snapshot") } } } @@ -71,26 +72,26 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error { // Open file handle. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() fi, err := f.Stat() if err != nil { - return err + return errors.Wrap(err, "statting file") } // Memory map the file. data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { - return err + return errors.Wrap(err, "mmapping") } defer syscall.Munmap(data) // Attach the mmap file to the bitmap. bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { - return err + return errors.Wrap(err, "unmarshalling") } // Perform consistency check. diff --git a/ctl/check_test.go b/ctl/check_test.go index 33a19feb3..56d70f97a 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -85,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) { var buf bytes.Buffer io.Copy(&buf, r) - if !strings.HasPrefix(err.Error(), "invalid roaring file") { + if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: invalid roaring file") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) } // Todo: need correct roaring file for happy path diff --git a/ctl/common.go b/ctl/common.go index 3629e233e..11042a704 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -19,6 +19,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" "github.com/spf13/pflag" ) @@ -42,7 +43,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath) if err != nil { - return nil, err + return nil, errors.Wrap(err, "loading keypair") } TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, @@ -51,7 +52,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error } client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting internal client") } return client, err } diff --git a/ctl/export.go b/ctl/export.go index 8c219a078..546d1ae1b 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -22,6 +22,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // ExportCommand represents a command for bulk exporting data from a server. @@ -68,7 +69,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { if cmd.Path != "" { f, err := os.Create(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "creating file") } defer f.Close() @@ -78,7 +79,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } // Determine slice count. @@ -90,21 +91,21 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } if err != nil { - return err + return errors.Wrap(err, "getting slice count") } // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { - return err + return errors.Wrap(err, "exporting") } } // Close writer, if applicable. if w, ok := w.(io.Closer); ok { if err := w.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } } diff --git a/ctl/import.go b/ctl/import.go index c3c117eb3..004e47e96 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -17,7 +17,6 @@ package ctl import ( "context" "encoding/csv" - "errors" "fmt" "io" "log" @@ -28,6 +27,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // ImportCommand represents a command for bulk importing data. @@ -94,14 +94,14 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } cmd.Client = client if cmd.CreateSchema { err := cmd.ensureSchema(ctx) if err != nil { - return err + return errors.Wrap(err, "ensuring schema") } } @@ -153,7 +153,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { // Open file for reading. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() @@ -173,7 +173,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { if err == io.EOF { break } else if err != nil { - return err + return errors.Wrap(err, "reading") } // Ignore blank rows. @@ -243,7 +243,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err logger.Printf("importing slice: %d, n=%d", slice, len(bits)) if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil { - return err + return errors.Wrap(err, "importing") } } @@ -260,7 +260,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { // Open file for reading. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() @@ -280,7 +280,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { if err == io.EOF { break } else if err != nil { - return err + return errors.Wrap(err, "reading") } // Ignore blank rows. @@ -340,7 +340,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er logger.Printf("importing keys: n=%d", len(bits)) if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { - return err + return errors.Wrap(err, "importing keys") } return nil @@ -356,7 +356,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er // Open file for reading. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() @@ -376,7 +376,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er if err == io.EOF { break } else if err != nil { - return err + return errors.Wrap(err, "reading") } // Ignore blank rows. @@ -437,7 +437,7 @@ func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.F logger.Printf("importing slice: %d, n=%d", slice, len(vals)) if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil { - return err + return errors.Wrap(err, "importing values") } } diff --git a/ctl/inspect.go b/ctl/inspect.go index f5825c35a..0c38c3fb5 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // InspectCommand represents a command for inspecting fragment data files. @@ -49,19 +50,19 @@ func (cmd *InspectCommand) Run(ctx context.Context) error { // Open file handle. f, err := os.Open(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() fi, err := f.Stat() if err != nil { - return err + return errors.Wrap(err, "statting file") } // Memory map the file. data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { - return err + return errors.Wrap(err, "mmapping") } defer syscall.Munmap(data) @@ -70,7 +71,7 @@ func (cmd *InspectCommand) Run(ctx context.Context) error { fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { - return err + return errors.Wrap(err, "unmarshalling") } fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) diff --git a/ctl/restore.go b/ctl/restore.go index 89ee8ac8d..ef863096e 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -16,12 +16,12 @@ package ctl import ( "context" - "errors" "io" "os" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // RestoreCommand represents a command for restoring a frame from a backup. @@ -60,19 +60,19 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } // Open backup file. f, err := os.Open(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() // Restore backup file to the cluster. if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return err + return errors.Wrap(err, "restoring") } return nil From f0b6fa4ee80d9e48040b54b1ef9fa63575107ba7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 8 May 2018 17:20:06 -0500 Subject: [PATCH 299/311] Wrap errors in server/ --- server/server.go | 14 +++++++------- server/server_test.go | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/server/server.go b/server/server.go index 8fd699ce5..4038d12f2 100644 --- a/server/server.go +++ b/server/server.go @@ -105,18 +105,18 @@ func (m *Command) Start() (err error) { // SetupServer err = m.SetupServer() if err != nil { - return err + return errors.Wrap(err, "setting up server") } // SetupNetworking err = m.SetupNetworking() if err != nil { - return err + return errors.Wrap(err, "setting up networking") } // Initialize server. if err = m.Server.Open(); err != nil { - return fmt.Errorf("server.Open: %v", err) + return errors.Wrap(err, "opening server") } m.logger.Printf("Listening as %s\n", m.Server.URI) @@ -271,7 +271,7 @@ func (m *Command) SetupNetworking() error { for _, address := range m.Config.Cluster.Hosts { uri, err := pilosa.NewURIFromAddress(address) if err != nil { - return err + return errors.Wrap(err, "getting URI") } m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{ URI: *uri, @@ -287,7 +287,7 @@ func (m *Command) SetupNetworking() error { gossipPort, err := strconv.Atoi(m.Config.Gossip.Port) if err != nil { - return err + return errors.Wrap(err, "parsing port") } // get the host portion of addr to use for binding @@ -298,7 +298,7 @@ func (m *Command) SetupNetworking() error { } else { transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { - return err + return errors.Wrap(err, "getting transport") } } @@ -312,7 +312,7 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.EventReceiver = gossipEventReceiver gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) if err != nil { - return err + return errors.Wrap(err, "getting memberset") } m.Server.Cluster.MemberSet = gossipMemberSet m.Server.Broadcaster = m.Server diff --git a/server/server_test.go b/server/server_test.go index 75e0e2242..512ce5c11 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -32,6 +32,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" + "github.com/pkg/errors" ) // Ensure program can process queries and maintain consistency. @@ -52,7 +53,7 @@ func TestMain_Set_Quick(t *testing.T) { // Execute SetBit() commands. for _, cmd := range cmds { - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && errors.Cause(err) != pilosa.ErrIndexExists { t.Fatal(err) } if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { From 0244f4b51d1a507f790ee90eabd66b9183201bb4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 9 May 2018 01:27:42 +0300 Subject: [PATCH 300/311] Added dataDir field to Server --- server.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/server.go b/server.go index 86f14c8b3..a74ae90f3 100644 --- a/server.go +++ b/server.go @@ -77,6 +77,7 @@ type Server struct { maxWritesPerRequest int defaultClient InternalClient + dataDir string } // ServerOption is a functional option type for pilosa.Server @@ -98,8 +99,7 @@ func OptServerReplicaN(n int) ServerOption { func OptServerDataDir(dir string) ServerOption { return func(s *Server) error { - s.Cluster.Path = dir - s.Holder.Path = dir + s.dataDir = dir return nil } } @@ -231,21 +231,16 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } - path, err := expandDirName(s.Holder.Path) + path, err := expandDirName(s.dataDir) if err != nil { return nil, err } - s.Holder.Path = path + s.Holder.Path = path s.Holder.Logger = s.logger s.Holder.Stats.SetLogger(s.logger) - path, err = expandDirName(s.Cluster.Path) - if err != nil { - return nil, err - } s.Cluster.Path = path - s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder From 68924fe0adb043166f8aaf7ef393d6fa1403ce77 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 8 May 2018 17:40:15 -0500 Subject: [PATCH 301/311] Wrap errors in client.go --- client.go | 126 +++++++++++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/client.go b/client.go index 5997cf4ef..7403a7680 100644 --- a/client.go +++ b/client.go @@ -19,7 +19,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -35,6 +34,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" ) // ClientOptions represents the configuration for a InternalHTTPClient @@ -58,7 +58,7 @@ func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTT uri, err := NewURIFromAddress(host) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting URI") } client := NewInternalHTTPClientFromURI(uri, remoteClient) @@ -93,7 +93,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -101,7 +101,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -126,7 +126,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { // Build request. req, err := http.NewRequest("GET", u, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -134,7 +134,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -154,14 +154,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt Options: opt, }) if err != nil { - return err + return errors.Wrap(err, "encoding request") } // Create URL & HTTP request. u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") @@ -171,14 +171,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } // Handle response based on status code. @@ -201,7 +201,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -209,7 +209,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -239,14 +239,14 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri // Encode request object. buf, err := proto.Marshal(queryRequest) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshaling") } // Create HTTP request. u := uri.Path(fmt.Sprintf("/index/%s/query", index)) req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) @@ -257,14 +257,14 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body and unmarshal response. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, errors.Wrap(err, "reading") } else if resp.StatusCode != http.StatusOK { return nil, errors.New(string(body)) } @@ -398,7 +398,7 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b u := nodePathToURL(node, "/import") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") @@ -408,14 +408,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body and unmarshal response. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } else if resp.StatusCode != http.StatusOK { return errors.New(string(body)) } @@ -486,7 +486,7 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu u := nodePathToURL(node, "/import-value") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") @@ -496,14 +496,14 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body and unmarshal response. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } else if resp.StatusCode != http.StatusOK { return errors.New(string(body)) } @@ -564,7 +564,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde // Generate HTTP request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Accept", "text/csv") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -572,7 +572,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -583,7 +583,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde // Copy body to writer. if _, err := io.Copy(w, resp.Body); err != nil { - return err + return errors.Wrap(err, "copying") } return nil @@ -618,13 +618,13 @@ func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, f // Backup every slice to the tar file. for i := uint64(0); i <= maxSlices[index]; i++ { if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil { - return err + return errors.Wrap(err, "backing up slice") } } // Close tar file. if err := tw.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } return nil @@ -644,9 +644,9 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, // Read entire buffer to determine file size. data, err := ioutil.ReadAll(r) if err != nil { - return err + return errors.Wrap(err, "reading") } else if err := r.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } // Write slice file header. @@ -656,12 +656,12 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, Size: int64(len(data)), ModTime: time.Now(), }); err != nil { - return err + return errors.Wrap(err, "writing header") } // Write buffer to file. if _, err := tw.Write(data); err != nil { - return fmt.Errorf("write buffer: %s", err) + return errors.Wrap(err, "writing buffer") } return nil @@ -711,7 +711,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -719,7 +719,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } // Return error if status is not OK. @@ -751,7 +751,7 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index if err == io.EOF { return nil } else if err != nil { - return err + return errors.Wrap(err, "opening") } // Parse slice from entry name. @@ -763,12 +763,12 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index // Read file into buffer. var buf bytes.Buffer if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil { - return err + return errors.Wrap(err, "copying") } // Restore file to all nodes that own it. if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil { - return err + return errors.Wrap(err, "restoring") } } } @@ -794,14 +794,14 @@ func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, i // Build request. req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("User-Agent", "pilosa/"+Version) resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } resp.Body.Close() @@ -825,14 +825,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin Options: opt, }) if err != nil { - return err + return errors.Wrap(err, "marshaling") } // Create URL & HTTP request. u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") @@ -842,14 +842,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } // Handle response based on status code. @@ -873,7 +873,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram // Build request. req, err := http.NewRequest("POST", u.String(), nil) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -881,7 +881,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } resp.Body.Close() @@ -899,7 +899,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame)) req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -907,7 +907,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -924,7 +924,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string // Decode response. var rsp getFrameViewsResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Views, nil } @@ -943,7 +943,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -951,7 +951,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -967,7 +967,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v // Decode response object. var rsp getFragmentBlocksResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Blocks, nil } @@ -982,13 +982,13 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s Block: uint64(block), }) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "marshaling") } u := uriPathToURL(c.defaultURI, "/fragment/block/data") req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/protobuf") req.Header.Set("Content-Length", strconv.Itoa(len(buf))) @@ -997,7 +997,7 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -1013,9 +1013,9 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s // Decode response object. var rsp internal.BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "reading") } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "unmarhsaling") } return rsp.RowIDs, rsp.ColumnIDs, nil } @@ -1027,13 +1027,13 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // Encode request. buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshaling") } // Build request. req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -1041,7 +1041,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -1055,7 +1055,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // Decode response object. var rsp postIndexAttrDiffResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Attrs, nil } @@ -1067,13 +1067,13 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin // Encode request. buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshaling") } // Build request. req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -1081,7 +1081,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -1097,7 +1097,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin // Decode response object. var rsp postFrameAttrDiffResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Attrs, nil } From f83911812173ea265f5de461d504bfc83eeb1975 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 8 May 2018 18:17:22 -0500 Subject: [PATCH 302/311] Wrap errors in executor --- executor.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/executor.go b/executor.go index 59e40511d..b6dc08d41 100644 --- a/executor.go +++ b/executor.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "errors" "fmt" "net/http" "sort" @@ -24,6 +23,7 @@ import ( "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) // DefaultFrame is the frame used if one is not specified. @@ -149,7 +149,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // executeCall executes a call. func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { if err := e.validateCallArgs(c); err != nil { - return nil, err + return nil, errors.Wrap(err, "validating args") } indexTag := fmt.Sprintf("index:%s", index) // Special handling for mutation and top-n calls. @@ -344,7 +344,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting column attrs") } bm.Attrs = attrs } else if err != nil { @@ -354,11 +354,11 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C if fr := idx.Frame(frame); fr != nil { rowID, _, err := c.UintArg(rowLabel) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting row") } attrs, err := fr.RowAttrStore().Attrs(rowID) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting row attrs") } bm.Attrs = attrs } @@ -400,7 +400,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return ValCount{}, err + return ValCount{}, errors.Wrap(err, "executing bitmap call") } filter = bm } @@ -425,7 +425,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { - return ValCount{}, err + return ValCount{}, errors.Wrap(err, "computing sum") } return ValCount{ Val: int64(vsum) + (int64(vcount) * field.Min), @@ -527,7 +527,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s // Execute original query. pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt) if err != nil { - return nil, err + return nil, errors.Wrap(err, "finding top results") } // If this call is against specific ids, or we didn't get results, @@ -544,7 +544,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt) if err != nil { - return nil, err + return nil, errors.Wrap(err, "retrieving full counts") } if n != 0 && int(n) < len(trimmedList) { @@ -883,7 +883,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * predicates, err := cond.IntSliceValue() if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting condition value") } // Only support two integers for the between operation. @@ -1573,7 +1573,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, // Start mapping across all primary owners. if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil { - return nil, err + return nil, errors.Wrap(err, "starting mapper") } // Iterate over all map responses and reduce. From 4452cd7260920f8892e361a065b209231cb72c1f Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 8 May 2018 18:29:04 -0500 Subject: [PATCH 303/311] Wrap errors in cluster.go --- cluster.go | 62 ++++++++++++++++++++-------------------- cluster_internal_test.go | 5 ++-- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/cluster.go b/cluster.go index b02bd50d2..c2062383a 100644 --- a/cluster.go +++ b/cluster.go @@ -721,7 +721,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // Determine if a node is being added or removed. action, diffNodeID, err := c.diff(to) if err != nil { - return nil, err + return nil, errors.Wrap(err, "diffing") } // Initialize the map with all the nodes in `to`. @@ -905,7 +905,7 @@ func (c *Cluster) Open() error { // Load topology file if it exists. if err := c.loadTopology(); err != nil { - return fmt.Errorf("load topology: %v", err) + return errors.Wrap(err, "loading topology") } c.ID = c.Topology.ClusterID @@ -1006,7 +1006,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { c.Logger.Printf("setStateAndBroadcast error: err=%s", err) } - return err + return errors.Wrap(err, "setting state") } // j.Run() runs in a goroutine because in the case where the @@ -1023,14 +1023,14 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { // Make sure j.Run() didn't return an error. if eg.Wait() != nil { - return err + return errors.Wrap(err, "running job") } c.Logger.Printf("received jobResult: %s", jobResult) switch jobResult { case ResizeJobStateDone: if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { - return err + return errors.Wrap(err, "completing finished job") } // Add/remove uri to/from the cluster. if j.action == ResizeJobActionRemove { @@ -1040,7 +1040,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { } case ResizeJobStateAborted: if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { - return err + return errors.Wrap(err, "completing aborted job") } } return nil @@ -1055,7 +1055,7 @@ func (c *Cluster) setStateAndBroadcast(state string) error { func (c *Cluster) sendTo(node *Node, msg proto.Message) error { if err := c.Broadcaster.SendTo(node, msg); err != nil { - return err + return errors.Wrap(err, "sending") } return nil } @@ -1124,7 +1124,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { j, err := c.generateResizeJobByAction(nodeAction) if err != nil { - return nil, err + return nil, errors.Wrap(err, "generating job") } c.Logger.Printf("generated ResizeJob: %d", j.ID) @@ -1171,7 +1171,7 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, for _, idx := range c.Holder.Indexes() { fragSources, err := c.fragSources(toCluster, idx) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting sources") } for id, sources := range fragSources { @@ -1223,7 +1223,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { - return err + return errors.Wrap(err, "merging cluster status") } c.Logger.Printf("MergeClusterStatus done, start goroutine") @@ -1248,7 +1248,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Sync the schema received in the resize instruction. c.Logger.Printf("Holder ApplySchema") if err := c.Holder.ApplySchema(instr.Schema); err != nil { - return err + return errors.Wrap(err, "applying schema") } // Create a client for calling remote nodes. @@ -1269,13 +1269,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Create view. v, err := f.CreateViewIfNotExists(src.View) if err != nil { - return err + return errors.Wrap(err, "creating view") } // Create the local fragment. frag, err := v.CreateFragmentIfNotExists(src.Slice) if err != nil { - return err + return errors.Wrap(err, "creating fragment") } // Stream slice from remote node. @@ -1291,7 +1291,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err if err == ErrFragmentNotFound { return nil } - return err + return errors.Wrap(err, "retrieving slice") } else if rd == nil { return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) } @@ -1304,7 +1304,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } return nil }(); err != nil { - return err + return errors.Wrap(err, "copying remote slice") } } return nil @@ -1438,7 +1438,7 @@ func (j *ResizeJob) Run() error { err := j.distributeResizeInstructions() if err != nil { j.result <- ResizeJobStateAborted - return err + return errors.Wrap(err, "distributing instructions") } return nil } @@ -1475,7 +1475,7 @@ func (j *ResizeJob) distributeResizeInstructions() error { } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { - return err + return errors.Wrap(err, "sending instruction") } } return nil @@ -1581,16 +1581,16 @@ func (c *Cluster) loadTopology() error { c.Topology = NewTopology() return nil } else if err != nil { - return err + return errors.Wrap(err, "reading file") } var pb internal.Topology if err := proto.Unmarshal(buf, &pb); err != nil { - return err + return errors.Wrap(err, "unmarshalling") } top, err := decodeTopology(&pb) if err != nil { - return err + return errors.Wrap(err, "decoding") } c.Topology = top @@ -1601,13 +1601,13 @@ func (c *Cluster) loadTopology() error { func (c *Cluster) saveTopology() error { if err := os.MkdirAll(c.Path, 0777); err != nil { - return err + return errors.Wrap(err, "creating directory") } if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { - return err + return errors.Wrap(err, "marshalling") } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { - return err + return errors.Wrap(err, "writing file") } return nil } @@ -1704,7 +1704,7 @@ func (c *Cluster) nodeJoin(node *Node) error { } if err := c.AddNode(node); err != nil { - return err + return errors.Wrap(err, "adding node for agreement") } // Only change to normal if there is no existing data. Otherwise, @@ -1742,7 +1742,7 @@ func (c *Cluster) nodeJoin(node *Node) error { // If the holder does not yet contain data, go ahead and add the node. if ok, err := c.Holder.HasData(); !ok && err == nil { if err := c.AddNode(node); err != nil { - return err + return errors.Wrap(err, "adding node") } return c.setStateAndBroadcast(ClusterStateNormal) } else if err != nil { @@ -1752,7 +1752,7 @@ func (c *Cluster) nodeJoin(node *Node) error { // If the cluster has data, we need to change to RESIZING and // kick off the resizing process. if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { - return err + return errors.Wrap(err, "broadcasting state") } c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} @@ -1784,7 +1784,7 @@ func (c *Cluster) NodeLeave(node *Node) error { _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) if err != nil { - return err + return errors.Wrap(err, "generating job") } return c.nodeLeave(node) @@ -1802,7 +1802,7 @@ func (c *Cluster) nodeLeave(node *Node) error { // If the holder does not yet contain data, go ahead and remove the node. if ok, err := c.Holder.HasData(); !ok && err == nil { if err := c.RemoveNode(n); err != nil { - return err + return errors.Wrap(err, "removing node") } return c.setStateAndBroadcast(ClusterStateNormal) } else if err != nil { @@ -1812,7 +1812,7 @@ func (c *Cluster) nodeLeave(node *Node) error { // If the cluster has data then change state to RESIZING and // kick off the resizing process. if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { - return err + return errors.Wrap(err, "broadcasting state") } c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} @@ -1836,7 +1836,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { // Add all nodes from the coordinator. for _, node := range officialNodes { if err := c.AddNode(node); err != nil { - return err + return errors.Wrap(err, "adding node") } } @@ -1857,7 +1857,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { for _, nodeID := range nodeIDsToRemove { if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { - return err + return errors.Wrap(err, "removing node") } } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index da9989760..6b1f6e72b 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -17,6 +17,7 @@ package pilosa import ( "io/ioutil" "reflect" + "strings" "testing" "github.com/pilosa/pilosa/internal" @@ -243,8 +244,8 @@ func TestFragSources(t *testing.T) { actual, err := (test.from).fragSources(test.to, test.idx) if test.err != "" { - if err.Error() != test.err { - t.Fatalf("expected error: %s", test.err) + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("expected error: %s, got: %s", test.err, err.Error()) } } else { if err != nil { From 4f51373734541cb246005b23fbbe3b1a11cfbfc3 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 8 May 2018 19:43:36 -0500 Subject: [PATCH 304/311] Wrap errors in holder --- holder.go | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/holder.go b/holder.go index f048aac44..5c4afc211 100644 --- a/holder.go +++ b/holder.go @@ -96,19 +96,19 @@ func (h *Holder) Open() error { h.Logger.Printf("open holder path: %s", h.Path) if err := os.MkdirAll(h.Path, 0777); err != nil { - return err + return errors.Wrap(err, "creating directory") } // Open path to read all index directories. f, err := os.Open(h.Path) if err != nil { - return err + return errors.Wrap(err, "opening directory") } defer f.Close() fis, err := f.Readdir(0) if err != nil { - return err + return errors.Wrap(err, "reading directory") } for _, fi := range fis { @@ -123,7 +123,7 @@ func (h *Holder) Open() error { h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { - return err + return errors.Wrap(err, "opening index") } if err := index.Open(); err != nil { if err == ErrName { @@ -158,7 +158,7 @@ func (h *Holder) Close() error { for _, index := range h.indexes { if err := index.Close(); err != nil { - return err + return errors.Wrap(err, "closing index") } } return nil @@ -245,20 +245,20 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { opt := IndexOptions{} idx, err := h.CreateIndexIfNotExists(index.Name, opt) if err != nil { - return err + return errors.Wrap(err, "creating index") } // Create frames that don't exist. for _, f := range index.Frames { opt := decodeFrameOptions(f.Meta) frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) if err != nil { - return err + return errors.Wrap(err, "creating frame") } // Create views that don't exist. for _, v := range f.Views { _, err := frame.CreateViewIfNotExists(v) if err != nil { - return err + return errors.Wrap(err, "creating view") } } } @@ -347,11 +347,11 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { // Otherwise create a new index. index, err := h.newIndex(h.IndexPath(name), name) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating") } if err := index.Open(); err != nil { - return nil, err + return nil, errors.Wrap(err, "opening") } // Update options. @@ -387,12 +387,12 @@ func (h *Holder) DeleteIndex(name string) error { // Close index. if err := index.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } // Delete index directory. if err := os.RemoveAll(h.IndexPath(name)); err != nil { - return err + return errors.Wrap(err, "removing directory") } // Remove reference. @@ -528,7 +528,7 @@ func (h *Holder) loadNodeID() (string, error) { nodeID := "" h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { - return "", err + return "", errors.Wrap(err, "creating directory") } nodeIDBytes, err := ioutil.ReadFile(idPath) @@ -538,10 +538,10 @@ func (h *Holder) loadNodeID() (string, error) { nodeID = uuid.NewV4().String() err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) if err != nil { - return "", err + return "", errors.Wrap(err, "writing file") } } else if err != nil { - return "", err + return "", errors.Wrap(err, "reading file") } return nodeID, nil @@ -667,7 +667,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // Read block checksums. blks, err := idx.ColumnAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "getting blocks") } s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag}) @@ -679,7 +679,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // Skip update and recomputation if no attributes have changed. m, err := client.ColumnAttrDiff(context.Background(), index, blks) if err != nil { - return err + return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } @@ -687,13 +687,13 @@ func (s *HolderSyncer) syncIndex(index string) error { // Update local copy. if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil { - return err + return errors.Wrap(err, "setting attrs") } // Recompute blocks. blks, err = idx.ColumnAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "recomputing blocks") } } @@ -713,7 +713,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Read block checksums. blks, err := f.RowAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "getting blocks") } s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) @@ -727,7 +727,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { if err == ErrFrameNotFound { continue // frame not created remotely yet, skip } else if err != nil { - return err + return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } @@ -735,13 +735,13 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { - return err + return errors.Wrap(err, "setting attrs") } // Recompute blocks. blks, err = f.RowAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "recomputing blocks") } } @@ -759,13 +759,13 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Ensure view exists locally. v, err := f.CreateViewIfNotExists(view) if err != nil { - return err + return errors.Wrap(err, "creating view") } // Ensure fragment exists locally. frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { - return err + return errors.Wrap(err, "creating fragment") } // Sync fragments together. @@ -777,7 +777,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err RemoteClient: s.RemoteClient, } if err := fs.SyncFragment(); err != nil { - return err + return errors.Wrap(err, "syncing") } return nil @@ -827,7 +827,7 @@ func (c *HolderCleaner) CleanHolder() error { } // Delete fragment. if err := view.DeleteFragment(fragSlice); err != nil { - return err + return errors.Wrap(err, "deleting fragment") } } } From 43ec69d08b31f6bd511701067d53b8777cf0b940 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 9 May 2018 09:32:18 -0500 Subject: [PATCH 305/311] API.URI was not being used. removed it. --- api.go | 5 ++--- client_test.go | 8 -------- ctl/backup_test.go | 6 +----- ctl/export_test.go | 9 ++------- ctl/import_test.go | 11 +---------- ctl/restore_test.go | 6 +----- server.go | 1 - test/handler.go | 7 ------- 8 files changed, 7 insertions(+), 46 deletions(-) diff --git a/api.go b/api.go index 4fe08f9a5..691e076ff 100644 --- a/api.go +++ b/api.go @@ -46,7 +46,6 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster - URI URI RemoteClient *http.Client Logger Logger } @@ -294,7 +293,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { - api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return ErrClusterDoesNotOwnSlice } @@ -957,7 +956,7 @@ func (api *API) LongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { - api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return nil, nil, ErrClusterDoesNotOwnSlice } diff --git a/client_test.go b/client_test.go index 88dc3accb..e2b806412 100644 --- a/client_test.go +++ b/client_test.go @@ -37,7 +37,6 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { for i := 0; i < numNodes; i++ { hldr[i] = test.MustOpenHolder() server[i] = test.NewServer() - server[i].Handler.API.URI = server[i].HostURI() server[i].Handler.API.Cluster = c server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() server[i].Handler.API.Holder = hldr[i].Holder @@ -218,7 +217,6 @@ func TestClient_Import(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -269,7 +267,6 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -318,7 +315,6 @@ func TestClient_ImportValue(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -386,7 +382,6 @@ func TestClient_BackupRestore(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -452,7 +447,6 @@ func TestClient_BackupInverseView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -489,7 +483,6 @@ func TestClient_BackupInvalidView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -518,7 +511,6 @@ func TestClient_FragmentBlocks(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.API.URI = s.HostURI() s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/backup_test.go b/ctl/backup_test.go index d80b72475..a073bc8a8 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -46,11 +46,7 @@ func TestBackupCommand_Run(t *testing.T) { s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.API.URI = *uri + s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/export_test.go b/ctl/export_test.go index b414f711f..48a90a065 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -59,11 +59,7 @@ func TestExportCommand_Run(t *testing.T) { defer hldr.Close() s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.API.URI = *uri + s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -75,8 +71,7 @@ func TestExportCommand_Run(t *testing.T) { cm.Index = "i" cm.Frame = "f" cm.View = pilosa.ViewStandard - err = cm.Run(context.Background()) - if err != nil { + if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } } diff --git a/ctl/import_test.go b/ctl/import_test.go index 6cbc8271a..08eea8323 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -65,11 +65,7 @@ func TestImportCommand_Run(t *testing.T) { defer hldr.Close() s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.API.URI = *uri + s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -103,12 +99,7 @@ func TestImportCommand_RunValue(t *testing.T) { defer hldr.Close() s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/restore_test.go b/ctl/restore_test.go index 04cb30415..896db57c5 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -48,11 +48,7 @@ func TestRestoreCommand_Run(t *testing.T) { s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.API.URI = *uri + s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/server.go b/server.go index 43caa9558..c8b631580 100644 --- a/server.go +++ b/server.go @@ -300,7 +300,6 @@ func (s *Server) Open() error { s.handler.API.Broadcaster = s.Broadcaster s.handler.API.BroadcastHandler = s s.handler.API.StatusHandler = s - s.handler.API.URI = s.URI s.handler.API.Cluster = s.Cluster // Initialize Holder. diff --git a/test/handler.go b/test/handler.go index c325d3243..50b081812 100644 --- a/test/handler.go +++ b/test/handler.go @@ -75,13 +75,6 @@ func NewServer() *Server { } s.Server = httptest.NewServer(s.Handler.Handler) - // Update handler to use hostname. - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - panic(err) - } - s.Handler.API.URI = *uri - // Handler test messages can no-op. s.Handler.API.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler From e4c11a28b118516458cdbadd9eaeef8d860e4fe7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 9 May 2018 10:42:11 -0500 Subject: [PATCH 306/311] make sure gossipMemberSet.Logger is set during server setup --- server/server.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 3098cddeb..a3824cbc3 100644 --- a/server/server.go +++ b/server/server.go @@ -300,10 +300,19 @@ func (m *Command) SetupNetworking() error { gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) m.Server.Cluster.EventReceiver = gossipEventReceiver - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) + gossipMemberSet, err := gossip.NewGossipMemberSet( + m.Server.NodeID, + m.Server.URI.Host(), + m.Config.Gossip, + gossipEventReceiver, + m.Server, + gossip.WithLogger(m.logger.Logger()), + gossip.WithTransport(transport), + ) if err != nil { return err } + gossipMemberSet.Logger = m.logger m.Server.Cluster.MemberSet = gossipMemberSet m.Server.Broadcaster = m.Server m.Server.BroadcastReceiver = gossipMemberSet From b74e5e7ec703ad3c4de1b0e9be136d9aac854318 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 9 May 2018 10:46:16 -0500 Subject: [PATCH 307/311] Update error message --- holder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holder.go b/holder.go index 5c4afc211..2475d61d0 100644 --- a/holder.go +++ b/holder.go @@ -777,7 +777,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err RemoteClient: s.RemoteClient, } if err := fs.SyncFragment(); err != nil { - return errors.Wrap(err, "syncing") + return errors.Wrap(err, "syncing fragment") } return nil From 5e043919ae9ce898b4937ce0bba39aee82b923e5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 9 May 2018 10:48:49 -0500 Subject: [PATCH 308/311] default GossipMemberSet.Logger to NopLogger --- gossip/gossip.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 25dd40cc4..49b66d8a2 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -169,7 +169,9 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - g := &GossipMemberSet{} + g := &GossipMemberSet{ + Logger: pilosa.NopLogger, + } // options for _, opt := range options { From a436d4d32a68b2c25532c9d2aa8d109745fefcb7 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 9 May 2018 15:00:39 -0500 Subject: [PATCH 309/311] fixed deadlock in setcooridnator --- cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index b02bd50d2..8e342024e 100644 --- a/cluster.go +++ b/cluster.go @@ -307,16 +307,16 @@ func (c *Cluster) isCoordinator() bool { // nodes with its version of Cluster.Status. func (c *Cluster) SetCoordinator(n *Node) error { c.mu.Lock() - defer c.mu.Unlock() // Verify that the new Coordinator value matches // this node. if c.Node.ID != n.ID { + c.mu.Unlock() return fmt.Errorf("coordinator node does not match this node") } // Update IsCoordinator on all nodes (locally). _ = c.updateCoordinator(n) - + c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.Broadcaster.SendSync( &internal.UpdateCoordinatorMessage{ From 57261bda9f84251bd7ebe8c5a5cfdd01d3495e62 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 9 May 2018 15:52:02 -0500 Subject: [PATCH 310/311] Fix spelling error --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 7403a7680..72225aef2 100644 --- a/client.go +++ b/client.go @@ -1015,7 +1015,7 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s if body, err := ioutil.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, errors.Wrap(err, "unmarhsaling") + return nil, nil, errors.Wrap(err, "unmarshalling") } return rsp.RowIDs, rsp.ColumnIDs, nil } From 9710ac978d79c92464b2c2308560e3c9f091d2ab Mon Sep 17 00:00:00 2001 From: Will James Date: Wed, 9 May 2018 21:30:49 -0400 Subject: [PATCH 311/311] Correct usage of ClearBit Usage of `ClearBit` was incorrectly shown as `SetBit`. --- docs/query-language.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 8ed0d5439..7d9f22886 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -160,8 +160,8 @@ Delete url value for repo 10. **Spec:** ``` -SetBit(, , , - [timestamp=TIMESTAMP]) +ClearBit(, , , + [timestamp=TIMESTAMP]) ``` **Description:**