From 8ceaef1eaaa96184dad0e84ed1860f0fcbd4cdfd Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 7 Mar 2017 14:53:37 -0600 Subject: [PATCH 01/50] fragment cache size is set on frame creation with a default value of 50,000 --- fragment.go | 32 ++++++++++++++++++++++++++++---- fragment_test.go | 44 +++++++++++++++++++++++++++++++++++++++++--- frame.go | 47 ++++++++++++++++++++++++++++++++++++++++++----- view.go | 15 +++++++++------ view_test.go | 4 ++-- 5 files changed, 122 insertions(+), 20 deletions(-) diff --git a/fragment.go b/fragment.go index 063b7e17c..fe04682dc 100644 --- a/fragment.go +++ b/fragment.go @@ -70,6 +70,7 @@ type Fragment struct { // Cache for bitmap counts. cacheType string // passed in by frame cache Cache + cacheSize int // Cache containing full bitmaps (not just counts). bitmapCache BitmapCache @@ -93,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64, cacheSize int) *Fragment { return &Fragment{ path: path, db: db, @@ -101,6 +102,7 @@ func NewFragment(path, db, frame, view string, slice uint64) *Fragment { view: view, slice: slice, cacheType: DefaultCacheType, + cacheSize: cacheSize, LogOutput: ioutil.Discard, MaxOpN: DefaultFragmentMaxOpN, @@ -217,21 +219,43 @@ func (f *Fragment) openStorage() error { } +// // It will be the one and only identifier after a package specifier. +// var testNameRegexp = regexp.MustCompile(`\.(Test[\p{L}_\p{N}]*)$`) + +// // Returns the name of the test function from the call stack. See +// // http://stackoverflow.com/q/35535635/149482 for another method. +// func GetTestName() string { +// pc := make([]uintptr, 32) +// n := runtime.Callers(0, pc) +// for i := 0; i < n; i++ { +// name := runtime.FuncForPC(pc[i]).Name() +// ms := testNameRegexp.FindStringSubmatch(name) +// if ms == nil { +// continue +// } +// return ms[1] +// } +// panic("test name could not be recovered") +// } + // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { case CacheTypeRanked: c := NewRankCache() - c.ThresholdLength = 50000 - c.ThresholdIndex = 45000 + c.ThresholdLength = f.cacheSize + c.ThresholdIndex = f.cacheSize - 500 f.cache = c case CacheTypeLRU: - f.cache = NewLRUCache(50000) + f.cache = NewLRUCache(f.cacheSize) default: return ErrInvalidCacheType } + // fmt.Println(GetTestName()) + // fmt.Printf("CACHE SIZE: %d\n", f.cacheSize) + // Read cache data from disk. path := f.CachePath() buf, err := ioutil.ReadFile(path) diff --git a/fragment_test.go b/fragment_test.go index b915bd449..ca6ffa3dc 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -277,6 +277,44 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { } } +// Ensure the fragment cache limit works +func TestFragment_TopN_CacheSize(t *testing.T) { + slice := uint64(0) + cacheLimit := 3 + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + f := &Fragment{ + Fragment: pilosa.NewFragment(file.Name(), "d", "f", slice, cacheLimit), + BitmapAttrStore: MustOpenAttrStore(), + } + f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + if err := f.Open(); err != nil { + panic(err) + } + defer f.Close() + + // Set bits on various bitmaps. + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) + f.MustSetBits(103, 8, 9, 10, 11, 12, 13) + f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.MustSetBits(105, 10, 11) + + // Retrieve top bitmaps. + if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { + t.Fatal(err) + } else if len(pairs) != cacheLimit { + t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } +} + // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) @@ -526,7 +564,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultFrameCache) if err := f.Open(); err != nil { b.Fatal(err) } @@ -587,7 +625,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultFrameCache), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -618,7 +656,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultFrameCache) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/frame.go b/frame.go index c70a07b68..be949b19d 100644 --- a/frame.go +++ b/frame.go @@ -20,6 +20,9 @@ const ( DefaultRowLabel = "id" DefaultCacheType = CacheTypeLRU DefaultInverseEnabled = false + + // Default ranked frame cache + DefaultFrameCache = 50000 ) // Frame represents a container for views. @@ -42,6 +45,9 @@ type Frame struct { cacheType string inverseEnabled bool + // Cache size for ranked frames + rankedCacheSize int + LogOutput io.Writer } @@ -62,9 +68,10 @@ func NewFrame(path, db, name string) (*Frame, error) { stats: NopStatsClient, - rowLabel: DefaultRowLabel, - cacheType: DefaultCacheType, - inverseEnabled: DefaultInverseEnabled, + rowLabel: DefaultRowLabel, + cacheType: DefaultCacheType, + inverseEnabled: DefaultInverseEnabled, + rankedCacheSize: DefaultFrameCache, LogOutput: ioutil.Discard, }, nil @@ -149,13 +156,42 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } +// SetRankedCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultFrameCache 50000 +func (f *Frame) SetRankedCacheSize(v int) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Ignore if no change occurred. + if v == 0 || f.rankedCacheSize == v { + return nil + } + + // Persist meta data to disk on change. + f.rankedCacheSize = v + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + +// RankedCacheSize returns the ranked frame cache size. +func (f *Frame) RankedCacheSize() int { + f.mu.Lock() + v := f.rankedCacheSize + f.mu.Unlock() + return v +} + // Options returns all options for this frame. func (f *Frame) Options() FrameOptions { f.mu.Lock() opt := FrameOptions{ RowLabel: f.rowLabel, - CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, + CacheSize: f.rankedCacheSize, } f.mu.Unlock() return opt @@ -559,8 +595,9 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` - CacheType string `json:"cacheType,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize int `json:"cacheSize,omitempty"` } // importBitSet represents slices of row and column ids. diff --git a/view.go b/view.go index 54dda6f56..0f554de92 100644 --- a/view.go +++ b/view.go @@ -30,6 +30,8 @@ type View struct { frame string name string + cacheSize int + // Fragments by slice. cacheType string // passed in by frame fragments map[uint64]*Fragment @@ -41,12 +43,13 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, db, frame, name string) *View { +func NewView(path, db, frame, name string, cacheSize int) *View { return &View{ - path: path, - db: db, - frame: frame, - name: name, + path: path, + db: db, + frame: frame, + name: name, + cacheSize: cacheSize, cacheType: DefaultCacheType, fragments: make(map[uint64]*Fragment), @@ -213,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.db, v.frame, v.name, slice) + frag := NewFragment(path, v.db, v.frame, v.name, slice, v.cacheSize) frag.cacheType = v.cacheType frag.LogOutput = v.LogOutput frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) diff --git a/view_test.go b/view_test.go index 87b3e5826..d10c90936 100644 --- a/view_test.go +++ b/view_test.go @@ -22,7 +22,7 @@ func NewView(db, frame, name string) *View { file.Close() v := &View{ - View: pilosa.NewView(file.Name(), db, frame, name), + View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), BitmapAttrStore: MustOpenAttrStore(), } v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore @@ -52,7 +52,7 @@ func (v *View) Reopen() error { return err } - v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name()) + v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore if err := v.Open(); err != nil { return err From 3b71ed1b2f1985b517c4a034b01997ba5cfff588 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 7 Mar 2017 15:15:03 -0600 Subject: [PATCH 02/50] make the fragment cache ThresholdIndex a function at 90% of ThresholdLength --- fragment.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index fe04682dc..0b1734593 100644 --- a/fragment.go +++ b/fragment.go @@ -43,6 +43,9 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 + + // Percentage of the ThresholdLength size to resort the cache at + ThresholdBufferPct = 0.9 ) const ( @@ -245,7 +248,7 @@ func (f *Fragment) openCache() error { case CacheTypeRanked: c := NewRankCache() c.ThresholdLength = f.cacheSize - c.ThresholdIndex = f.cacheSize - 500 + c.ThresholdIndex = int(float64(f.cacheSize) * ThresholdBufferPct) f.cache = c case CacheTypeLRU: f.cache = NewLRUCache(f.cacheSize) From 591c35c95f5e2f124da35e81c93ff7bb8da2cea1 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 7 Mar 2017 15:34:01 -0600 Subject: [PATCH 03/50] set the cache size on frame creation --- db.go | 4 ++++ fragment.go | 22 ---------------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/db.go b/db.go index ef95a54bf..3abff2a9c 100644 --- a/db.go +++ b/db.go @@ -391,6 +391,10 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { if opt.RowLabel != "" { f.rowLabel = opt.RowLabel } + if opt.CacheSize != 0 { + f.rankedCacheSize = opt.CacheSize + } + f.inverseEnabled = opt.InverseEnabled if err := f.saveMeta(); err != nil { f.Close() diff --git a/fragment.go b/fragment.go index 0b1734593..4f6362193 100644 --- a/fragment.go +++ b/fragment.go @@ -222,25 +222,6 @@ func (f *Fragment) openStorage() error { } -// // It will be the one and only identifier after a package specifier. -// var testNameRegexp = regexp.MustCompile(`\.(Test[\p{L}_\p{N}]*)$`) - -// // Returns the name of the test function from the call stack. See -// // http://stackoverflow.com/q/35535635/149482 for another method. -// func GetTestName() string { -// pc := make([]uintptr, 32) -// n := runtime.Callers(0, pc) -// for i := 0; i < n; i++ { -// name := runtime.FuncForPC(pc[i]).Name() -// ms := testNameRegexp.FindStringSubmatch(name) -// if ms == nil { -// continue -// } -// return ms[1] -// } -// panic("test name could not be recovered") -// } - // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. @@ -256,9 +237,6 @@ func (f *Fragment) openCache() error { return ErrInvalidCacheType } - // fmt.Println(GetTestName()) - // fmt.Printf("CACHE SIZE: %d\n", f.cacheSize) - // Read cache data from disk. path := f.CachePath() buf, err := ioutil.ReadFile(path) From 4e7c2af5722da2f8b80c22d417e6f3e712d699f9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 17 Mar 2017 11:28:17 -0500 Subject: [PATCH 04/50] fixed test to use ranked frame type --- fragment_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index ca6ffa3dc..56e2381e2 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -288,7 +288,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), "d", "f", slice, cacheLimit), + Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", slice, cacheLimit), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -305,13 +305,24 @@ func TestFragment_TopN_CacheSize(t *testing.T) { f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) f.MustSetBits(105, 10, 11) + f.RecalculateCache() + + p := []pilosa.Pair{ + {Key: 104, Count: 7}, + {Key: 103, Count: 6}, + {Key: 102, Count: 5}} + // {Key: 101, Count: 4}, + // {Key: 100, Count: 3}} + // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) != cacheLimit { t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { - t.Fatalf("unexpected pair(0): %v", pairs[0]) + t.Fatalf("unexpected pair(0): %v", pairs) + } else if !reflect.DeepEqual(pairs, p) { + t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) } } From 9321637d18327a21329ee5319fee1f83e6d531b4 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 17 Mar 2017 13:45:30 -0500 Subject: [PATCH 05/50] trim the ranked cache set on the 1st recalculate call --- cache.go | 1 + fragment_test.go | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cache.go b/cache.go index 3504806c6..23bc52c30 100644 --- a/cache.go +++ b/cache.go @@ -211,6 +211,7 @@ func (c *RankCache) recalculate() { c.rankings = rankings if len(c.rankings) > c.ThresholdIndex { c.ThresholdValue = rankings[c.ThresholdIndex].Count + c.rankings = c.rankings[0:c.ThresholdIndex] } else { c.ThresholdValue = 1 } diff --git a/fragment_test.go b/fragment_test.go index 56e2381e2..7f3d35add 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -309,16 +309,13 @@ func TestFragment_TopN_CacheSize(t *testing.T) { p := []pilosa.Pair{ {Key: 104, Count: 7}, - {Key: 103, Count: 6}, - {Key: 102, Count: 5}} - // {Key: 101, Count: 4}, - // {Key: 100, Count: 3}} + {Key: 103, Count: 6}} // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) - } else if len(pairs) != cacheLimit { - t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) + } else if len(pairs) > cacheLimit { + t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { From 7c2f6a9bae2f1de825a9ecbcc9d448d2e9488b48 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 7 Dec 2016 14:09:08 -0600 Subject: [PATCH 06/50] adds a basic gossip implementation using hashicorp/memberlist Changes Gossiper to NodeSet. Adds StaticNodeSet (for testing) and GossipNodeSet (for memberlist) implementations. Changes NodeSet interface to return a pilosa-specific generic instead of `NodeSet interface`. Removes NumMembers from interface (which is specific to memberlist). Implements a Messenger interface with which to send inter-node messages via NodeSet. The Pilosa implementation occurs in the GossipNodeSet. Implements the Messenger as an object on Server, Handler, and Index. Uses HealthStatus constants. Removes commented-out code. for gossip, make sure to bind to both host and port, and advertise those as well adjust messenger to work with the db schema logic add dependencies: hashicorp/memberlist, golang.org/x/sync add dependency: golang.org/x/net Uses `errgroup` to handle errors from broadcast messages. Marshals message one time instead of once for every node. Adds error handling for some errors that were being swallowed. --- cluster.go | 184 ++++++++++++++++++++++++++++++++++++++++++++++- cluster_test.go | 40 +++++++++++ config.go | 50 +++++++++++-- db.go | 5 +- executor_test.go | 4 +- frame.go | 21 +++++- glide.lock | 26 ++++++- glide.yaml | 5 ++ gossip.go | 165 ++++++++++++++++++++++++++++++++++++++++++ handler.go | 54 +++++++++++++- index.go | 28 +++++++- index_test.go | 27 +++++++ messenger.go | 73 +++++++++++++++++++ server.go | 27 +++---- 14 files changed, 682 insertions(+), 27 deletions(-) create mode 100644 gossip.go create mode 100644 messenger.go diff --git a/cluster.go b/cluster.go index beb33496b..8268fe508 100644 --- a/cluster.go +++ b/cluster.go @@ -1,8 +1,17 @@ package pilosa import ( + "bytes" "encoding/binary" + "fmt" "hash/fnv" + "io/ioutil" + "net/http" + "net/url" + + "golang.org/x/sync/errgroup" + + "github.com/gogo/protobuf/proto" ) const ( @@ -11,6 +20,10 @@ const ( // DefaultReplicaN is the default number of replicas per partition. DefaultReplicaN = 1 + + // HealthStatus is the return value of the /health endpoint for a node in the cluster. + HealthStatusUp = "UP" + HealthStatusDown = "DOWN" ) // Node represents a node in the cluster. @@ -81,7 +94,8 @@ func (a Nodes) Clone() []*Node { // Cluster represents a collection of nodes. type Cluster struct { - Nodes []*Node + Nodes []*Node + NodeSet NodeSet // Hashing algorithm used to assign partitions to nodes. Hasher Hasher @@ -102,6 +116,33 @@ func NewCluster() *Cluster { } } +// NodeSetHosts returns the list of host strings for NodeSet members +func (c *Cluster) NodeSetHosts() []string { + if c.NodeSet == nil { + return []string{} + } + a := make([]string, 0, len(c.NodeSet.Nodes())) + for _, m := range c.NodeSet.Nodes() { + a = append(a, m.Host) + } + return a +} + +// Health returns a list of nodes in the cluster along with each node's state (UP/DOWN). +func (c *Cluster) Health() map[string]string { + h := make(map[string]string) + for _, n := range c.Nodes { + h[n.Host] = HealthStatusDown + } + // we are assuming that NodeSetHosts is a subset of c.Nodes + for _, m := range c.NodeSetHosts() { + if _, ok := h[m]; ok { + h[m] = HealthStatusUp + } + } + return h +} + // NodeByHost returns a node reference by host. func (c *Cluster) NodeByHost(host string) *Node { for _, n := range c.Nodes { @@ -157,6 +198,21 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } +// NodeSet represents an interface to maintaining Node state. +type NodeSet interface { + // Returns a list of all Nodes in the cluster + Nodes() []*Node + + // Attempts to join a cluster having `nodes` as its existing members + Join(nodes []*Node) (int, error) + + // Open starts any network activity implemented by the NodeSet + Open() error + + // SetMessageHandler provides the NodeSet with a function to call on ReceiveMessage + SetMessageHandler(f func(proto.Message) error) +} + // Hasher represents an interface to hash integers into buckets. type Hasher interface { // Hashes the key into a number between [0,N). @@ -179,3 +235,129 @@ func (h *jmphasher) Hash(key uint64, n int) int { } return int(b) } + +// HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. +type HTTPNodeSet struct { + nodes []*Node + messageHandler func(m proto.Message) error +} + +// NewHTTPNodeSet returns a new instance of HTTPNodeSet. +func NewHTTPNodeSet() *HTTPNodeSet { + return &HTTPNodeSet{} +} + +func (h *HTTPNodeSet) Nodes() []*Node { + return h.nodes +} + +func (h *HTTPNodeSet) Join(nodes []*Node) (int, error) { + h.nodes = nodes + return 0, nil +} + +func (h *HTTPNodeSet) Open() error { + return nil +} + +// SendMessage asyncronously broadcasts a protobuf message to all nodes. +func (h *HTTPNodeSet) SendMessage(pb proto.Message) error { + + // Marshal the pb to []byte + buf, err := MarshalMessage(pb) + if err != nil { + return err + } + + var g errgroup.Group + for _, n := range h.nodes { + node := n + g.Go(func() error { + return h.sendNodeMessage(node, buf) + }) + } + return g.Wait() +} + +// ReceiveMessage is called when a node recieves a message. +func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { + return h.messageHandler(pb) +} + +func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { + var client *http.Client + client = http.DefaultClient + + // Create HTTP request. + req, err := http.NewRequest("POST", (&url.URL{ + Scheme: "http", + Host: node.Host, + Path: "/message", + }).String(), bytes.NewReader(msg)) + if err != nil { + return err + } + + // Require protobuf encoding. + req.Header.Set("Content-Type", "application/x-protobuf") + + // Send request to remote node. + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read response into buffer. + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return err + } + + // Check status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) + } + + return nil +} + +// SetMessageHandler provides the Messenger with a function to handle incoming messages. +func (h *HTTPNodeSet) SetMessageHandler(f func(proto.Message) error) { + h.messageHandler = f +} + +// StaticNodeSet represents a basic NodeSet for testing +type StaticNodeSet struct { + Messenger + nodes []*Node +} + +func NewStaticNodeSet() *StaticNodeSet { + return &StaticNodeSet{} +} + +func (s *StaticNodeSet) Nodes() []*Node { + return s.nodes +} + +func (s *StaticNodeSet) Join(nodes []*Node) (int, error) { + s.nodes = nodes + return 0, nil +} + +func (s *StaticNodeSet) Open() error { + return nil +} + +func (s *StaticNodeSet) SetMessageHandler(f func(proto.Message) error) { + return +} + +func (s *StaticNodeSet) SendMessage(pb proto.Message) error { + return nil +} +func (s *StaticNodeSet) ReceiveMessage(pb proto.Message) error { + return nil +} diff --git a/cluster_test.go b/cluster_test.go index f5ff29edf..2e9b01d89 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -77,6 +77,46 @@ func TestHasher(t *testing.T) { } } +// Ensure that an empty cluster returns a valid (empty) NodeSet +func TestCluster_NodeSetHosts(t *testing.T) { + + c := pilosa.Cluster{} + + if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) { + t.Fatalf("unexpected slice of hosts: %s", h) + } +} + +// Ensure cluster can compare its Nodes and Members +func TestCluster_Health(t *testing.T) { + c := pilosa.Cluster{ + Nodes: []*pilosa.Node{ + {Host: "serverA:1000"}, + {Host: "serverB:1000"}, + {Host: "serverC:1000"}, + }, + NodeSet: &pilosa.StaticNodeSet{}, + } + + j, err := c.NodeSet.Join([]*pilosa.Node{ + &pilosa.Node{Host: "serverA:1000"}, + &pilosa.Node{Host: "serverC:1000"}, + &pilosa.Node{Host: "serverD:1000"}, + }) + if err != nil { + t.Fatalf("unexpected gossiper nodes: %s", j) + } + + // Verify a DOWN node is reported, and extraneous nodes are ignored + if a := c.Health(); !reflect.DeepEqual(a, map[string]string{ + "serverA:1000": "UP", + "serverB:1000": "DOWN", + "serverC:1000": "UP", + }) { + t.Fatalf("unexpected health: %s", spew.Sdump(a)) + } +} + // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() diff --git a/config.go b/config.go index 7d2efc5fe..4c0554ad4 100644 --- a/config.go +++ b/config.go @@ -1,11 +1,15 @@ package pilosa -import "time" +import ( + "net" + "time" +) const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultGossipPort = 14000 ) // Config represents the configuration for the command. @@ -14,9 +18,11 @@ type Config struct { Host string `toml:"host"` Cluster struct { - ReplicaN int `toml:"replicas"` - Nodes []string `toml:"hosts"` - PollingInterval Duration `toml:"polling-interval"` + ReplicaN int `toml:"replicas"` + MessengerType string `toml:"messenger-type"` + Nodes []string `toml:"hosts"` + PollingInterval Duration `toml:"polling-interval"` + Gossip *ConfigGossip `toml:"gossip"` } `toml:"cluster"` Plugins struct { @@ -30,6 +36,15 @@ type Config struct { LogPath string `toml:"log-path"` } +type ConfigNode struct { + Host string `toml:"host"` +} + +type ConfigGossip struct { + Port int `toml:"port"` + Seed string `toml:"seed"` +} + // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ @@ -61,6 +76,29 @@ func (c *Config) PilosaCluster() *Cluster { cluster.Nodes = append(cluster.Nodes, &Node{Host: hostport}) } + // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. + if c.Cluster.MessengerType == "broadcast" { + cluster.NodeSet = NewHTTPNodeSet() + cluster.NodeSet.Join(cluster.Nodes) + } else if (c.Cluster.MessengerType == "gossip") && (c.Cluster.Gossip != nil) { + gossipPort := DefaultGossipPort + gossipSeed := DefaultHost + if c.Cluster.Gossip.Port != 0 { + gossipPort = c.Cluster.Gossip.Port + } + if c.Cluster.Gossip.Seed != "" { + gossipSeed = c.Cluster.Gossip.Seed + } + // get the host portion of addr to use for binding + gossipHost, _, err := net.SplitHostPort(c.Host) + if err != nil { + gossipHost = c.Host + } + cluster.NodeSet = NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) + } else { + cluster.NodeSet = NewStaticNodeSet() + } + return cluster } diff --git a/db.go b/db.go index 3abff2a9c..ca03188a3 100644 --- a/db.go +++ b/db.go @@ -43,7 +43,8 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - stats StatsClient + messenger Messenger + stats StatsClient LogOutput io.Writer } @@ -67,6 +68,7 @@ func NewDB(path, name string) (*DB, error) { columnLabel: DefaultColumnLabel, + messenger: NopMessenger, stats: NopStatsClient, LogOutput: ioutil.Discard, }, nil @@ -416,6 +418,7 @@ func (db *DB) newFrame(path, name string) (*Frame, error) { } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.messenger = db.messenger return f, nil } diff --git a/executor_test.go b/executor_test.go index be005e732..7a0c95622 100644 --- a/executor_test.go +++ b/executor_test.go @@ -470,7 +470,9 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Bitmap(frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0}) { //TODO: this is incorrect because the calling node doesn't know about slice 2 + // NOTE: while the following is technically incorrect (it should be {0, 2}) because the calling node doesn't know about slice 2 yet, + // we are ok with this and assuming that the calling node will become aware of slice 2 via inter-node messaging + } else if !reflect.DeepEqual(slices, []uint64{0}) { t.Fatalf("unexpected slices: %+v", slices) } diff --git a/frame.go b/frame.go index be949b19d..b299d2ac2 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,8 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - stats StatsClient + messenger Messenger + stats StatsClient // Frame settings. rowLabel string @@ -66,7 +67,8 @@ func NewFrame(path, db, name string) (*Frame, error) { views: make(map[string]*View), bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), - stats: NopStatsClient, + messenger: NopMessenger, + stats: NopStatsClient, rowLabel: DefaultRowLabel, cacheType: DefaultCacheType, @@ -409,6 +411,21 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { view.BitmapAttrStore = f.bitmapAttrStore f.views[view.Name()] = view + // TODO: this needs to be refactored for views + /* + // Send a MaxSlice message + f.messenger.SendMessage( + &internal.CreateSliceMessage{ + DB: f.db, + Slice: slice, + }) + + frag.BitmapAttrStore = f.bitmapAttrStore + + // Save to lookup. + f.fragments[slice] = frag + */ + return view, nil } diff --git a/glide.lock b/glide.lock index 7fffeca5e..e4b6ce617 100644 --- a/glide.lock +++ b/glide.lock @@ -1,6 +1,10 @@ hash: 743e8f978eb4ad8f80a2ab71b05caebbf50b6769b71aa457bc4f144fef8c6595 updated: 2017-04-18T15:33:39.035615802-05:00 imports: +- name: github.com/armon/go-metrics + version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 +- name: github.com/aws/aws-sdk-go + version: 819b71cf8430e434c1eee7e7e8b0f2b8870be899 - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml @@ -24,13 +28,21 @@ imports: subpackages: - lru - name: github.com/golang/protobuf - version: 888eb0692c857ec880338addf316bd662d5e630e + version: 8ee79997227bf9b34611aee7946ae64735e6fd93 subpackages: - proto - name: github.com/gorilla/context version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 - name: github.com/gorilla/mux version: 392c28fe23e1c45ddba891b0320b3b5df220beea +- name: github.com/hashicorp/errwrap + version: 7554cd9344cec97297fa6649b055a8c98c2a1e55 +- name: github.com/hashicorp/go-msgpack + version: fa3f63826f7c23912c15263591e65d54d080b458 + subpackages: + - codec +- name: github.com/hashicorp/go-multierror + version: ed905158d87462226a13fe39ddf685ea65f1c11f - name: github.com/hashicorp/hcl version: 630949a3c5fa3c613328e1b8256052cbc2327c9b subpackages: @@ -42,10 +54,14 @@ imports: - json/parser - json/scanner - json/token +- name: github.com/hashicorp/memberlist + version: 9800c50ab79c002353852a9b1095e9591b161513 - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 - name: github.com/magiconair/properties version: b3b15ef068fd0b17ddf408a23669f20811d194d2 +- name: github.com/miekg/dns + version: ca336a1f95a6b89be9c250df26c7a41742eb4a6f - name: github.com/mitchellh/mapstructure version: db1efb556f84b25a0a13a04aad883943538ad2e0 - name: github.com/pelletier/go-buffruneio @@ -68,6 +84,14 @@ imports: version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/spf13/viper version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4 +- name: golang.org/x/net + version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9 + subpackages: + - context +- name: golang.org/x/sync + version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c + subpackages: + - errgroup - name: golang.org/x/sys version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: diff --git a/glide.yaml b/glide.yaml index d4615b0ba..f317a0a1b 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,3 +31,8 @@ import: - package: github.com/spf13/viper - package: github.com/gorilla/mux version: ^1.3.0 +- package: github.com/aws/aws-sdk-go + version: ^1.6.10 +- package: github.com/hashicorp/memberlist +- package: golang.org/x/sync +- package: golang.org/x/net diff --git a/gossip.go b/gossip.go new file mode 100644 index 000000000..79d301a97 --- /dev/null +++ b/gossip.go @@ -0,0 +1,165 @@ +package pilosa + +import ( + "io" + "log" + "os" + + "github.com/gogo/protobuf/proto" + "github.com/hashicorp/memberlist" +) + +// GossipNodeSet represents a gossip implementation of NodeSet using memberlist +// GossipNodeSet also represents an implementation of memberlist.Delegate +type GossipNodeSet struct { + Memberlist *memberlist.Memberlist + Broadcasts *memberlist.TransmitLimitedQueue + + config *GossipConfig + + messageHandler func(m proto.Message) error + + // The writer for any logging. + LogOutput io.Writer +} + +func (g *GossipNodeSet) Nodes() []*Node { + a := make([]*Node, 0, g.Memberlist.NumMembers()) + for _, n := range g.Memberlist.Members() { + a = append(a, &Node{Host: n.Name}) + } + return a +} + +func (g *GossipNodeSet) Join(nodes []*Node) (int, error) { + return g.Memberlist.Join(Nodes(nodes).Hosts()) +} + +func (g *GossipNodeSet) Open() error { + ml, err := memberlist.Create(g.config.memberlistConfig) + if err != nil { + return err + } + g.Memberlist = ml + + // attach to gossip seed node + g.Join([]*Node{&Node{Host: g.config.gossipSeed}}) //TODO: support a list of seeds + + g.Broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return g.Memberlist.NumMembers() + }, + RetransmitMult: 3, + } + return nil +} + +func (g *GossipNodeSet) SetMessageHandler(f func(proto.Message) error) { + g.messageHandler = f +} + +// implementation of the messenger.Messenger interface +func (g *GossipNodeSet) SendMessage(pb proto.Message) error { + msg, err := MarshalMessage(pb) + if err != nil { + return err + } + + b := &broadcast{ + msg: msg, + notify: nil, + } + g.Broadcasts.QueueBroadcast(b) + return nil +} + +func (g *GossipNodeSet) ReceiveMessage(pb proto.Message) error { + err := g.messageHandler(pb) + if err != nil { + return err + } + return nil +} + +// implementation of the memberlist.Delegate interface +func (g *GossipNodeSet) NodeMeta(limit int) []byte { + return []byte{} +} + +func (g *GossipNodeSet) NotifyMsg(b []byte) { + m, err := UnmarshalMessage(b) + if err != nil { + g.logger().Printf("unmarshal message error: %s", err) + return + } + if err := g.ReceiveMessage(m); err != nil { + g.logger().Printf("receive message error: %s", err) + return + } +} + +func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { + return g.Broadcasts.GetBroadcasts(overhead, limit) +} + +func (g *GossipNodeSet) LocalState(join bool) []byte { + return []byte{} +} + +func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { + return +} + +// logger returns a logger for the GossipNodeSet. +func (g *GossipNodeSet) logger() *log.Logger { + return log.New(g.LogOutput, "", log.LstdFlags) +} + +// broadcast represents an implementation of memberlist.Broadcast +type broadcast struct { + msg []byte + notify chan<- struct{} +} + +func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { + return false +} + +func (b *broadcast) Message() []byte { + return b.msg +} + +func (b *broadcast) Finished() { + if b.notify != nil { + close(b.notify) + } +} + +//////////////////////////////////////////////////////////////// + +type GossipConfig struct { + gossipSeed string + memberlistConfig *memberlist.Config +} + +// NewGossipNodeSet returns a new instance of GossipNodeSet. +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string) *GossipNodeSet { + g := &GossipNodeSet{ + LogOutput: os.Stderr, + } + + //TODO: pull memberlist config from pilosa.cfg file + g.config = &GossipConfig{ + memberlistConfig: memberlist.DefaultLocalConfig(), + gossipSeed: gossipSeed, + } + g.config.memberlistConfig.Name = name + g.config.memberlistConfig.BindAddr = gossipHost + g.config.memberlistConfig.BindPort = gossipPort + g.config.memberlistConfig.AdvertiseAddr = gossipHost + g.config.memberlistConfig.AdvertisePort = gossipPort + g.config.memberlistConfig.GossipNodes = 1 + g.config.memberlistConfig.Delegate = g + + return g +} diff --git a/handler.go b/handler.go index 0e25c3f2f..d3c5a28cc 100644 --- a/handler.go +++ b/handler.go @@ -25,7 +25,8 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Index *Index + Index *Index + Messenger Messenger // Local hostname & cluster configuration. Host string @@ -49,6 +50,7 @@ type Handler struct { func NewHandler() *Handler { handler := &Handler{ LogOutput: os.Stderr, + Messenger: NopMessenger, } handler.Router = NewRouter(handler) return handler @@ -111,10 +113,23 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } } +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getStatusResponse{ + Health: h.Cluster.Health(), + }); err != nil { + h.logger().Printf("write status response error: %s", err) + } +} + type getSchemaResponse struct { DBs []*DBInfo `json:"dbs"` } +type getStatusResponse struct { + Health map[string]string `json:"health"` +} + // handlePostQuery handles /query requests. func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { dbName := mux.Vars(r)["db"] @@ -177,6 +192,36 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } +// handlePostMessage handles /message requests. +func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal message to specific proto type. + m, err := UnmarshalMessage(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.Messenger.ReceiveMessage(m); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + return +} + func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { var ms map[string]uint64 if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { @@ -324,6 +369,13 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { return } + // Send the delete message to all nodes. + // NOTE: this calls a second DeleteDB on the local node + h.Messenger.SendMessage( + &internal.DeleteDBMessage{ + DB: req.DB, + }) + // Encode response. if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/index.go b/index.go index 7597984ba..01409a277 100644 --- a/index.go +++ b/index.go @@ -11,6 +11,9 @@ import ( "sort" "sync" "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. @@ -23,6 +26,8 @@ type Index struct { // Databases by name. dbs map[string]*DB + Messenger Messenger + // Close management wg sync.WaitGroup closing chan struct{} @@ -45,7 +50,8 @@ func NewIndex() *Index { dbs: make(map[string]*DB), closing: make(chan struct{}, 0), - Stats: NopStatsClient, + Messenger: NopMessenger, + Stats: NopStatsClient, CacheFlushInterval: DefaultCacheFlushInterval, @@ -243,6 +249,7 @@ func (i *Index) newDB(path, name string) (*DB, error) { } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) + db.messenger = i.Messenger return db, nil } @@ -338,6 +345,25 @@ func (i *Index) flushCaches() { } } +// HandleMessage handles protobuf Messages broadcasted to nodes in the +// cluster from the Cluster's NodeSet. +func (i *Index) HandleMessage(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.CreateSliceMessage: + d := i.DB(obj.DB) + if d == nil { + return fmt.Errorf("Local DB not found: %s", obj.DB) + } + d.SetRemoteMaxSlice(obj.Slice) + case *internal.DeleteDBMessage: + err := i.DeleteDB(obj.DB) + if err != nil { + return err + } + } + return nil +} + func (i *Index) logger() *log.Logger { return log.New(i.LogOutput, "", log.LstdFlags) } // IndexSyncer is an active anti-entropy tool that compares the local index diff --git a/index_test.go b/index_test.go index 4a1be4cf2..50ce1979e 100644 --- a/index_test.go +++ b/index_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -162,6 +163,32 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } } +// Ensure index can handle Messenger messages. +func TestIndex_HandleMessage(t *testing.T) { + // Create a local index. + idx0 := MustOpenIndex() + defer idx0.Close() + + idx0.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + + msg0 := &internal.CreateSliceMessage{ + DB: "d", + Slice: 8, + } + idx0.HandleMessage(msg0) + if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{"d": 8}) { + t.Fatalf("unexpected max slice: %s", ms) + } + + msg1 := &internal.DeleteDBMessage{ + DB: "d", + } + idx0.HandleMessage(msg1) + if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{}) { + t.Fatalf("unexpected delete db: %s", ms) + } +} + // Index is a test wrapper for pilosa.Index. type Index struct { *pilosa.Index diff --git a/messenger.go b/messenger.go new file mode 100644 index 000000000..115627ae4 --- /dev/null +++ b/messenger.go @@ -0,0 +1,73 @@ +package pilosa + +import ( + "fmt" + "reflect" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" +) + +func init() { + NopMessenger = &nopMessenger{} +} + +var NopMessenger Messenger + +// nopMessenger represents a Messenger that doesn't do anything. +type nopMessenger struct{} + +func (c *nopMessenger) SendMessage(pb proto.Message) error { + fmt.Println("NOPMessenger: Send") + return nil +} +func (c *nopMessenger) ReceiveMessage(pb proto.Message) error { + fmt.Println("NOPMessenger: Receive") + return nil +} + +type Messenger interface { + SendMessage(pb proto.Message) error + ReceiveMessage(pb proto.Message) error +} + +const ( + MessageTypeCreateSlice = 1 + MessageTypeDeleteDB = 2 +) + +func MarshalMessage(m proto.Message) ([]byte, error) { + var typ uint8 + switch obj := m.(type) { + case *internal.CreateSliceMessage: + typ = MessageTypeCreateSlice + case *internal.DeleteDBMessage: + typ = MessageTypeDeleteDB + default: + return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) + } + buf, err := proto.Marshal(m) + if err != nil { + return nil, err + } + return append([]byte{typ}, buf...), nil +} + +func UnmarshalMessage(buf []byte) (proto.Message, error) { + typ, buf := buf[0], buf[1:] + + var m proto.Message + switch typ { + case MessageTypeCreateSlice: + m = &internal.CreateSliceMessage{} + case MessageTypeDeleteDB: + m = &internal.DeleteDBMessage{} + default: + return nil, fmt.Errorf("invalid message type: %d", typ) + } + + if err := proto.Unmarshal(buf, m); err != nil { + return nil, err + } + return m, nil +} diff --git a/server.go b/server.go index 55702acf9..f4cef42f6 100644 --- a/server.go +++ b/server.go @@ -32,8 +32,9 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index - Handler *Handler + Index *Index + Handler *Handler + Messenger Messenger // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -52,8 +53,9 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), - Handler: NewHandler(), + Index: NewIndex(), + Handler: NewHandler(), + Messenger: NopMessenger, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -96,6 +98,11 @@ func (s *Server) Open() error { return err } + // Open NodeSet communication + if err := s.Cluster.NodeSet.Open(); err != nil { + return err + } + // Create executor for executing queries. e := NewExecutor() e.Index = s.Index @@ -202,21 +209,15 @@ func (s *Server) monitorMaxSlices() { if s.Host != node.Host { maxSlices, _ := checkMaxSlices(node.Host) for db, newmax := range maxSlices { - // if we don't know about a db locally, create it - // so that the /schema endpoint can report it + // if we don't know about a db locally, log an error because + // db's should be created and synced prior to slice creation if localdb := s.Index.DB(db); localdb != nil { if newmax > oldmaxslices[db] { oldmaxslices[db] = newmax localdb.SetRemoteMaxSlice(newmax) } } else { - d := s.Index.DB(db) - if d == nil { - s.logger().Printf("Local DB not found: %s", db) - return - } - oldmaxslices[db] = newmax - d.SetRemoteMaxSlice(newmax) + s.logger().Printf("Local DB not found: %s", db) } } } From 2b35df879cb3c7e0b474ac9c47116db789fabe8e Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 14 Mar 2017 17:44:23 -0500 Subject: [PATCH 07/50] WIP: first pass at sending `CreateDB()` through Messenger --- client.go | 1 + cluster.go | 4 ++++ index.go | 17 ++++++++++++++++- messenger.go | 5 +++++ server.go | 8 +++++--- server/server.go | 9 +++++++++ 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/client.go b/client.go index e4401a69a..d340ce640 100644 --- a/client.go +++ b/client.go @@ -159,6 +159,7 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { case http.StatusOK: return nil // ok case http.StatusConflict: + fmt.Println("ErrDatabaseExists: 1") return ErrDatabaseExists default: return errors.New(string(body)) diff --git a/cluster.go b/cluster.go index 8268fe508..061f76c62 100644 --- a/cluster.go +++ b/cluster.go @@ -285,6 +285,7 @@ func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { } func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { + fmt.Println("sendNodeMessage:", node.Host) var client *http.Client client = http.DefaultClient @@ -302,7 +303,9 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { req.Header.Set("Content-Type", "application/x-protobuf") // Send request to remote node. + fmt.Println("Send") resp, err := client.Do(req) + fmt.Println("Got back") if err != nil { return err } @@ -314,6 +317,7 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { if err != nil { return err } + fmt.Println("code:", resp.StatusCode) // Check status code. if resp.StatusCode != http.StatusOK { diff --git a/index.go b/index.go index 01409a277..f7269d670 100644 --- a/index.go +++ b/index.go @@ -187,12 +187,14 @@ func (i *Index) DBs() []*DB { } // CreateDB creates a database. +// An error is returned if the database already exists. func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) { i.mu.Lock() defer i.mu.Unlock() // Ensure db doesn't already exist. if i.dbs[name] != nil { + fmt.Println("ErrDatabaseExists: 2") return nil, ErrDatabaseExists } return i.createDB(name, opt) @@ -204,7 +206,7 @@ func (i *Index) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) { i.mu.Lock() defer i.mu.Unlock() - // Find frame in cache first. + // Find database in cache first. if db := i.dbs[name]; db != nil { return db, nil } @@ -239,6 +241,13 @@ func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { i.Stats.Count("dbN", 1) + // Send a CreateDB message + i.Messenger.SendMessage( + &internal.CreateDBMessage{ + DB: name, + ColumnLabel: opt.ColumnLabel, + }) + return db, nil } @@ -360,6 +369,12 @@ func (i *Index) HandleMessage(pb proto.Message) error { if err != nil { return err } + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.ColumnLabel} + _, err := i.CreateDB(obj.DB, opt) + if err != nil { + return err + } } return nil } diff --git a/messenger.go b/messenger.go index 115627ae4..5e2d128e2 100644 --- a/messenger.go +++ b/messenger.go @@ -34,6 +34,7 @@ type Messenger interface { const ( MessageTypeCreateSlice = 1 MessageTypeDeleteDB = 2 + MessageTypeCreateDB = 3 ) func MarshalMessage(m proto.Message) ([]byte, error) { @@ -43,6 +44,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateSlice case *internal.DeleteDBMessage: typ = MessageTypeDeleteDB + case *internal.CreateDBMessage: + typ = MessageTypeCreateDB default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -62,6 +65,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateSliceMessage{} case MessageTypeDeleteDB: m = &internal.DeleteDBMessage{} + case MessageTypeCreateDB: + m = &internal.CreateDBMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/server.go b/server.go index f4cef42f6..cde64c3ca 100644 --- a/server.go +++ b/server.go @@ -120,9 +120,11 @@ func (s *Server) Open() error { go func() { http.Serve(ln, s.Handler) }() // Start background monitoring. - s.wg.Add(2) - go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() + /* + s.wg.Add(2) + go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() + go func() { defer s.wg.Done(); s.monitorMaxSlices() }() + */ return nil } diff --git a/server/server.go b/server/server.go index 251a054aa..da83da52c 100644 --- a/server/server.go +++ b/server/server.go @@ -94,6 +94,15 @@ func (m *Command) Run(args ...string) (err error) { } m.Server.Cluster = m.Config.PilosaCluster() + // Setup Messenger. + fmt.Fprintf(m.Stderr, "Using Messenger type: %s\n", m.Config.Cluster.MessengerType) + m.Server.Messenger = m.Server.Cluster.NodeSet.(pilosa.Messenger) + m.Server.Handler.Messenger = m.Server.Messenger + m.Server.Index.Messenger = m.Server.Messenger + + // Set message handler. + m.Server.Cluster.NodeSet.SetMessageHandler(m.Server.Index.HandleMessage) + // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) From 8fa45649b678745e6eb5e8d9262415426c3500d1 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Mar 2017 17:19:42 -0500 Subject: [PATCH 08/50] WIP: Added support for NodeState. Modified the gossip implementation to optionally send messages directly (instead of over gossip). Added support for local and remote NodeState, as well as function to merge remote into local state. Refactored Protobuf models to include `DBMeta` and `FrameMeta` as well as support for NodeState. --- cluster.go | 38 +++++++++++++++---- db.go | 36 +++++++++++++++--- frame.go | 48 ++++++++++++++++-------- gossip.go | 95 +++++++++++++++++++++++++++++++++++++++--------- handler.go | 32 ++++++++++++++-- index.go | 30 ++++++++------- messenger.go | 26 +++++++++---- server.go | 52 ++++++++++++++++++++++++++ server/server.go | 4 +- 9 files changed, 291 insertions(+), 70 deletions(-) diff --git a/cluster.go b/cluster.go index 061f76c62..bbdc8f51f 100644 --- a/cluster.go +++ b/cluster.go @@ -211,6 +211,12 @@ type NodeSet interface { // SetMessageHandler provides the NodeSet with a function to call on ReceiveMessage SetMessageHandler(f func(proto.Message) error) + + // SetRemoteStateHandler provides the function to call on MergeRemoteState + SetRemoteStateHandler(f func(proto.Message) error) + + // SetLocalStateSource provides the function to get the current node's local state. + SetLocalStateSource(f func() (proto.Message, error)) } // Hasher represents an interface to hash integers into buckets. @@ -240,6 +246,8 @@ func (h *jmphasher) Hash(key uint64, n int) int { type HTTPNodeSet struct { nodes []*Node messageHandler func(m proto.Message) error + // remoteStateHandler func(m proto.Message) error + // localStateSource func() (proto.Message, error) } // NewHTTPNodeSet returns a new instance of HTTPNodeSet. @@ -261,7 +269,7 @@ func (h *HTTPNodeSet) Open() error { } // SendMessage asyncronously broadcasts a protobuf message to all nodes. -func (h *HTTPNodeSet) SendMessage(pb proto.Message) error { +func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { // Marshal the pb to []byte buf, err := MarshalMessage(pb) @@ -279,13 +287,12 @@ func (h *HTTPNodeSet) SendMessage(pb proto.Message) error { return g.Wait() } -// ReceiveMessage is called when a node recieves a message. +// ReceiveMessage is called when a node receives a message. func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { return h.messageHandler(pb) } func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { - fmt.Println("sendNodeMessage:", node.Host) var client *http.Client client = http.DefaultClient @@ -303,9 +310,7 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { req.Header.Set("Content-Type", "application/x-protobuf") // Send request to remote node. - fmt.Println("Send") resp, err := client.Do(req) - fmt.Println("Got back") if err != nil { return err } @@ -317,7 +322,6 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { if err != nil { return err } - fmt.Println("code:", resp.StatusCode) // Check status code. if resp.StatusCode != http.StatusOK { @@ -332,6 +336,18 @@ func (h *HTTPNodeSet) SetMessageHandler(f func(proto.Message) error) { h.messageHandler = f } +// SetRemoteStateHandler provides the Messenger with a function to merge remote state. +func (h *HTTPNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { + // not implemented + // h.remoteStateHandler = f +} + +// SetLocalStateSource currently no-ops. +func (h *HTTPNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { + // not implemented + // h.localStateSource = f +} + // StaticNodeSet represents a basic NodeSet for testing type StaticNodeSet struct { Messenger @@ -359,7 +375,15 @@ func (s *StaticNodeSet) SetMessageHandler(f func(proto.Message) error) { return } -func (s *StaticNodeSet) SendMessage(pb proto.Message) error { +func (s *StaticNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { + return +} + +func (s *StaticNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { + return +} + +func (s *StaticNodeSet) SendMessage(pb proto.Message, method string) error { return nil } func (s *StaticNodeSet) ReceiveMessage(pb proto.Message) error { diff --git a/db.go b/db.go index ca03188a3..a7f6f8ff1 100644 --- a/db.go +++ b/db.go @@ -173,7 +173,7 @@ func (db *DB) openFrames() error { // loadMeta reads meta data for the database, if any. func (db *DB) loadMeta() error { - var pb internal.DB + var pb internal.DBMeta // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(db.path, ".meta")) @@ -199,7 +199,7 @@ func (db *DB) loadMeta() error { // saveMeta writes meta data for the database. func (db *DB) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.DB{ + buf, err := proto.Marshal(&internal.DBMeta{ TimeQuantum: string(db.timeQuantum), ColumnLabel: db.columnLabel, }) @@ -251,10 +251,10 @@ func (db *DB) MaxSlice() uint64 { return max } -func (db *DB) SetRemoteMaxSlice(v uint64) { +func (db *DB) SetRemoteMaxSlice(newmax uint64) { db.mu.Lock() defer db.mu.Unlock() - db.remoteMaxSlice = v + db.remoteMaxSlice = newmax } // MaxInverseSlice returns the max inverse slice in the database according to this node. @@ -396,6 +396,9 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { if opt.CacheSize != 0 { f.rankedCacheSize = opt.CacheSize } + if opt.TimeQuantum.Valid() { + f.timeQuantum = opt.TimeQuantum + } f.inverseEnabled = opt.InverseEnabled if err := f.saveMeta(); err != nil { @@ -509,9 +512,32 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo { return dbs } +// encodeDBs converts a into its internal representation. +func encodeDBs(a []*DB) []*internal.DB { + other := make([]*internal.DB, len(a)) + for i := range a { + other[i] = encodeDB(a[i]) + } + return other +} + +// encodeDB converts d into its internal representation. +func encodeDB(d *DB) *internal.DB { + return &internal.DB{ + Name: d.name, + Meta: &internal.DBMeta{ + ColumnLabel: d.columnLabel, + TimeQuantum: string(d.timeQuantum), + }, + MaxSlice: d.remoteMaxSlice, + Frames: encodeFrames(d.Frames()), + } +} + // DBOptions represents options to set when initializing a db. type DBOptions struct { - ColumnLabel string `json:"columnLabel,omitempty"` + ColumnLabel string `json:"columnLabel,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // hasTime returns true if a contains a non-nil time. diff --git a/frame.go b/frame.go index b299d2ac2..51cb2c41b 100644 --- a/frame.go +++ b/frame.go @@ -264,7 +264,7 @@ func (f *Frame) openViews() error { // loadMeta reads meta data for the frame, if any. func (f *Frame) loadMeta() error { - var pb internal.Frame + var pb internal.FrameMeta // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) @@ -413,17 +413,12 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { // TODO: this needs to be refactored for views /* - // Send a MaxSlice message - f.messenger.SendMessage( - &internal.CreateSliceMessage{ - DB: f.db, - Slice: slice, - }) - - frag.BitmapAttrStore = f.bitmapAttrStore - - // Save to lookup. - f.fragments[slice] = frag + // Send a MaxSlice message + f.messenger.SendMessage( + &internal.CreateSliceMessage{ + DB: f.db, + Slice: slice, + }, "gossip") */ return view, nil @@ -591,6 +586,26 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) return nil } +// encodeFrames converts a into its internal representation. +func encodeFrames(a []*Frame) []*internal.Frame { + other := make([]*internal.Frame, len(a)) + for i := range a { + other[i] = encodeFrame(a[i]) + } + return other +} + +// encodeFrame converts f into its internal representation. +func encodeFrame(f *Frame) *internal.Frame { + return &internal.Frame{ + Name: f.name, + Meta: &internal.FrameMeta{ + TimeQuantum: string(f.timeQuantum), + RowLabel: f.rowLabel, + }, + } +} + type frameSlice []*Frame func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -611,10 +626,11 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - RowLabel string `json:"rowLabel,omitempty"` - InverseEnabled bool `json:"inverseEnabled,omitempty"` - CacheType string `json:"cacheType,omitempty"` - CacheSize int `json:"cacheSize,omitempty"` + RowLabel string `json:"rowLabel,omitempty"` + InverseEnabled bool `json:"inverseEnabled,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize int `json:"cacheSize,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // importBitSet represents slices of row and column ids. diff --git a/gossip.go b/gossip.go index 79d301a97..e2b3ddcde 100644 --- a/gossip.go +++ b/gossip.go @@ -1,38 +1,44 @@ package pilosa import ( + "fmt" "io" "log" "os" + "golang.org/x/sync/errgroup" + "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" + "github.com/pilosa/pilosa/internal" ) // GossipNodeSet represents a gossip implementation of NodeSet using memberlist // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { - Memberlist *memberlist.Memberlist - Broadcasts *memberlist.TransmitLimitedQueue + memberlist *memberlist.Memberlist + broadcasts *memberlist.TransmitLimitedQueue config *GossipConfig - messageHandler func(m proto.Message) error + messageHandler func(m proto.Message) error + remoteStateHandler func(m proto.Message) error + localStateSource func() (proto.Message, error) // The writer for any logging. LogOutput io.Writer } func (g *GossipNodeSet) Nodes() []*Node { - a := make([]*Node, 0, g.Memberlist.NumMembers()) - for _, n := range g.Memberlist.Members() { + a := make([]*Node, 0, g.memberlist.NumMembers()) + for _, n := range g.memberlist.Members() { a = append(a, &Node{Host: n.Name}) } return a } func (g *GossipNodeSet) Join(nodes []*Node) (int, error) { - return g.Memberlist.Join(Nodes(nodes).Hosts()) + return g.memberlist.Join(Nodes(nodes).Hosts()) } func (g *GossipNodeSet) Open() error { @@ -40,14 +46,14 @@ func (g *GossipNodeSet) Open() error { if err != nil { return err } - g.Memberlist = ml + g.memberlist = ml // attach to gossip seed node g.Join([]*Node{&Node{Host: g.config.gossipSeed}}) //TODO: support a list of seeds - g.Broadcasts = &memberlist.TransmitLimitedQueue{ + g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { - return g.Memberlist.NumMembers() + return g.memberlist.NumMembers() }, RetransmitMult: 3, } @@ -58,18 +64,49 @@ func (g *GossipNodeSet) SetMessageHandler(f func(proto.Message) error) { g.messageHandler = f } +func (g *GossipNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { + g.remoteStateHandler = f +} + +func (g *GossipNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { + g.localStateSource = f +} + // implementation of the messenger.Messenger interface -func (g *GossipNodeSet) SendMessage(pb proto.Message) error { +func (g *GossipNodeSet) SendMessage(pb proto.Message, method string) error { msg, err := MarshalMessage(pb) if err != nil { return err } - b := &broadcast{ - msg: msg, - notify: nil, + // Broadcast asyncronously sends the message directly to each node. + // An error from any node raises an error on the entire operation. + // This is a blocking operation. + // + // Gossip uses the gossip protocol to eventually deliver the message + // to every node. + switch method { + case "broadcast": + var eg errgroup.Group + for _, n := range g.memberlist.Members() { + // Don't send the message to the local node. + if n == g.memberlist.LocalNode() { + continue + } + node := n + eg.Go(func() error { + return g.memberlist.SendToTCP(node, msg) + }) + } + return eg.Wait() + case "gossip": + b := &broadcast{ + msg: msg, + notify: nil, + } + g.broadcasts.QueueBroadcast(b) } - g.Broadcasts.QueueBroadcast(b) + return nil } @@ -87,6 +124,8 @@ func (g *GossipNodeSet) NodeMeta(limit int) []byte { } func (g *GossipNodeSet) NotifyMsg(b []byte) { + loc := g.memberlist.LocalNode() + fmt.Println("Received Msg:", loc) m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -99,14 +138,37 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) { } func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { - return g.Broadcasts.GetBroadcasts(overhead, limit) + return g.broadcasts.GetBroadcasts(overhead, limit) } func (g *GossipNodeSet) LocalState(join bool) []byte { - return []byte{} + + pb, err := g.localStateSource() + if err != nil { + g.logger().Printf("error getting local state, err=%s", err) + return []byte{} + } + + // Marshal nodestate data to bytes. + buf, err := proto.Marshal(pb) + if err != nil { + g.logger().Printf("error marshaling nodestate data, err=%s", err) + return []byte{} + } + return buf } func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { + // Unmarshal nodestate data. + var pb internal.NodeState + if err := proto.Unmarshal(buf, &pb); err != nil { + g.logger().Printf("error unmarshaling nodestate data, err=%s", err) + return + } + err := g.remoteStateHandler(&pb) + if err != nil { + g.logger().Printf("merge state error: %s", err) + } return } @@ -158,7 +220,6 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = gossipHost g.config.memberlistConfig.AdvertisePort = gossipPort - g.config.memberlistConfig.GossipNodes = 1 g.config.memberlistConfig.Delegate = g return g diff --git a/handler.go b/handler.go index d3c5a28cc..ad337b2bc 100644 --- a/handler.go +++ b/handler.go @@ -370,11 +370,13 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - // NOTE: this calls a second DeleteDB on the local node - h.Messenger.SendMessage( + err := h.Messenger.SendMessage( &internal.DeleteDBMessage{ DB: req.DB, - }) + }, "broadcast") + if err != nil { + h.logger().Printf("problem sending DeleteDB message: %s", err) + } // Encode response. if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil { @@ -511,6 +513,20 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { return } + // Send the create message to all nodes. + err = h.Messenger.SendMessage( + &internal.CreateFrameMessage{ + DB: req.DB, + Frame: req.Frame, + Meta: &internal.FrameMeta{ + RowLabel: req.Options.RowLabel, + TimeQuantum: string(req.Options.TimeQuantum), + }, + }, "broadcast") + if err != nil { + h.logger().Printf("problem sending CreateFrame message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) @@ -569,6 +585,16 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { return } + // Send the delete message to all nodes. + err := h.Messenger.SendMessage( + &internal.DeleteFrameMessage{ + DB: req.DB, + Frame: req.Frame, + }, "broadcast") + if err != nil { + h.logger().Printf("problem sending DeleteFrame message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/index.go b/index.go index f7269d670..954629eea 100644 --- a/index.go +++ b/index.go @@ -194,7 +194,6 @@ func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) { // Ensure db doesn't already exist. if i.dbs[name] != nil { - fmt.Println("ErrDatabaseExists: 2") return nil, ErrDatabaseExists } return i.createDB(name, opt) @@ -236,18 +235,12 @@ func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { // Update options. db.SetColumnLabel(opt.ColumnLabel) + db.SetTimeQuantum(opt.TimeQuantum) i.dbs[db.Name()] = db i.Stats.Count("dbN", 1) - // Send a CreateDB message - i.Messenger.SendMessage( - &internal.CreateDBMessage{ - DB: name, - ColumnLabel: opt.ColumnLabel, - }) - return db, nil } @@ -364,17 +357,28 @@ func (i *Index) HandleMessage(pb proto.Message) error { return fmt.Errorf("Local DB not found: %s", obj.DB) } d.SetRemoteMaxSlice(obj.Slice) - case *internal.DeleteDBMessage: - err := i.DeleteDB(obj.DB) + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := i.CreateDB(obj.DB, opt) if err != nil { return err } - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.ColumnLabel} - _, err := i.CreateDB(obj.DB, opt) + case *internal.DeleteDBMessage: + if err := i.DeleteDB(obj.DB); err != nil { + return err + } + case *internal.CreateFrameMessage: + db := i.DB(obj.DB) + opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + _, err := db.CreateFrame(obj.Frame, opt) if err != nil { return err } + case *internal.DeleteFrameMessage: + db := i.DB(obj.DB) + if err := db.DeleteFrame(obj.Frame); err != nil { + return err + } } return nil } diff --git a/messenger.go b/messenger.go index 5e2d128e2..f28eb7ddb 100644 --- a/messenger.go +++ b/messenger.go @@ -17,7 +17,7 @@ var NopMessenger Messenger // nopMessenger represents a Messenger that doesn't do anything. type nopMessenger struct{} -func (c *nopMessenger) SendMessage(pb proto.Message) error { +func (c *nopMessenger) SendMessage(pb proto.Message, method string) error { fmt.Println("NOPMessenger: Send") return nil } @@ -27,14 +27,16 @@ func (c *nopMessenger) ReceiveMessage(pb proto.Message) error { } type Messenger interface { - SendMessage(pb proto.Message) error + SendMessage(pb proto.Message, method string) error ReceiveMessage(pb proto.Message) error } const ( MessageTypeCreateSlice = 1 - MessageTypeDeleteDB = 2 - MessageTypeCreateDB = 3 + MessageTypeCreateDB = 2 + MessageTypeDeleteDB = 3 + MessageTypeCreateFrame = 4 + MessageTypeDeleteFrame = 5 ) func MarshalMessage(m proto.Message) ([]byte, error) { @@ -42,10 +44,14 @@ func MarshalMessage(m proto.Message) ([]byte, error) { switch obj := m.(type) { case *internal.CreateSliceMessage: typ = MessageTypeCreateSlice - case *internal.DeleteDBMessage: - typ = MessageTypeDeleteDB case *internal.CreateDBMessage: typ = MessageTypeCreateDB + case *internal.DeleteDBMessage: + typ = MessageTypeDeleteDB + case *internal.CreateFrameMessage: + typ = MessageTypeCreateFrame + case *internal.DeleteFrameMessage: + typ = MessageTypeDeleteFrame default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -63,10 +69,14 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { switch typ { case MessageTypeCreateSlice: m = &internal.CreateSliceMessage{} - case MessageTypeDeleteDB: - m = &internal.DeleteDBMessage{} case MessageTypeCreateDB: m = &internal.CreateDBMessage{} + case MessageTypeDeleteDB: + m = &internal.DeleteDBMessage{} + case MessageTypeCreateFrame: + m = &internal.CreateFrameMessage{} + case MessageTypeDeleteFrame: + m = &internal.DeleteFrameMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/server.go b/server.go index cde64c3ca..0df1182ba 100644 --- a/server.go +++ b/server.go @@ -153,6 +153,49 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +func (s *Server) LocalState() (proto.Message, error) { + // TODO: are there errors to handle? + pb := encodeLocalState(s) + return pb, nil +} + +// HandleRemoteState provides the current, local state. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +func (s *Server) HandleRemoteState(pb proto.Message) error { + return s.mergeRemoteState(pb.(*internal.NodeState)) +} + +func (s *Server) mergeRemoteState(ns *internal.NodeState) error { + // TODO: update some node state value in the cluster (it should be in cluster.node i guess) + + // Create databases that don't exist. + for _, db := range ns.DBs { + opt := DBOptions{ + ColumnLabel: db.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + } + d, err := s.Index.CreateDBIfNotExists(db.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range db.Frames { + opt := FrameOptions{ + RowLabel: f.Meta.RowLabel, + TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), + } + _, err := d.CreateFrameIfNotExists(f.Name, opt) + if err != nil { + return err + } + } + } + + return nil +} + func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { @@ -189,6 +232,15 @@ func (s *Server) monitorAntiEntropy() { } } +// encodeLocalState converts s into its internal representation. +func encodeLocalState(s *Server) *internal.NodeState { + return &internal.NodeState{ + Host: s.Host, + State: "OK", // TODO: make this work, pull from cluster.Node + DBs: encodeDBs(s.Index.DBs()), + } +} + // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. func (s *Server) monitorMaxSlices() { // Ignore if only one node in the cluster. diff --git a/server/server.go b/server/server.go index da83da52c..3a62c5c60 100644 --- a/server/server.go +++ b/server/server.go @@ -100,8 +100,10 @@ func (m *Command) Run(args ...string) (err error) { m.Server.Handler.Messenger = m.Server.Messenger m.Server.Index.Messenger = m.Server.Messenger - // Set message handler. + // Set message and state handlers. m.Server.Cluster.NodeSet.SetMessageHandler(m.Server.Index.HandleMessage) + m.Server.Cluster.NodeSet.SetRemoteStateHandler(m.Server.HandleRemoteState) + m.Server.Cluster.NodeSet.SetLocalStateSource(m.Server.LocalState) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) From 74b81eeac1ae915134551b3e834c47fba71a4156 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Mar 2017 19:05:40 -0500 Subject: [PATCH 09/50] adjustments to support the new Config structure --- cmd/server.go | 5 ++++- config.go | 16 ++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 82363f849..d25066764 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -75,7 +75,7 @@ on the configured port.`, flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") - flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number hosts each piece of data should be stored on.") + flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&Server.Config.Cluster.Nodes, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") @@ -83,6 +83,9 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") + flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "", "Type of Messenger to use for inter-host messaging.") + flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") + flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") return serveCmd } diff --git a/config.go b/config.go index 4c0554ad4..17edd639d 100644 --- a/config.go +++ b/config.go @@ -18,11 +18,11 @@ type Config struct { Host string `toml:"host"` Cluster struct { - ReplicaN int `toml:"replicas"` - MessengerType string `toml:"messenger-type"` - Nodes []string `toml:"hosts"` - PollingInterval Duration `toml:"polling-interval"` - Gossip *ConfigGossip `toml:"gossip"` + ReplicaN int `toml:"replicas"` + MessengerType string `toml:"messenger-type"` + Nodes []string `toml:"hosts"` + PollingInterval Duration `toml:"polling-interval"` + Gossip ConfigGossip `toml:"gossip"` } `toml:"cluster"` Plugins struct { @@ -36,10 +36,6 @@ type Config struct { LogPath string `toml:"log-path"` } -type ConfigNode struct { - Host string `toml:"host"` -} - type ConfigGossip struct { Port int `toml:"port"` Seed string `toml:"seed"` @@ -80,7 +76,7 @@ func (c *Config) PilosaCluster() *Cluster { if c.Cluster.MessengerType == "broadcast" { cluster.NodeSet = NewHTTPNodeSet() cluster.NodeSet.Join(cluster.Nodes) - } else if (c.Cluster.MessengerType == "gossip") && (c.Cluster.Gossip != nil) { + } else if c.Cluster.MessengerType == "gossip" { gossipPort := DefaultGossipPort gossipSeed := DefaultHost if c.Cluster.Gossip.Port != 0 { From 8e2592e70696318799e2d1924aab4c13f7d1b6bb Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 20 Mar 2017 11:07:53 -0500 Subject: [PATCH 10/50] store cache size in Frame metadata --- frame.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame.go b/frame.go index 51cb2c41b..5fea71e3d 100644 --- a/frame.go +++ b/frame.go @@ -273,6 +273,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled + f.rankedCacheSize = DefaultFrameCache return nil } else if err != nil { return err @@ -286,6 +287,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled + f.rankedCacheSize = int(pb.CacheSize) // Copy cache type. f.cacheType = pb.CacheType @@ -304,6 +306,7 @@ func (f *Frame) saveMeta() error { RowLabel: f.rowLabel, CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheSize: int64(f.rankedCacheSize), }) if err != nil { return err @@ -602,6 +605,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, + CacheSize: int64(f.rankedCacheSize), }, } } From 9a1a631dd4dac53beaf07de6ec5c22608000fc30 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 20 Mar 2017 11:19:02 -0500 Subject: [PATCH 11/50] test storing frame cache size --- fragment.go | 2 +- frame_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 4f6362193..e83f76ed2 100644 --- a/fragment.go +++ b/fragment.go @@ -44,7 +44,7 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 - // Percentage of the ThresholdLength size to resort the cache at + // ThresholdBufferPct is the percentage of the ThresholdLength size to resort the cache at ThresholdBufferPct = 0.9 ) diff --git a/frame_test.go b/frame_test.go index 5b659f826..16b935937 100644 --- a/frame_test.go +++ b/frame_test.go @@ -126,3 +126,24 @@ func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time } return changed } + +// Ensure frame can set its cache +func TestFrame_SetCacheSize(t *testing.T) { + f := MustOpenFrame() + defer f.Close() + cacheSize := 100 + + // Set & retrieve frame cache size. + if err := f.SetRankedCacheSize(cacheSize); err != nil { + t.Fatal(err) + } else if q := f.RankedCacheSize(); q != cacheSize { + t.Fatalf("unexpected frame cache size: %d", q) + } + + // Reload frame and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.RankedCacheSize(); q != cacheSize { + t.Fatalf("unexpected frame cache size (reopen): %d", q) + } +} From ce1e8e2e75253f7cc489eaab47f2a931a877e69f Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 20 Mar 2017 15:10:26 -0500 Subject: [PATCH 12/50] changed the frame cache name --- fragment.go | 2 +- frame.go | 38 +++++++++++++++++++------------------- frame_test.go | 6 +++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/fragment.go b/fragment.go index e83f76ed2..de188d0e8 100644 --- a/fragment.go +++ b/fragment.go @@ -44,7 +44,7 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 - // ThresholdBufferPct is the percentage of the ThresholdLength size to resort the cache at + // ThresholdBufferPct is the percentage of the ThresholdLength for which to maintain a sorted, ranked list. ThresholdBufferPct = 0.9 ) diff --git a/frame.go b/frame.go index 5fea71e3d..ebfe15ab0 100644 --- a/frame.go +++ b/frame.go @@ -22,7 +22,7 @@ const ( DefaultInverseEnabled = false // Default ranked frame cache - DefaultFrameCache = 50000 + DefaultCacheSize = 50000 ) // Frame represents a container for views. @@ -47,7 +47,7 @@ type Frame struct { inverseEnabled bool // Cache size for ranked frames - rankedCacheSize int + cacheSize int LogOutput io.Writer } @@ -70,10 +70,10 @@ func NewFrame(path, db, name string) (*Frame, error) { messenger: NopMessenger, stats: NopStatsClient, - rowLabel: DefaultRowLabel, - cacheType: DefaultCacheType, - inverseEnabled: DefaultInverseEnabled, - rankedCacheSize: DefaultFrameCache, + rowLabel: DefaultRowLabel, + inverseEnabled: DefaultInverseEnabled, + cacheType: DefaultCacheType, + cacheSize: DefaultCacheSize, LogOutput: ioutil.Discard, }, nil @@ -158,19 +158,19 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } -// SetRankedCacheSize sets the cache size for ranked fames. Persists to meta file on update. -// defaults to DefaultFrameCache 50000 -func (f *Frame) SetRankedCacheSize(v int) error { +// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultCacheSize 50000 +func (f *Frame) SetCacheSize(v int) error { f.mu.Lock() defer f.mu.Unlock() // Ignore if no change occurred. - if v == 0 || f.rankedCacheSize == v { + if v == 0 || f.cacheSize == v { return nil } // Persist meta data to disk on change. - f.rankedCacheSize = v + f.cacheSize = v if err := f.saveMeta(); err != nil { return err } @@ -178,10 +178,10 @@ func (f *Frame) SetRankedCacheSize(v int) error { return nil } -// RankedCacheSize returns the ranked frame cache size. -func (f *Frame) RankedCacheSize() int { +// CacheSize returns the ranked frame cache size. +func (f *Frame) CacheSize() int { f.mu.Lock() - v := f.rankedCacheSize + v := f.cacheSize f.mu.Unlock() return v } @@ -193,7 +193,7 @@ func (f *Frame) Options() FrameOptions { RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, CacheType: f.cacheType, - CacheSize: f.rankedCacheSize, + CacheSize: f.cacheSize, } f.mu.Unlock() return opt @@ -273,7 +273,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled - f.rankedCacheSize = DefaultFrameCache + f.cacheSize = DefaultCacheSize return nil } else if err != nil { return err @@ -287,7 +287,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled - f.rankedCacheSize = int(pb.CacheSize) + f.cacheSize = int(pb.CacheSize) // Copy cache type. f.cacheType = pb.CacheType @@ -306,7 +306,7 @@ func (f *Frame) saveMeta() error { RowLabel: f.rowLabel, CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, - CacheSize: int64(f.rankedCacheSize), + CacheSize: int64(f.cacheSize), }) if err != nil { return err @@ -605,7 +605,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, - CacheSize: int64(f.rankedCacheSize), + CacheSize: int64(f.cacheSize), }, } } diff --git a/frame_test.go b/frame_test.go index 16b935937..5ac6589bf 100644 --- a/frame_test.go +++ b/frame_test.go @@ -134,16 +134,16 @@ func TestFrame_SetCacheSize(t *testing.T) { cacheSize := 100 // Set & retrieve frame cache size. - if err := f.SetRankedCacheSize(cacheSize); err != nil { + if err := f.SetCacheSize(cacheSize); err != nil { t.Fatal(err) - } else if q := f.RankedCacheSize(); q != cacheSize { + } else if q := f.CacheSize(); q != cacheSize { t.Fatalf("unexpected frame cache size: %d", q) } // Reload frame and verify that it is persisted. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if q := f.RankedCacheSize(); q != cacheSize { + } else if q := f.CacheSize(); q != cacheSize { t.Fatalf("unexpected frame cache size (reopen): %d", q) } } From 3abfea140b953faa5cf6fe020217ce10efc2da37 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 21 Mar 2017 08:50:15 -0500 Subject: [PATCH 13/50] placeholder for `localNode` in HTTPNodeSet --- cluster.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster.go b/cluster.go index bbdc8f51f..44cbe90cd 100644 --- a/cluster.go +++ b/cluster.go @@ -245,6 +245,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { // HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. type HTTPNodeSet struct { nodes []*Node + localNode *Node // TODO: this needs to be set somewhere messageHandler func(m proto.Message) error // remoteStateHandler func(m proto.Message) error // localStateSource func() (proto.Message, error) @@ -279,6 +280,10 @@ func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { var g errgroup.Group for _, n := range h.nodes { + // Don't send the message to the local node. + if n == h.localNode { + continue + } node := n g.Go(func() error { return h.sendNodeMessage(node, buf) From 13e08ac3a546edce3da167bd394cfc246a712624 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 21 Mar 2017 16:06:22 -0500 Subject: [PATCH 14/50] add tests for `HTTPNodeSet` --- cluster.go | 6 +++- cluster_test.go | 6 ++-- gossip.go | 3 -- handler_test.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++ messenger_test.go | 37 +++++++++++++++++++ 5 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 messenger_test.go diff --git a/cluster.go b/cluster.go index 44cbe90cd..5d7502ed8 100644 --- a/cluster.go +++ b/cluster.go @@ -294,7 +294,11 @@ func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { // ReceiveMessage is called when a node receives a message. func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { - return h.messageHandler(pb) + if h.messageHandler != nil { + return h.messageHandler(pb) + } + // The messageHandler has not been set. + return nil } func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { diff --git a/cluster_test.go b/cluster_test.go index 2e9b01d89..4c3ec4134 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -109,9 +109,9 @@ func TestCluster_Health(t *testing.T) { // Verify a DOWN node is reported, and extraneous nodes are ignored if a := c.Health(); !reflect.DeepEqual(a, map[string]string{ - "serverA:1000": "UP", - "serverB:1000": "DOWN", - "serverC:1000": "UP", + "serverA:1000": pilosa.HealthStatusUp, + "serverB:1000": pilosa.HealthStatusDown, + "serverC:1000": pilosa.HealthStatusUp, }) { t.Fatalf("unexpected health: %s", spew.Sdump(a)) } diff --git a/gossip.go b/gossip.go index e2b3ddcde..115918533 100644 --- a/gossip.go +++ b/gossip.go @@ -1,7 +1,6 @@ package pilosa import ( - "fmt" "io" "log" "os" @@ -124,8 +123,6 @@ func (g *GossipNodeSet) NodeMeta(limit int) []byte { } func (g *GossipNodeSet) NotifyMsg(b []byte) { - loc := g.memberlist.LocalNode() - fmt.Println("Received Msg:", loc) m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) diff --git a/handler_test.go b/handler_test.go index c19a83a93..ab724b1f8 100644 --- a/handler_test.go +++ b/handler_test.go @@ -868,3 +868,93 @@ func MustReadAll(r io.Reader) []byte { } return buf } + +type MessageBin struct { + Cluster *pilosa.Cluster + messageReceived proto.Message +} + +func NewMessageBin() *MessageBin { + return &MessageBin{} +} + +func (m *MessageBin) messageHandler(pb proto.Message) error { + m.messageReceived = pb + return nil +} + +func NewHTTPMessageBin(s *Server, nodes []*pilosa.Node) (*MessageBin, error) { + ns := pilosa.NewHTTPNodeSet() + mb := NewMessageBin() + ns.SetMessageHandler(mb.messageHandler) + c := pilosa.Cluster{ + Nodes: nodes, + NodeSet: ns, + } + mb.Cluster = &c + s.Handler.Cluster = &c + s.Handler.Messenger = ns + + i, err := c.NodeSet.Join(c.Nodes) + if i != int(0) { + return nil, err + } + if err != nil { + return nil, err + } + + return mb, nil +} + +// Ensure that an HTTP message sent to the cluster reaches all nodes. +func TestHTTPNodeSet_Base(t *testing.T) { + + // servers + s1 := NewServer() + s2 := NewServer() + s3 := NewServer() + nodes := []*pilosa.Node{ + {Host: s1.Host()}, + {Host: s2.Host()}, + {Host: s3.Host()}, + } + + // node 1 + mb1, err := NewHTTPMessageBin(s1, nodes) + if err != nil { + t.Fatalf("unable to create message bin: %s", err) + } + + // node2 + mb2, err := NewHTTPMessageBin(s2, nodes) + if err != nil { + t.Fatalf("unable to create message bin: %s", err) + } + + // node3 + mb3, err := NewHTTPMessageBin(s3, nodes) + if err != nil { + t.Fatalf("unable to create message bin: %s", err) + } + + // message + msg := &internal.CreateSliceMessage{ + DB: "d", + Slice: 8, + } + + // send message + if err := mb1.Cluster.NodeSet.(pilosa.Messenger).SendMessage(msg, ""); err != nil { + t.Fatalf("failure sending message: %s", err) + } + + if !reflect.DeepEqual(mb1.messageReceived, msg) { + t.Fatalf("unexpected message received by node1: %s", mb1.messageReceived) + } + if !reflect.DeepEqual(mb2.messageReceived, msg) { + t.Fatalf("unexpected message received by node2: %s", mb2.messageReceived) + } + if !reflect.DeepEqual(mb3.messageReceived, msg) { + t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived) + } +} diff --git a/messenger_test.go b/messenger_test.go new file mode 100644 index 000000000..ee2d720a9 --- /dev/null +++ b/messenger_test.go @@ -0,0 +1,37 @@ +package pilosa_test + +import ( + "reflect" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" +) + +// Ensure a message can be marshaled and unmarshaled. +func TestMessage_Marshal(t *testing.T) { + + testMessageMarshal(t, &internal.CreateSliceMessage{ + DB: "d", + Slice: 8, + }) + + testMessageMarshal(t, &internal.DeleteDBMessage{ + DB: "d", + }) +} + +func testMessageMarshal(t *testing.T, m proto.Message) { + marshalled, err := pilosa.MarshalMessage(m) + if err != nil { + t.Fatal(err) + } + unmarshalled, err := pilosa.UnmarshalMessage(marshalled) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(unmarshalled, m) { + t.Fatalf("unexpected message marshalling: %s", unmarshalled) + } +} From cebca697b13b2bda1311e85f2dceb36fb48ea343 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 6 Apr 2017 14:28:45 -0500 Subject: [PATCH 15/50] fixing tests that were affected by the messenger/view merge --- cmd/server_test.go | 2 ++ fragment_test.go | 14 +++++++------- frame.go | 2 +- server.go | 8 +++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 927ce8b66..f1310abc2 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -106,6 +106,8 @@ data-dir = "` + actualDataDir + `" // confirm log file was written info, err := logFile.Stat() if err != nil || info.Size() == 0 { + // NOTE: this test assumes that something is being written to the log + // currently, that is relying on log: "index sync monitor initializing" return errors.New("Log file was not written!") } return nil diff --git a/fragment_test.go b/fragment_test.go index 7f3d35add..83f9b42be 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -288,7 +288,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", slice, cacheLimit), + Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", pilosa.ViewStandard, slice, cacheLimit), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -308,15 +308,15 @@ func TestFragment_TopN_CacheSize(t *testing.T) { f.RecalculateCache() p := []pilosa.Pair{ - {Key: 104, Count: 7}, - {Key: 103, Count: 6}} + {ID: 104, Count: 7}, + {ID: 103, Count: 6}} // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > cacheLimit { t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) - } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { + } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) @@ -572,7 +572,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultFrameCache) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultCacheSize) if err := f.Open(); err != nil { b.Fatal(err) } @@ -633,7 +633,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultFrameCache), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultCacheSize), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -664,7 +664,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultFrameCache) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultCacheSize) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/frame.go b/frame.go index ebfe15ab0..a15b6ee63 100644 --- a/frame.go +++ b/frame.go @@ -428,7 +428,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { } func (f *Frame) newView(path, name string) *View { - view := NewView(path, f.db, f.name, name) + view := NewView(path, f.db, f.name, name, f.cacheSize) view.cacheType = f.cacheType view.LogOutput = f.LogOutput view.BitmapAttrStore = f.bitmapAttrStore diff --git a/server.go b/server.go index 0df1182ba..5008e2d5c 100644 --- a/server.go +++ b/server.go @@ -120,11 +120,9 @@ func (s *Server) Open() error { go func() { http.Serve(ln, s.Handler) }() // Start background monitoring. - /* - s.wg.Add(2) - go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() - */ + s.wg.Add(2) + go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() + go func() { defer s.wg.Done(); s.monitorMaxSlices() }() return nil } From d99edc2eefa2ac4c262dafb3f38f54be0990a65b Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 6 Apr 2017 23:22:31 -0500 Subject: [PATCH 16/50] remove errant debugging line --- client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client.go b/client.go index d340ce640..e4401a69a 100644 --- a/client.go +++ b/client.go @@ -159,7 +159,6 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { case http.StatusOK: return nil // ok case http.StatusConflict: - fmt.Println("ErrDatabaseExists: 1") return ErrDatabaseExists default: return errors.New(string(body)) From e864a45d5715c407e9b5537710a75edb940e6aff Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 7 Apr 2017 11:12:51 -0500 Subject: [PATCH 17/50] cleaned up the comments and code for the cache threshold. Also made the max size the set value of the cache rather than the calculated index. This ensures the user will not receive less cached values than they expect --- cache.go | 39 ++++++++++++++++++++++++++------------- fragment.go | 8 +------- fragment_test.go | 4 +++- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/cache.go b/cache.go index 23bc52c30..2fe4ea567 100644 --- a/cache.go +++ b/cache.go @@ -12,6 +12,11 @@ import ( "github.com/pilosa/pilosa/internal" ) +const ( + // ThresholdFactor is used to calculate the threshold for new items entering the cache + ThresholdFactor = 1.1 +) + // Cache represents a cache for bitmap counts. type Cache interface { Add(bitmapID uint64, n uint64) @@ -111,15 +116,23 @@ type RankCache struct { updateN int updateTime time.Time - ThresholdLength int - ThresholdIndex int - ThresholdValue uint64 + // CacheLimit is the user defined size of the cache + maxEntries int + + // thresholdBuffer is used the calculate the lowest cached threshold value + // This threshold determines what new items are added to the cache + thresholdBuffer int + + // thresholdValue is the value of the last item in the cache + thresholdValue uint64 } // NewRankCache returns a new instance of RankCache. -func NewRankCache() *RankCache { +func NewRankCache(maxEntries int) *RankCache { return &RankCache{ - entries: make(map[uint64]uint64), + maxEntries: maxEntries, + thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), + entries: make(map[uint64]uint64), } } @@ -128,7 +141,7 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() // Ignore if the bit count on the bitmap is below the threshold. - if n < c.ThresholdValue { + if n < c.thresholdValue { return } @@ -141,7 +154,7 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - if n < c.ThresholdValue { + if n < c.thresholdValue { return } @@ -209,20 +222,20 @@ func (c *RankCache) recalculate() { // Store the count of the item at the threshold index. c.rankings = rankings - if len(c.rankings) > c.ThresholdIndex { - c.ThresholdValue = rankings[c.ThresholdIndex].Count - c.rankings = c.rankings[0:c.ThresholdIndex] + if len(c.rankings) > c.maxEntries { + c.thresholdValue = rankings[c.maxEntries].Count + c.rankings = c.rankings[0:c.maxEntries] } else { - c.ThresholdValue = 1 + c.thresholdValue = 1 } // Reset counters. c.updateTime, c.updateN = time.Now(), 0 // If size is larger than the threshold then trim it. - if len(c.entries) > c.ThresholdLength { + if len(c.entries) > c.thresholdBuffer { for id, cnt := range c.entries { - if cnt <= c.ThresholdValue { + if cnt <= c.thresholdValue { delete(c.entries, id) } } diff --git a/fragment.go b/fragment.go index de188d0e8..1f4a03b92 100644 --- a/fragment.go +++ b/fragment.go @@ -43,9 +43,6 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 - - // ThresholdBufferPct is the percentage of the ThresholdLength for which to maintain a sorted, ranked list. - ThresholdBufferPct = 0.9 ) const ( @@ -227,10 +224,7 @@ func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { case CacheTypeRanked: - c := NewRankCache() - c.ThresholdLength = f.cacheSize - c.ThresholdIndex = int(float64(f.cacheSize) * ThresholdBufferPct) - f.cache = c + f.cache = NewRankCache(f.cacheSize) case CacheTypeLRU: f.cache = NewLRUCache(f.cacheSize) default: diff --git a/fragment_test.go b/fragment_test.go index 83f9b42be..89bc660d3 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -309,7 +309,9 @@ func TestFragment_TopN_CacheSize(t *testing.T) { p := []pilosa.Pair{ {ID: 104, Count: 7}, - {ID: 103, Count: 6}} + {ID: 103, Count: 6}, + {ID: 102, Count: 5}, + } // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { From 8e2e9254f72d76e454d27a19c8e1e3ecbce01338 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 7 Apr 2017 13:41:40 -0500 Subject: [PATCH 18/50] fixed comment --- cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache.go b/cache.go index 2fe4ea567..19866eea1 100644 --- a/cache.go +++ b/cache.go @@ -116,7 +116,7 @@ type RankCache struct { updateN int updateTime time.Time - // CacheLimit is the user defined size of the cache + // maxEntries is the user defined size of the cache maxEntries int // thresholdBuffer is used the calculate the lowest cached threshold value From b0f1fc7523f2364613562922731fbc55dc15ba3f Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 12 Apr 2017 17:45:48 -0500 Subject: [PATCH 19/50] Makes Messenger a first-class object under Server (with pointers in Handler and Index). Primary message interface is the MessageBroker which is an attribute of the Messenger. MessageBroker implementations: - Gossip (memberlist) - Broadcast (uses HTTP, received by existing Handler) - Static (no-ops) Changes CacheSize from `int` to `uint32` for consistency with protobuf. Removes unnecessary dependencies in glide: - `github.com/aws/aws-sdk-go` - `golang.org/x/net` (although this gets included by memberlist) TODO: - [ ] Add tests around the Messenger and MessageBroker objects. - [ ] Refactor CreateSliceMessage to work with views. - [ ] Support propogation of meta data on PATCH calls. fixing some issues from last rebase --- cache.go | 10 +- cluster.go | 124 +------------------ cluster_test.go | 6 +- cmd/server.go | 2 +- config.go | 57 +++++++-- db.go | 8 +- fragment.go | 4 +- fragment_test.go | 4 +- frame.go | 32 ++--- frame_test.go | 2 +- glide.lock | 2 - glide.yaml | 1 - gossip.go | 305 +++++++++++++++++++++++++---------------------- handler.go | 11 +- handler_test.go | 81 ++++--------- index.go | 44 +------ index_test.go | 3 +- messenger.go | 262 ++++++++++++++++++++++++++++++++++++++-- server.go | 67 ++--------- server/server.go | 13 +- view.go | 4 +- 21 files changed, 544 insertions(+), 498 deletions(-) diff --git a/cache.go b/cache.go index 19866eea1..7da28eb74 100644 --- a/cache.go +++ b/cache.go @@ -44,9 +44,9 @@ type LRUCache struct { } // NewLRUCache returns a new instance of LRUCache. -func NewLRUCache(maxEntries int) *LRUCache { +func NewLRUCache(maxEntries uint32) *LRUCache { c := &LRUCache{ - cache: lru.New(maxEntries), + cache: lru.New(int(maxEntries)), counts: make(map[uint64]uint64), } c.cache.OnEvicted = c.onEvicted @@ -117,7 +117,7 @@ type RankCache struct { updateTime time.Time // maxEntries is the user defined size of the cache - maxEntries int + maxEntries uint32 // thresholdBuffer is used the calculate the lowest cached threshold value // This threshold determines what new items are added to the cache @@ -128,7 +128,7 @@ type RankCache struct { } // NewRankCache returns a new instance of RankCache. -func NewRankCache(maxEntries int) *RankCache { +func NewRankCache(maxEntries uint32) *RankCache { return &RankCache{ maxEntries: maxEntries, thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), @@ -222,7 +222,7 @@ func (c *RankCache) recalculate() { // Store the count of the item at the threshold index. c.rankings = rankings - if len(c.rankings) > c.maxEntries { + if len(c.rankings) > int(c.maxEntries) { c.thresholdValue = rankings[c.maxEntries].Count c.rankings = c.rankings[0:c.maxEntries] } else { diff --git a/cluster.go b/cluster.go index 5d7502ed8..a3678ad00 100644 --- a/cluster.go +++ b/cluster.go @@ -1,15 +1,8 @@ package pilosa import ( - "bytes" "encoding/binary" - "fmt" "hash/fnv" - "io/ioutil" - "net/http" - "net/url" - - "golang.org/x/sync/errgroup" "github.com/gogo/protobuf/proto" ) @@ -198,25 +191,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// NodeSet represents an interface to maintaining Node state. +// NodeSet represents an interface for Node membership and inter-node communication. type NodeSet interface { // Returns a list of all Nodes in the cluster Nodes() []*Node - // Attempts to join a cluster having `nodes` as its existing members - Join(nodes []*Node) (int, error) - // Open starts any network activity implemented by the NodeSet Open() error - - // SetMessageHandler provides the NodeSet with a function to call on ReceiveMessage - SetMessageHandler(f func(proto.Message) error) - - // SetRemoteStateHandler provides the function to call on MergeRemoteState - SetRemoteStateHandler(f func(proto.Message) error) - - // SetLocalStateSource provides the function to get the current node's local state. - SetLocalStateSource(f func() (proto.Message, error)) } // Hasher represents an interface to hash integers into buckets. @@ -244,11 +225,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { // HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. type HTTPNodeSet struct { - nodes []*Node - localNode *Node // TODO: this needs to be set somewhere - messageHandler func(m proto.Message) error - // remoteStateHandler func(m proto.Message) error - // localStateSource func() (proto.Message, error) + nodes []*Node } // NewHTTPNodeSet returns a new instance of HTTPNodeSet. @@ -260,103 +237,15 @@ func (h *HTTPNodeSet) Nodes() []*Node { return h.nodes } -func (h *HTTPNodeSet) Join(nodes []*Node) (int, error) { - h.nodes = nodes - return 0, nil -} - func (h *HTTPNodeSet) Open() error { return nil } -// SendMessage asyncronously broadcasts a protobuf message to all nodes. -func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { - - // Marshal the pb to []byte - buf, err := MarshalMessage(pb) - if err != nil { - return err - } - - var g errgroup.Group - for _, n := range h.nodes { - // Don't send the message to the local node. - if n == h.localNode { - continue - } - node := n - g.Go(func() error { - return h.sendNodeMessage(node, buf) - }) - } - return g.Wait() -} - -// ReceiveMessage is called when a node receives a message. -func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { - if h.messageHandler != nil { - return h.messageHandler(pb) - } - // The messageHandler has not been set. +func (h *HTTPNodeSet) Join(nodes []*Node) error { + h.nodes = nodes return nil } -func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { - var client *http.Client - client = http.DefaultClient - - // Create HTTP request. - req, err := http.NewRequest("POST", (&url.URL{ - Scheme: "http", - Host: node.Host, - Path: "/message", - }).String(), bytes.NewReader(msg)) - if err != nil { - return err - } - - // Require protobuf encoding. - req.Header.Set("Content-Type", "application/x-protobuf") - - // Send request to remote node. - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - // Read response into buffer. - body, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return err - } - - // Check status code. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) - } - - return nil -} - -// SetMessageHandler provides the Messenger with a function to handle incoming messages. -func (h *HTTPNodeSet) SetMessageHandler(f func(proto.Message) error) { - h.messageHandler = f -} - -// SetRemoteStateHandler provides the Messenger with a function to merge remote state. -func (h *HTTPNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { - // not implemented - // h.remoteStateHandler = f -} - -// SetLocalStateSource currently no-ops. -func (h *HTTPNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { - // not implemented - // h.localStateSource = f -} - // StaticNodeSet represents a basic NodeSet for testing type StaticNodeSet struct { Messenger @@ -371,11 +260,6 @@ func (s *StaticNodeSet) Nodes() []*Node { return s.nodes } -func (s *StaticNodeSet) Join(nodes []*Node) (int, error) { - s.nodes = nodes - return 0, nil -} - func (s *StaticNodeSet) Open() error { return nil } diff --git a/cluster_test.go b/cluster_test.go index 4c3ec4134..1dd6bb486 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -95,16 +95,16 @@ func TestCluster_Health(t *testing.T) { {Host: "serverB:1000"}, {Host: "serverC:1000"}, }, - NodeSet: &pilosa.StaticNodeSet{}, + NodeSet: &pilosa.HTTPNodeSet{}, } - j, err := c.NodeSet.Join([]*pilosa.Node{ + err := c.NodeSet.(*pilosa.HTTPNodeSet).Join([]*pilosa.Node{ &pilosa.Node{Host: "serverA:1000"}, &pilosa.Node{Host: "serverC:1000"}, &pilosa.Node{Host: "serverD:1000"}, }) if err != nil { - t.Fatalf("unexpected gossiper nodes: %s", j) + t.Fatalf("unexpected gossiper nodes: %s", err) } // Verify a DOWN node is reported, and extraneous nodes are ignored diff --git a/cmd/server.go b/cmd/server.go index d25066764..27c238eb6 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -83,7 +83,7 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "", "Type of Messenger to use for inter-host messaging.") + flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "static", "Type of Messenger to use for inter-host messaging. Choose from [static, broadcast, gossip]") flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") diff --git a/config.go b/config.go index 17edd639d..b74125ce7 100644 --- a/config.go +++ b/config.go @@ -2,14 +2,16 @@ package pilosa import ( "net" + "strconv" "time" ) const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultGossipPort = 14000 + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultMessengerType = "static" + DefaultGossipPort = "14000" ) // Config represents the configuration for the command. @@ -47,6 +49,7 @@ func NewConfig() *Config { Host: DefaultHost + ":" + DefaultPort, } c.Cluster.ReplicaN = DefaultReplicaN + c.Cluster.MessengerType = DefaultMessengerType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) @@ -63,6 +66,24 @@ func NewConfigForHosts(hosts []string) *Config { return conf } +// PilosaMessenger returns a new instance of Messenger based on the config. +func (c *Config) PilosaMessenger() *Messenger { + messenger := NewMessenger() + switch c.Cluster.MessengerType { + case "broadcast": + n := NewHTTPMessageBroker() + n.messenger = messenger + messenger.Broker = n + case "gossip": + n := NewGossipMessageBroker() + n.messenger = messenger + messenger.Broker = n + case "static": + // nop + } + return messenger +} + // PilosaCluster returns a new instance of Cluster based on the config. func (c *Config) PilosaCluster() *Cluster { cluster := NewCluster() @@ -73,11 +94,16 @@ func (c *Config) PilosaCluster() *Cluster { } // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - if c.Cluster.MessengerType == "broadcast" { + switch c.Cluster.MessengerType { + case "broadcast": cluster.NodeSet = NewHTTPNodeSet() - cluster.NodeSet.Join(cluster.Nodes) - } else if c.Cluster.MessengerType == "gossip" { - gossipPort := DefaultGossipPort + cluster.NodeSet.(*HTTPNodeSet).Join(cluster.Nodes) + case "gossip": + gport, err := strconv.Atoi(DefaultGossipPort) + if err != nil { + // what? + } + gossipPort := gport gossipSeed := DefaultHost if c.Cluster.Gossip.Port != 0 { gossipPort = c.Cluster.Gossip.Port @@ -91,13 +117,28 @@ func (c *Config) PilosaCluster() *Cluster { gossipHost = c.Host } cluster.NodeSet = NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) - } else { + case "static": + cluster.NodeSet = NewStaticNodeSet() + default: cluster.NodeSet = NewStaticNodeSet() } return cluster } +// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker +// after cluster configuration. +func (c *Config) AssociateMessageBroker(s *Server) { + switch c.Cluster.MessengerType { + case "broadcast": + // nop + case "gossip": + s.Cluster.NodeSet.(*GossipNodeSet).config.memberlistConfig.Delegate = s.Messenger.Broker.(*GossipMessageBroker) + case "static": + // nop + } +} + // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/db.go b/db.go index a7f6f8ff1..cc27e18c9 100644 --- a/db.go +++ b/db.go @@ -43,7 +43,7 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - messenger Messenger + messenger *Messenger stats StatsClient LogOutput io.Writer @@ -68,7 +68,6 @@ func NewDB(path, name string) (*DB, error) { columnLabel: DefaultColumnLabel, - messenger: NopMessenger, stats: NopStatsClient, LogOutput: ioutil.Discard, }, nil @@ -394,10 +393,7 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { f.rowLabel = opt.RowLabel } if opt.CacheSize != 0 { - f.rankedCacheSize = opt.CacheSize - } - if opt.TimeQuantum.Valid() { - f.timeQuantum = opt.TimeQuantum + f.cacheSize = opt.CacheSize } f.inverseEnabled = opt.InverseEnabled diff --git a/fragment.go b/fragment.go index 1f4a03b92..5704e774d 100644 --- a/fragment.go +++ b/fragment.go @@ -70,7 +70,7 @@ type Fragment struct { // Cache for bitmap counts. cacheType string // passed in by frame cache Cache - cacheSize int + cacheSize uint32 // Cache containing full bitmaps (not just counts). bitmapCache BitmapCache @@ -94,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64, cacheSize int) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64, cacheSize uint32) *Fragment { return &Fragment{ path: path, db: db, diff --git a/fragment_test.go b/fragment_test.go index 89bc660d3..e54b11f82 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -280,7 +280,7 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) - cacheLimit := 3 + cacheLimit := uint32(3) file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -316,7 +316,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) - } else if len(pairs) > cacheLimit { + } else if len(pairs) > int(cacheLimit) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) diff --git a/frame.go b/frame.go index a15b6ee63..c5a40b1d1 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,7 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - messenger Messenger + messenger *Messenger stats StatsClient // Frame settings. @@ -47,7 +47,7 @@ type Frame struct { inverseEnabled bool // Cache size for ranked frames - cacheSize int + cacheSize uint32 LogOutput io.Writer } @@ -67,8 +67,7 @@ func NewFrame(path, db, name string) (*Frame, error) { views: make(map[string]*View), bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), - messenger: NopMessenger, - stats: NopStatsClient, + stats: NopStatsClient, rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, @@ -160,7 +159,7 @@ func (f *Frame) InverseEnabled() bool { // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 -func (f *Frame) SetCacheSize(v int) error { +func (f *Frame) SetCacheSize(v uint32) error { f.mu.Lock() defer f.mu.Unlock() @@ -179,7 +178,7 @@ func (f *Frame) SetCacheSize(v int) error { } // CacheSize returns the ranked frame cache size. -func (f *Frame) CacheSize() int { +func (f *Frame) CacheSize() uint32 { f.mu.Lock() v := f.cacheSize f.mu.Unlock() @@ -194,6 +193,7 @@ func (f *Frame) Options() FrameOptions { InverseEnabled: f.inverseEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, + TimeQuantum: f.timeQuantum, } f.mu.Unlock() return opt @@ -287,7 +287,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled - f.cacheSize = int(pb.CacheSize) + f.cacheSize = pb.CacheSize // Copy cache type. f.cacheType = pb.CacheType @@ -301,12 +301,12 @@ func (f *Frame) loadMeta() error { // saveMeta writes meta data for the frame. func (f *Frame) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.Frame{ + buf, err := proto.Marshal(&internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, - CacheSize: int64(f.cacheSize), + CacheSize: f.cacheSize, }) if err != nil { return err @@ -414,16 +414,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { view.BitmapAttrStore = f.bitmapAttrStore f.views[view.Name()] = view - // TODO: this needs to be refactored for views - /* - // Send a MaxSlice message - f.messenger.SendMessage( - &internal.CreateSliceMessage{ - DB: f.db, - Slice: slice, - }, "gossip") - */ - return view, nil } @@ -605,7 +595,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, - CacheSize: int64(f.cacheSize), + CacheSize: f.cacheSize, }, } } @@ -633,7 +623,7 @@ type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` - CacheSize int `json:"cacheSize,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } diff --git a/frame_test.go b/frame_test.go index 5ac6589bf..4814ca67e 100644 --- a/frame_test.go +++ b/frame_test.go @@ -131,7 +131,7 @@ func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time func TestFrame_SetCacheSize(t *testing.T) { f := MustOpenFrame() defer f.Close() - cacheSize := 100 + cacheSize := uint32(100) // Set & retrieve frame cache size. if err := f.SetCacheSize(cacheSize); err != nil { diff --git a/glide.lock b/glide.lock index e4b6ce617..ae504264c 100644 --- a/glide.lock +++ b/glide.lock @@ -3,8 +3,6 @@ updated: 2017-04-18T15:33:39.035615802-05:00 imports: - name: github.com/armon/go-metrics version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 -- name: github.com/aws/aws-sdk-go - version: 819b71cf8430e434c1eee7e7e8b0f2b8870be899 - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml diff --git a/glide.yaml b/glide.yaml index f317a0a1b..d81cbaf28 100644 --- a/glide.yaml +++ b/glide.yaml @@ -35,4 +35,3 @@ import: version: ^1.6.10 - package: github.com/hashicorp/memberlist - package: golang.org/x/sync -- package: golang.org/x/net diff --git a/gossip.go b/gossip.go index 115918533..b25b634dd 100644 --- a/gossip.go +++ b/gossip.go @@ -16,14 +16,9 @@ import ( // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { memberlist *memberlist.Memberlist - broadcasts *memberlist.TransmitLimitedQueue config *GossipConfig - messageHandler func(m proto.Message) error - remoteStateHandler func(m proto.Message) error - localStateSource func() (proto.Message, error) - // The writer for any logging. LogOutput io.Writer } @@ -36,10 +31,6 @@ func (g *GossipNodeSet) Nodes() []*Node { return a } -func (g *GossipNodeSet) Join(nodes []*Node) (int, error) { - return g.memberlist.Join(Nodes(nodes).Hosts()) -} - func (g *GossipNodeSet) Open() error { ml, err := memberlist.Create(g.config.memberlistConfig) if err != nil { @@ -48,152 +39,19 @@ func (g *GossipNodeSet) Open() error { g.memberlist = ml // attach to gossip seed node - g.Join([]*Node{&Node{Host: g.config.gossipSeed}}) //TODO: support a list of seeds - - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - return g.memberlist.NumMembers() - }, - RetransmitMult: 3, - } - return nil -} - -func (g *GossipNodeSet) SetMessageHandler(f func(proto.Message) error) { - g.messageHandler = f -} - -func (g *GossipNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { - g.remoteStateHandler = f -} - -func (g *GossipNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { - g.localStateSource = f -} - -// implementation of the messenger.Messenger interface -func (g *GossipNodeSet) SendMessage(pb proto.Message, method string) error { - msg, err := MarshalMessage(pb) - if err != nil { - return err - } - - // Broadcast asyncronously sends the message directly to each node. - // An error from any node raises an error on the entire operation. - // This is a blocking operation. - // - // Gossip uses the gossip protocol to eventually deliver the message - // to every node. - switch method { - case "broadcast": - var eg errgroup.Group - for _, n := range g.memberlist.Members() { - // Don't send the message to the local node. - if n == g.memberlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return g.memberlist.SendToTCP(node, msg) - }) - } - return eg.Wait() - case "gossip": - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) - } - - return nil -} - -func (g *GossipNodeSet) ReceiveMessage(pb proto.Message) error { - err := g.messageHandler(pb) + nodes := []*Node{&Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds + _, err = g.memberlist.Join(Nodes(nodes).Hosts()) if err != nil { return err } return nil } -// implementation of the memberlist.Delegate interface -func (g *GossipNodeSet) NodeMeta(limit int) []byte { - return []byte{} -} - -func (g *GossipNodeSet) NotifyMsg(b []byte) { - m, err := UnmarshalMessage(b) - if err != nil { - g.logger().Printf("unmarshal message error: %s", err) - return - } - if err := g.ReceiveMessage(m); err != nil { - g.logger().Printf("receive message error: %s", err) - return - } -} - -func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { - return g.broadcasts.GetBroadcasts(overhead, limit) -} - -func (g *GossipNodeSet) LocalState(join bool) []byte { - - pb, err := g.localStateSource() - if err != nil { - g.logger().Printf("error getting local state, err=%s", err) - return []byte{} - } - - // Marshal nodestate data to bytes. - buf, err := proto.Marshal(pb) - if err != nil { - g.logger().Printf("error marshaling nodestate data, err=%s", err) - return []byte{} - } - return buf -} - -func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { - // Unmarshal nodestate data. - var pb internal.NodeState - if err := proto.Unmarshal(buf, &pb); err != nil { - g.logger().Printf("error unmarshaling nodestate data, err=%s", err) - return - } - err := g.remoteStateHandler(&pb) - if err != nil { - g.logger().Printf("merge state error: %s", err) - } - return -} - // logger returns a logger for the GossipNodeSet. func (g *GossipNodeSet) logger() *log.Logger { return log.New(g.LogOutput, "", log.LstdFlags) } -// broadcast represents an implementation of memberlist.Broadcast -type broadcast struct { - msg []byte - notify chan<- struct{} -} - -func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { - return false -} - -func (b *broadcast) Message() []byte { - return b.msg -} - -func (b *broadcast) Finished() { - if b.notify != nil { - close(b.notify) - } -} - //////////////////////////////////////////////////////////////// type GossipConfig struct { @@ -217,7 +75,164 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = gossipHost g.config.memberlistConfig.AdvertisePort = gossipPort - g.config.memberlistConfig.Delegate = g return g } + +//////////////////////////////////////////////////////////////// + +// GossipMessageBroker represents a gossip implementation of pilosa.MessageBroker +// GossipMessageBroker also represents an implementation of memberlist.Delegate +type GossipMessageBroker struct { + broadcasts *memberlist.TransmitLimitedQueue + + messenger *Messenger + + // The writer for any logging. + LogOutput io.Writer +} + +// implementation of the messenger.Messenger interface +func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { + msg, err := MarshalMessage(pb) + if err != nil { + return err + } + + mlist := g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist + + // Direct sends the message directly to every node. + // An error from any node raises an error on the entire operation. + // + // Gossip uses the gossip protocol to eventually deliver the message + // to every node. + switch method { + case "direct": + var eg errgroup.Group + for _, n := range mlist.Members() { + // Don't send the message to the local node. + if n == mlist.LocalNode() { + continue + } + node := n + eg.Go(func() error { + return mlist.SendToTCP(node, msg) + }) + } + return eg.Wait() + case "gossip": + b := &broadcast{ + msg: msg, + notify: nil, + } + g.broadcasts.QueueBroadcast(b) + } + + return nil +} + +func (g *GossipMessageBroker) Receive(pb proto.Message) error { + if err := g.messenger.ReceiveMessage(pb); err != nil { + return err + } + return nil +} + +func (g *GossipMessageBroker) SetMessenger(m *Messenger) { + g.messenger = m +} + +// implementation of the memberlist.Delegate interface +func (g *GossipMessageBroker) NodeMeta(limit int) []byte { + return []byte{} +} + +func (g *GossipMessageBroker) NotifyMsg(b []byte) { + m, err := UnmarshalMessage(b) + if err != nil { + g.logger().Printf("unmarshal message error: %s", err) + return + } + if err := g.Receive(m); err != nil { + g.logger().Printf("receive message error: %s", err) + return + } +} + +func (g *GossipMessageBroker) GetBroadcasts(overhead, limit int) [][]byte { + return g.broadcasts.GetBroadcasts(overhead, limit) +} + +func (g *GossipMessageBroker) LocalState(join bool) []byte { + pb, err := g.messenger.LocalState() + if err != nil { + g.logger().Printf("error getting local state, err=%s", err) + return []byte{} + } + + // Marshal nodestate data to bytes. + buf, err := proto.Marshal(pb) + if err != nil { + g.logger().Printf("error marshalling nodestate data, err=%s", err) + return []byte{} + } + return buf +} + +func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { + // Unmarshal nodestate data. + var pb internal.NodeState + if err := proto.Unmarshal(buf, &pb); err != nil { + g.logger().Printf("error unmarshalling nodestate data, err=%s", err) + return + } + err := g.messenger.HandleRemoteState(&pb) + if err != nil { + g.logger().Printf("merge state error: %s", err) + } +} + +// logger returns a logger for the GossipMessageBroker. +func (g *GossipMessageBroker) logger() *log.Logger { + return log.New(g.LogOutput, "", log.LstdFlags) +} + +//////////////////////////////////////////////////////////////// + +// NewGossipMessageBroker returns a new instance of GossipMessageBroker. +func NewGossipMessageBroker() *GossipMessageBroker { + g := &GossipMessageBroker{ + LogOutput: os.Stderr, + } + + g.broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() + }, + RetransmitMult: 3, + } + + return g +} + +//////////////////////////////////////////////////////////////// + +// broadcast represents an implementation of memberlist.Broadcast +type broadcast struct { + msg []byte + notify chan<- struct{} +} + +func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { + return false +} + +func (b *broadcast) Message() []byte { + return b.msg +} + +func (b *broadcast) Finished() { + if b.notify != nil { + close(b.notify) + } +} diff --git a/handler.go b/handler.go index ad337b2bc..0a7cb278d 100644 --- a/handler.go +++ b/handler.go @@ -26,7 +26,7 @@ import ( // Handler represents an HTTP handler. type Handler struct { Index *Index - Messenger Messenger + Messenger *Messenger // Local hostname & cluster configuration. Host string @@ -50,7 +50,6 @@ type Handler struct { func NewHandler() *Handler { handler := &Handler{ LogOutput: os.Stderr, - Messenger: NopMessenger, } handler.Router = NewRouter(handler) return handler @@ -214,7 +213,7 @@ func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { return } - if err := h.Messenger.ReceiveMessage(m); err != nil { + if err := h.Messenger.Broker.Receive(m); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -373,7 +372,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { err := h.Messenger.SendMessage( &internal.DeleteDBMessage{ DB: req.DB, - }, "broadcast") + }, "direct") if err != nil { h.logger().Printf("problem sending DeleteDB message: %s", err) } @@ -522,7 +521,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { RowLabel: req.Options.RowLabel, TimeQuantum: string(req.Options.TimeQuantum), }, - }, "broadcast") + }, "direct") if err != nil { h.logger().Printf("problem sending CreateFrame message: %s", err) } @@ -590,7 +589,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, - }, "broadcast") + }, "direct") if err != nil { h.logger().Printf("problem sending DeleteFrame message: %s", err) } diff --git a/handler_test.go b/handler_test.go index ab724b1f8..92bcc7601 100644 --- a/handler_test.go +++ b/handler_test.go @@ -792,6 +792,10 @@ func NewHandler() *Handler { } h.Handler.Executor = &h.Executor h.Handler.LogOutput = ioutil.Discard + + // Handler test messages can no-op. + h.Messenger = pilosa.NewMessenger() + return h } @@ -823,6 +827,9 @@ func NewServer() *Server { // Update handler to use hostname. s.Handler.Host = s.Host() + // Handler test messages can no-op. + s.Handler.Messenger = pilosa.NewMessenger() + // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() @@ -869,74 +876,37 @@ func MustReadAll(r io.Reader) []byte { return buf } -type MessageBin struct { - Cluster *pilosa.Cluster - messageReceived proto.Message -} - -func NewMessageBin() *MessageBin { - return &MessageBin{} -} - -func (m *MessageBin) messageHandler(pb proto.Message) error { - m.messageReceived = pb - return nil -} - -func NewHTTPMessageBin(s *Server, nodes []*pilosa.Node) (*MessageBin, error) { - ns := pilosa.NewHTTPNodeSet() - mb := NewMessageBin() - ns.SetMessageHandler(mb.messageHandler) - c := pilosa.Cluster{ - Nodes: nodes, - NodeSet: ns, - } - mb.Cluster = &c - s.Handler.Cluster = &c - s.Handler.Messenger = ns - - i, err := c.NodeSet.Join(c.Nodes) - if i != int(0) { - return nil, err - } - if err != nil { - return nil, err - } - - return mb, nil -} +/* +// TODO: move this test to messenger.go (with NewServer()) // Ensure that an HTTP message sent to the cluster reaches all nodes. func TestHTTPNodeSet_Base(t *testing.T) { // servers s1 := NewServer() + s1.Messenger = pilosa.NewMessenger() + n1 := NewHTTPMessageBroker() + n1.messenger = s1.Messenger + s1.Messenger.Broker = n1 + s2 := NewServer() + s2.Messenger = pilosa.NewMessenger() + n2 := NewHTTPMessageBroker() + n2.messenger = s2.Messenger + s2.Messenger.Broker = n2 + s3 := NewServer() + s3.Messenger = pilosa.NewMessenger() + n3 := NewHTTPMessageBroker() + n3.messenger = s3.Messenger + s3.Messenger.Broker = n3 + nodes := []*pilosa.Node{ {Host: s1.Host()}, {Host: s2.Host()}, {Host: s3.Host()}, } - // node 1 - mb1, err := NewHTTPMessageBin(s1, nodes) - if err != nil { - t.Fatalf("unable to create message bin: %s", err) - } - - // node2 - mb2, err := NewHTTPMessageBin(s2, nodes) - if err != nil { - t.Fatalf("unable to create message bin: %s", err) - } - - // node3 - mb3, err := NewHTTPMessageBin(s3, nodes) - if err != nil { - t.Fatalf("unable to create message bin: %s", err) - } - // message msg := &internal.CreateSliceMessage{ DB: "d", @@ -944,7 +914,7 @@ func TestHTTPNodeSet_Base(t *testing.T) { } // send message - if err := mb1.Cluster.NodeSet.(pilosa.Messenger).SendMessage(msg, ""); err != nil { + if err := s1.Messenger.SendMessage(msg, ""); err != nil { t.Fatalf("failure sending message: %s", err) } @@ -958,3 +928,4 @@ func TestHTTPNodeSet_Base(t *testing.T) { t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived) } } +*/ diff --git a/index.go b/index.go index 954629eea..5e9f5187e 100644 --- a/index.go +++ b/index.go @@ -11,9 +11,6 @@ import ( "sort" "sync" "time" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. @@ -26,7 +23,7 @@ type Index struct { // Databases by name. dbs map[string]*DB - Messenger Messenger + Messenger *Messenger // Close management wg sync.WaitGroup @@ -50,8 +47,7 @@ func NewIndex() *Index { dbs: make(map[string]*DB), closing: make(chan struct{}, 0), - Messenger: NopMessenger, - Stats: NopStatsClient, + Stats: NopStatsClient, CacheFlushInterval: DefaultCacheFlushInterval, @@ -347,42 +343,6 @@ func (i *Index) flushCaches() { } } -// HandleMessage handles protobuf Messages broadcasted to nodes in the -// cluster from the Cluster's NodeSet. -func (i *Index) HandleMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateSliceMessage: - d := i.DB(obj.DB) - if d == nil { - return fmt.Errorf("Local DB not found: %s", obj.DB) - } - d.SetRemoteMaxSlice(obj.Slice) - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} - _, err := i.CreateDB(obj.DB, opt) - if err != nil { - return err - } - case *internal.DeleteDBMessage: - if err := i.DeleteDB(obj.DB); err != nil { - return err - } - case *internal.CreateFrameMessage: - db := i.DB(obj.DB) - opt := FrameOptions{RowLabel: obj.Meta.RowLabel} - _, err := db.CreateFrame(obj.Frame, opt) - if err != nil { - return err - } - case *internal.DeleteFrameMessage: - db := i.DB(obj.DB) - if err := db.DeleteFrame(obj.Frame); err != nil { - return err - } - } - return nil -} - func (i *Index) logger() *log.Logger { return log.New(i.LogOutput, "", log.LstdFlags) } // IndexSyncer is an active anti-entropy tool that compares the local index diff --git a/index_test.go b/index_test.go index 50ce1979e..c09fe69f5 100644 --- a/index_test.go +++ b/index_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -163,6 +162,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } } +/* TODO: move this to messenger.go // Ensure index can handle Messenger messages. func TestIndex_HandleMessage(t *testing.T) { // Create a local index. @@ -188,6 +188,7 @@ func TestIndex_HandleMessage(t *testing.T) { t.Fatalf("unexpected delete db: %s", ms) } } +*/ // Index is a test wrapper for pilosa.Index. type Index struct { diff --git a/messenger.go b/messenger.go index f28eb7ddb..03e15ba0e 100644 --- a/messenger.go +++ b/messenger.go @@ -1,36 +1,276 @@ package pilosa import ( + "bytes" + "errors" "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" "reflect" + "golang.org/x/sync/errgroup" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) +// Messenger represents an internal message handler. +type Messenger struct { + + // Broker handles Send/Receive Messages. + Broker MessageBroker + + Index *Index + + // Local hostname & cluster configuration. + Host string + Cluster *Cluster + + // The writer for any logging. + LogOutput io.Writer +} + +// NewMessenger returns a new instance of Messenger with a default logger. +func NewMessenger() *Messenger { + return &Messenger{ + Broker: NopMessageBroker, + LogOutput: os.Stderr, + } +} + +func (m *Messenger) SendMessage(pb proto.Message, method string) error { + if m.Broker == nil { + return errors.New("Messenger.Broker is not defined.") + } + return m.Broker.Send(pb, method) +} +func (m *Messenger) ReceiveMessage(pb proto.Message) error { + return m.handleMessage(pb) +} + +// handleMessage handles protobuf Messages sent to nodes in the cluster. +func (m *Messenger) handleMessage(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.CreateSliceMessage: + d := m.Index.DB(obj.DB) + if d == nil { + return fmt.Errorf("Local DB not found: %s", obj.DB) + } + d.SetRemoteMaxSlice(obj.Slice) + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := m.Index.CreateDB(obj.DB, opt) + if err != nil { + return err + } + case *internal.DeleteDBMessage: + fmt.Println("DELETE:", obj.DB) + if err := m.Index.DeleteDB(obj.DB); err != nil { + return err + } + case *internal.CreateFrameMessage: + db := m.Index.DB(obj.DB) + opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + _, err := db.CreateFrame(obj.Frame, opt) + if err != nil { + return err + } + case *internal.DeleteFrameMessage: + db := m.Index.DB(obj.DB) + if err := db.DeleteFrame(obj.Frame); err != nil { + return err + } + } + return nil +} + +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +// It seems odd to have this as part of Messenger, but with the +// exception of Server, it's currenntly the only object with access +// to the necessary information (Host, Index, Cluster). +func (m *Messenger) LocalState() (proto.Message, error) { + if m.Index == nil { + return nil, errors.New("Messenger.Index is nil.") + } + return &internal.NodeState{ + Host: m.Host, + State: "OK", // TODO: make this work, pull from m.Cluster.Node + DBs: encodeDBs(m.Index.DBs()), + }, nil +} + +// HandleRemoteState receives incoming NodeState from remote nodes. +func (m *Messenger) HandleRemoteState(pb proto.Message) error { + return m.mergeRemoteState(pb.(*internal.NodeState)) +} + +func (m *Messenger) mergeRemoteState(ns *internal.NodeState) error { + // TODO: update some node state value in the cluster (it should be in cluster.node i guess) + + // Create databases that don't exist. + for _, db := range ns.DBs { + opt := DBOptions{ + ColumnLabel: db.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + } + d, err := m.Index.CreateDBIfNotExists(db.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range db.Frames { + opt := FrameOptions{ + RowLabel: f.Meta.RowLabel, + TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), + CacheSize: f.Meta.CacheSize, + } + _, err := d.CreateFrameIfNotExists(f.Name, opt) + if err != nil { + return err + } + } + } + + return nil +} + +////////////////////////////////////////////////////////////////// + +// MessageBroker is an interface for handling incoming/outgoing messages. +type MessageBroker interface { + Send(pb proto.Message, method string) error + Receive(pb proto.Message) error + SetMessenger(m *Messenger) +} + +////////////////////////////////////////////////////////////////// + func init() { - NopMessenger = &nopMessenger{} + NopMessageBroker = &nopMessageBroker{} } -var NopMessenger Messenger +var NopMessageBroker MessageBroker -// nopMessenger represents a Messenger that doesn't do anything. -type nopMessenger struct{} +// nopMessageBroker represents a MessageBroker that doesn't do anything. +type nopMessageBroker struct{} -func (c *nopMessenger) SendMessage(pb proto.Message, method string) error { - fmt.Println("NOPMessenger: Send") +func (c *nopMessageBroker) Send(pb proto.Message, method string) error { + fmt.Println("NOPMessageBroker: Send") return nil } -func (c *nopMessenger) ReceiveMessage(pb proto.Message) error { - fmt.Println("NOPMessenger: Receive") +func (c *nopMessageBroker) Receive(pb proto.Message) error { + fmt.Println("NOPMessageBroker: Receive") + return nil +} +func (c *nopMessageBroker) SetMessenger(m *Messenger) {} + +////////////////////////////////////////////////////////////////// + +// HTTPMessageBroker represents a NodeSet that broadcasts messages over HTTP. +type HTTPMessageBroker struct { + messenger *Messenger +} + +// NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. +func NewHTTPMessageBroker() *HTTPMessageBroker { + return &HTTPMessageBroker{} +} + +// Send sends a protobuf message to all nodes simultaneously. +// It waits for all nodes to respond before the function returns (and returns any errors). +func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { + // Marshal the pb to []byte + buf, err := MarshalMessage(pb) + if err != nil { + return err + } + + nodes, err := h.nodes() + if err != nil { + return err + } + + var g errgroup.Group + for _, n := range nodes { + // Don't send the message to the local node. + if n.Host == h.messenger.Host { + continue + } + node := n + g.Go(func() error { + return h.sendNodeMessage(node, buf) + }) + } + return g.Wait() +} + +// Receive is called when a node receives a message. +func (h *HTTPMessageBroker) Receive(pb proto.Message) error { + if err := h.messenger.ReceiveMessage(pb); err != nil { + return err + } return nil } -type Messenger interface { - SendMessage(pb proto.Message, method string) error - ReceiveMessage(pb proto.Message) error +func (h *HTTPMessageBroker) SetMessenger(m *Messenger) {} + +func (h *HTTPMessageBroker) nodes() ([]*Node, error) { + if h.messenger == nil { + return nil, errors.New("HTTPMessageBroker has no reference to Messenger.") + } + nodeset, ok := h.messenger.Cluster.NodeSet.(*HTTPNodeSet) + if !ok { + return nil, errors.New("NodeSet cannot be caste to HTTPNodeSet.") + } + return nodeset.Nodes(), nil } +func (h *HTTPMessageBroker) sendNodeMessage(node *Node, msg []byte) error { + var client *http.Client + client = http.DefaultClient + + // Create HTTP request. + req, err := http.NewRequest("POST", (&url.URL{ + Scheme: "http", + Host: node.Host, + Path: "/message", + }).String(), bytes.NewReader(msg)) + if err != nil { + return err + } + + // Require protobuf encoding. + req.Header.Set("Content-Type", "application/x-protobuf") + + // Send request to remote node. + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read response into buffer. + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return err + } + + // Check status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) + } + + return nil +} + +////////////////////////////////////////////////////////////////// + const ( MessageTypeCreateSlice = 1 MessageTypeCreateDB = 2 diff --git a/server.go b/server.go index 5008e2d5c..a93d29f57 100644 --- a/server.go +++ b/server.go @@ -34,7 +34,7 @@ type Server struct { // Data storage and HTTP interface. Index *Index Handler *Handler - Messenger Messenger + Messenger *Messenger // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -55,7 +55,7 @@ func NewServer() *Server { Index: NewIndex(), Handler: NewHandler(), - Messenger: NopMessenger, + Messenger: NewMessenger(), AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -64,6 +64,7 @@ func NewServer() *Server { } s.Handler.Index = s.Index + s.Messenger.Index = s.Index return s } @@ -109,11 +110,21 @@ func (s *Server) Open() error { e.Host = s.Host e.Cluster = s.Cluster + // Initialize Messenger. + s.Messenger.Index = s.Index + s.Messenger.Host = s.Host + s.Messenger.Cluster = s.Cluster + s.Messenger.LogOutput = s.LogOutput + // Initialize HTTP handler. + s.Handler.Messenger = s.Messenger s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput + + // Initialize Index. + s.Index.Messenger = s.Messenger s.Index.LogOutput = s.LogOutput // Serve HTTP. @@ -151,49 +162,6 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } -// LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. -func (s *Server) LocalState() (proto.Message, error) { - // TODO: are there errors to handle? - pb := encodeLocalState(s) - return pb, nil -} - -// HandleRemoteState provides the current, local state. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) HandleRemoteState(pb proto.Message) error { - return s.mergeRemoteState(pb.(*internal.NodeState)) -} - -func (s *Server) mergeRemoteState(ns *internal.NodeState) error { - // TODO: update some node state value in the cluster (it should be in cluster.node i guess) - - // Create databases that don't exist. - for _, db := range ns.DBs { - opt := DBOptions{ - ColumnLabel: db.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), - } - d, err := s.Index.CreateDBIfNotExists(db.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range db.Frames { - opt := FrameOptions{ - RowLabel: f.Meta.RowLabel, - TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), - } - _, err := d.CreateFrameIfNotExists(f.Name, opt) - if err != nil { - return err - } - } - } - - return nil -} - func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { @@ -230,15 +198,6 @@ func (s *Server) monitorAntiEntropy() { } } -// encodeLocalState converts s into its internal representation. -func encodeLocalState(s *Server) *internal.NodeState { - return &internal.NodeState{ - Host: s.Host, - State: "OK", // TODO: make this work, pull from cluster.Node - DBs: encodeDBs(s.Index.DBs()), - } -} - // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. func (s *Server) monitorMaxSlices() { // Ignore if only one node in the cluster. diff --git a/server/server.go b/server/server.go index 3a62c5c60..ba41f3dd0 100644 --- a/server/server.go +++ b/server/server.go @@ -92,18 +92,11 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } + m.Server.Messenger = m.Config.PilosaMessenger() m.Server.Cluster = m.Config.PilosaCluster() - // Setup Messenger. - fmt.Fprintf(m.Stderr, "Using Messenger type: %s\n", m.Config.Cluster.MessengerType) - m.Server.Messenger = m.Server.Cluster.NodeSet.(pilosa.Messenger) - m.Server.Handler.Messenger = m.Server.Messenger - m.Server.Index.Messenger = m.Server.Messenger - - // Set message and state handlers. - m.Server.Cluster.NodeSet.SetMessageHandler(m.Server.Index.HandleMessage) - m.Server.Cluster.NodeSet.SetRemoteStateHandler(m.Server.HandleRemoteState) - m.Server.Cluster.NodeSet.SetLocalStateSource(m.Server.LocalState) + // Associate objects to the MessageBroker based on config. + m.Config.AssociateMessageBroker(m.Server) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) diff --git a/view.go b/view.go index 0f554de92..9ded867ce 100644 --- a/view.go +++ b/view.go @@ -30,7 +30,7 @@ type View struct { frame string name string - cacheSize int + cacheSize uint32 // Fragments by slice. cacheType string // passed in by frame @@ -43,7 +43,7 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, db, frame, name string, cacheSize int) *View { +func NewView(path, db, frame, name string, cacheSize uint32) *View { return &View{ path: path, db: db, From 5cc4080ccfbd7985075bf3645bddb2336c14fab4 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 14:43:22 -0500 Subject: [PATCH 20/50] remove unused SetMessenger method --- gossip.go | 4 ---- messenger.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/gossip.go b/gossip.go index b25b634dd..9b18b45dd 100644 --- a/gossip.go +++ b/gossip.go @@ -138,10 +138,6 @@ func (g *GossipMessageBroker) Receive(pb proto.Message) error { return nil } -func (g *GossipMessageBroker) SetMessenger(m *Messenger) { - g.messenger = m -} - // implementation of the memberlist.Delegate interface func (g *GossipMessageBroker) NodeMeta(limit int) []byte { return []byte{} diff --git a/messenger.go b/messenger.go index 03e15ba0e..03579bee6 100644 --- a/messenger.go +++ b/messenger.go @@ -145,7 +145,6 @@ func (m *Messenger) mergeRemoteState(ns *internal.NodeState) error { type MessageBroker interface { Send(pb proto.Message, method string) error Receive(pb proto.Message) error - SetMessenger(m *Messenger) } ////////////////////////////////////////////////////////////////// @@ -167,7 +166,6 @@ func (c *nopMessageBroker) Receive(pb proto.Message) error { fmt.Println("NOPMessageBroker: Receive") return nil } -func (c *nopMessageBroker) SetMessenger(m *Messenger) {} ////////////////////////////////////////////////////////////////// @@ -217,8 +215,6 @@ func (h *HTTPMessageBroker) Receive(pb proto.Message) error { return nil } -func (h *HTTPMessageBroker) SetMessenger(m *Messenger) {} - func (h *HTTPMessageBroker) nodes() ([]*Node, error) { if h.messenger == nil { return nil, errors.New("HTTPMessageBroker has no reference to Messenger.") From 340cbc0cd05a5702f703724b581972846d5d5301 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 15:59:54 -0500 Subject: [PATCH 21/50] fix comment s/list/map --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index a3678ad00..e9c6bb2c1 100644 --- a/cluster.go +++ b/cluster.go @@ -121,7 +121,7 @@ func (c *Cluster) NodeSetHosts() []string { return a } -// Health returns a list of nodes in the cluster along with each node's state (UP/DOWN). +// Health returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value. func (c *Cluster) Health() map[string]string { h := make(map[string]string) for _, n := range c.Nodes { From 10a6cead17445dba35573a1c26d408ff67491cde Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 16:14:05 -0500 Subject: [PATCH 22/50] remove unused methods on StaticNodeSet these used to implement the NodeSet interface, but they've since been removed from NodeSet --- cluster.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cluster.go b/cluster.go index e9c6bb2c1..13f620b88 100644 --- a/cluster.go +++ b/cluster.go @@ -264,18 +264,6 @@ func (s *StaticNodeSet) Open() error { return nil } -func (s *StaticNodeSet) SetMessageHandler(f func(proto.Message) error) { - return -} - -func (s *StaticNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { - return -} - -func (s *StaticNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { - return -} - func (s *StaticNodeSet) SendMessage(pb proto.Message, method string) error { return nil } From 25f3ac92bddfdbecbc9a10a3f4b445ec96cfc81c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 16:55:40 -0500 Subject: [PATCH 23/50] move complex, implementation-specific config bits out of config.go moved into pilosa/server/server.go with minimal changes for now - had to tweak a few things to be able to get at unexported fields. Next step will be to move the implementations of NodeSet and MessageBroker out of the pilosa package so that pilosa core knows nothing about them. pilosa/server will deal with any complexities involved in setting them up. --- config.go | 79 +----------------------------------------------- gossip.go | 7 ++++- messenger.go | 4 +-- server/server.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 85 insertions(+), 84 deletions(-) diff --git a/config.go b/config.go index b74125ce7..150bd1ec9 100644 --- a/config.go +++ b/config.go @@ -1,10 +1,6 @@ package pilosa -import ( - "net" - "strconv" - "time" -) +import "time" const ( // DefaultHost is the default hostname and port to use. @@ -66,79 +62,6 @@ func NewConfigForHosts(hosts []string) *Config { return conf } -// PilosaMessenger returns a new instance of Messenger based on the config. -func (c *Config) PilosaMessenger() *Messenger { - messenger := NewMessenger() - switch c.Cluster.MessengerType { - case "broadcast": - n := NewHTTPMessageBroker() - n.messenger = messenger - messenger.Broker = n - case "gossip": - n := NewGossipMessageBroker() - n.messenger = messenger - messenger.Broker = n - case "static": - // nop - } - return messenger -} - -// PilosaCluster returns a new instance of Cluster based on the config. -func (c *Config) PilosaCluster() *Cluster { - cluster := NewCluster() - cluster.ReplicaN = c.Cluster.ReplicaN - - for _, hostport := range c.Cluster.Nodes { - cluster.Nodes = append(cluster.Nodes, &Node{Host: hostport}) - } - - // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - switch c.Cluster.MessengerType { - case "broadcast": - cluster.NodeSet = NewHTTPNodeSet() - cluster.NodeSet.(*HTTPNodeSet).Join(cluster.Nodes) - case "gossip": - gport, err := strconv.Atoi(DefaultGossipPort) - if err != nil { - // what? - } - gossipPort := gport - gossipSeed := DefaultHost - if c.Cluster.Gossip.Port != 0 { - gossipPort = c.Cluster.Gossip.Port - } - if c.Cluster.Gossip.Seed != "" { - gossipSeed = c.Cluster.Gossip.Seed - } - // get the host portion of addr to use for binding - gossipHost, _, err := net.SplitHostPort(c.Host) - if err != nil { - gossipHost = c.Host - } - cluster.NodeSet = NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) - case "static": - cluster.NodeSet = NewStaticNodeSet() - default: - cluster.NodeSet = NewStaticNodeSet() - } - - return cluster -} - -// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker -// after cluster configuration. -func (c *Config) AssociateMessageBroker(s *Server) { - switch c.Cluster.MessengerType { - case "broadcast": - // nop - case "gossip": - s.Cluster.NodeSet.(*GossipNodeSet).config.memberlistConfig.Delegate = s.Messenger.Broker.(*GossipMessageBroker) - case "static": - // nop - } -} - // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/gossip.go b/gossip.go index 9b18b45dd..178fa999e 100644 --- a/gossip.go +++ b/gossip.go @@ -23,6 +23,10 @@ type GossipNodeSet struct { LogOutput io.Writer } +func (g *GossipNodeSet) AttachBroker(mb *GossipMessageBroker) { + g.config.memberlistConfig.Delegate = mb +} + func (g *GossipNodeSet) Nodes() []*Node { a := make([]*Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { @@ -196,9 +200,10 @@ func (g *GossipMessageBroker) logger() *log.Logger { //////////////////////////////////////////////////////////////// // NewGossipMessageBroker returns a new instance of GossipMessageBroker. -func NewGossipMessageBroker() *GossipMessageBroker { +func NewGossipMessageBroker(m *Messenger) *GossipMessageBroker { g := &GossipMessageBroker{ LogOutput: os.Stderr, + messenger: m, } g.broadcasts = &memberlist.TransmitLimitedQueue{ diff --git a/messenger.go b/messenger.go index 03579bee6..1ffbfa51e 100644 --- a/messenger.go +++ b/messenger.go @@ -175,8 +175,8 @@ type HTTPMessageBroker struct { } // NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. -func NewHTTPMessageBroker() *HTTPMessageBroker { - return &HTTPMessageBroker{} +func NewHTTPMessageBroker(m *Messenger) *HTTPMessageBroker { + return &HTTPMessageBroker{messenger: m} } // Send sends a protobuf message to all nodes simultaneously. diff --git a/server/server.go b/server/server.go index ba41f3dd0..11bb055a9 100644 --- a/server/server.go +++ b/server/server.go @@ -9,8 +9,10 @@ import ( "fmt" "io" "math/rand" + "net" "os" "path/filepath" + "strconv" "strings" "time" @@ -92,11 +94,11 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.Messenger = m.Config.PilosaMessenger() - m.Server.Cluster = m.Config.PilosaCluster() + m.Server.Messenger = PilosaMessenger(m.Config) + m.Server.Cluster = PilosaCluster(m.Config) // Associate objects to the MessageBroker based on config. - m.Config.AssociateMessageBroker(m.Server) + AssociateMessageBroker(m.Server, m.Config) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) @@ -109,6 +111,77 @@ func (m *Command) Run(args ...string) (err error) { return nil } +// PilosaMessenger returns a new instance of Messenger based on the config. +func PilosaMessenger(c *pilosa.Config) *pilosa.Messenger { + messenger := pilosa.NewMessenger() + switch c.Cluster.MessengerType { + case "broadcast": + n := pilosa.NewHTTPMessageBroker(messenger) + messenger.Broker = n + case "gossip": + n := pilosa.NewGossipMessageBroker(messenger) + messenger.Broker = n + case "static": + // nop + } + return messenger +} + +// PilosaCluster returns a new instance of Cluster based on the config. +func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { + cluster := pilosa.NewCluster() + cluster.ReplicaN = c.Cluster.ReplicaN + + for _, hostport := range c.Cluster.Nodes { + cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) + } + + // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. + switch c.Cluster.MessengerType { + case "broadcast": + cluster.NodeSet = pilosa.NewHTTPNodeSet() + cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(cluster.Nodes) + case "gossip": + gport, err := strconv.Atoi(pilosa.DefaultGossipPort) + if err != nil { + // what? + } + gossipPort := gport + gossipSeed := pilosa.DefaultHost + if c.Cluster.Gossip.Port != 0 { + gossipPort = c.Cluster.Gossip.Port + } + if c.Cluster.Gossip.Seed != "" { + gossipSeed = c.Cluster.Gossip.Seed + } + // get the host portion of addr to use for binding + gossipHost, _, err := net.SplitHostPort(c.Host) + if err != nil { + gossipHost = c.Host + } + cluster.NodeSet = pilosa.NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) + case "static": + cluster.NodeSet = pilosa.NewStaticNodeSet() + default: + cluster.NodeSet = pilosa.NewStaticNodeSet() + } + + return cluster +} + +// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker +// after cluster configuration. +func AssociateMessageBroker(s *pilosa.Server, c *pilosa.Config) { + switch c.Cluster.MessengerType { + case "broadcast": + // nop + case "gossip": + s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.Messenger.Broker.(*pilosa.GossipMessageBroker)) + case "static": + // nop + } +} + func normalizeHost(host string) (string, error) { if !strings.Contains(host, ":") { host = host + ":" From 1aca2ce1de6c10045301e41102110a0aa3efcdb7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 11:06:58 -0500 Subject: [PATCH 24/50] implement LocalState and internal message reception on Server greatly simplifies Messenger, to the point of making it basically shell around MessageBroker. Next stesp are to make the receiving of internal messages more well-defined and behind an interface, remove/merge Messenger and MessageBroker, and have the implementations of MessageBroker in separate packages. --- gossip.go | 16 +++--- handler.go | 3 +- messenger.go | 126 +++-------------------------------------------- server.go | 90 +++++++++++++++++++++++++++++++-- server/server.go | 10 ++-- 5 files changed, 108 insertions(+), 137 deletions(-) diff --git a/gossip.go b/gossip.go index 178fa999e..59abef70d 100644 --- a/gossip.go +++ b/gossip.go @@ -90,7 +90,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed type GossipMessageBroker struct { broadcasts *memberlist.TransmitLimitedQueue - messenger *Messenger + server *Server // The writer for any logging. LogOutput io.Writer @@ -103,7 +103,7 @@ func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { return err } - mlist := g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist + mlist := g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist // Direct sends the message directly to every node. // An error from any node raises an error on the entire operation. @@ -136,7 +136,7 @@ func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { } func (g *GossipMessageBroker) Receive(pb proto.Message) error { - if err := g.messenger.ReceiveMessage(pb); err != nil { + if err := g.server.ReceiveMessage(pb); err != nil { return err } return nil @@ -164,7 +164,7 @@ func (g *GossipMessageBroker) GetBroadcasts(overhead, limit int) [][]byte { } func (g *GossipMessageBroker) LocalState(join bool) []byte { - pb, err := g.messenger.LocalState() + pb, err := g.server.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) return []byte{} @@ -186,7 +186,7 @@ func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { g.logger().Printf("error unmarshalling nodestate data, err=%s", err) return } - err := g.messenger.HandleRemoteState(&pb) + err := g.server.HandleRemoteState(&pb) if err != nil { g.logger().Printf("merge state error: %s", err) } @@ -200,15 +200,15 @@ func (g *GossipMessageBroker) logger() *log.Logger { //////////////////////////////////////////////////////////////// // NewGossipMessageBroker returns a new instance of GossipMessageBroker. -func NewGossipMessageBroker(m *Messenger) *GossipMessageBroker { +func NewGossipMessageBroker(s *Server) *GossipMessageBroker { g := &GossipMessageBroker{ LogOutput: os.Stderr, - messenger: m, + server: s, } g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { - return g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() + return g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() }, RetransmitMult: 3, } diff --git a/handler.go b/handler.go index 0a7cb278d..7616fd2d8 100644 --- a/handler.go +++ b/handler.go @@ -27,6 +27,7 @@ import ( type Handler struct { Index *Index Messenger *Messenger + Server *Server // Local hostname & cluster configuration. Host string @@ -213,7 +214,7 @@ func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { return } - if err := h.Messenger.Broker.Receive(m); err != nil { + if err := h.Server.ReceiveMessage(m); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/messenger.go b/messenger.go index 1ffbfa51e..3672a407a 100644 --- a/messenger.go +++ b/messenger.go @@ -20,15 +20,9 @@ import ( // Messenger represents an internal message handler. type Messenger struct { - // Broker handles Send/Receive Messages. + // Broker handles Send Broker MessageBroker - Index *Index - - // Local hostname & cluster configuration. - Host string - Cluster *Cluster - // The writer for any logging. LogOutput io.Writer } @@ -47,104 +41,12 @@ func (m *Messenger) SendMessage(pb proto.Message, method string) error { } return m.Broker.Send(pb, method) } -func (m *Messenger) ReceiveMessage(pb proto.Message) error { - return m.handleMessage(pb) -} - -// handleMessage handles protobuf Messages sent to nodes in the cluster. -func (m *Messenger) handleMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateSliceMessage: - d := m.Index.DB(obj.DB) - if d == nil { - return fmt.Errorf("Local DB not found: %s", obj.DB) - } - d.SetRemoteMaxSlice(obj.Slice) - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} - _, err := m.Index.CreateDB(obj.DB, opt) - if err != nil { - return err - } - case *internal.DeleteDBMessage: - fmt.Println("DELETE:", obj.DB) - if err := m.Index.DeleteDB(obj.DB); err != nil { - return err - } - case *internal.CreateFrameMessage: - db := m.Index.DB(obj.DB) - opt := FrameOptions{RowLabel: obj.Meta.RowLabel} - _, err := db.CreateFrame(obj.Frame, opt) - if err != nil { - return err - } - case *internal.DeleteFrameMessage: - db := m.Index.DB(obj.DB) - if err := db.DeleteFrame(obj.Frame); err != nil { - return err - } - } - return nil -} - -// LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -// It seems odd to have this as part of Messenger, but with the -// exception of Server, it's currenntly the only object with access -// to the necessary information (Host, Index, Cluster). -func (m *Messenger) LocalState() (proto.Message, error) { - if m.Index == nil { - return nil, errors.New("Messenger.Index is nil.") - } - return &internal.NodeState{ - Host: m.Host, - State: "OK", // TODO: make this work, pull from m.Cluster.Node - DBs: encodeDBs(m.Index.DBs()), - }, nil -} - -// HandleRemoteState receives incoming NodeState from remote nodes. -func (m *Messenger) HandleRemoteState(pb proto.Message) error { - return m.mergeRemoteState(pb.(*internal.NodeState)) -} - -func (m *Messenger) mergeRemoteState(ns *internal.NodeState) error { - // TODO: update some node state value in the cluster (it should be in cluster.node i guess) - - // Create databases that don't exist. - for _, db := range ns.DBs { - opt := DBOptions{ - ColumnLabel: db.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), - } - d, err := m.Index.CreateDBIfNotExists(db.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range db.Frames { - opt := FrameOptions{ - RowLabel: f.Meta.RowLabel, - TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), - CacheSize: f.Meta.CacheSize, - } - _, err := d.CreateFrameIfNotExists(f.Name, opt) - if err != nil { - return err - } - } - } - - return nil -} ////////////////////////////////////////////////////////////////// // MessageBroker is an interface for handling incoming/outgoing messages. type MessageBroker interface { Send(pb proto.Message, method string) error - Receive(pb proto.Message) error } ////////////////////////////////////////////////////////////////// @@ -162,21 +64,17 @@ func (c *nopMessageBroker) Send(pb proto.Message, method string) error { fmt.Println("NOPMessageBroker: Send") return nil } -func (c *nopMessageBroker) Receive(pb proto.Message) error { - fmt.Println("NOPMessageBroker: Receive") - return nil -} ////////////////////////////////////////////////////////////////// // HTTPMessageBroker represents a NodeSet that broadcasts messages over HTTP. type HTTPMessageBroker struct { - messenger *Messenger + server *Server } // NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. -func NewHTTPMessageBroker(m *Messenger) *HTTPMessageBroker { - return &HTTPMessageBroker{messenger: m} +func NewHTTPMessageBroker(s *Server) *HTTPMessageBroker { + return &HTTPMessageBroker{server: s} } // Send sends a protobuf message to all nodes simultaneously. @@ -196,7 +94,7 @@ func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { var g errgroup.Group for _, n := range nodes { // Don't send the message to the local node. - if n.Host == h.messenger.Host { + if n.Host == h.server.Host { continue } node := n @@ -207,19 +105,11 @@ func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { return g.Wait() } -// Receive is called when a node receives a message. -func (h *HTTPMessageBroker) Receive(pb proto.Message) error { - if err := h.messenger.ReceiveMessage(pb); err != nil { - return err - } - return nil -} - func (h *HTTPMessageBroker) nodes() ([]*Node, error) { - if h.messenger == nil { - return nil, errors.New("HTTPMessageBroker has no reference to Messenger.") + if h.server == nil { + return nil, errors.New("HTTPMessageBroker has no reference to Server.") } - nodeset, ok := h.messenger.Cluster.NodeSet.(*HTTPNodeSet) + nodeset, ok := h.server.Cluster.NodeSet.(*HTTPNodeSet) if !ok { return nil, errors.New("NodeSet cannot be caste to HTTPNodeSet.") } diff --git a/server.go b/server.go index a93d29f57..7dd35fa8f 100644 --- a/server.go +++ b/server.go @@ -1,6 +1,7 @@ package pilosa import ( + "errors" "fmt" "io" "io/ioutil" @@ -64,7 +65,7 @@ func NewServer() *Server { } s.Handler.Index = s.Index - s.Messenger.Index = s.Index + s.Handler.Server = s // TODO remove return s } @@ -111,9 +112,6 @@ func (s *Server) Open() error { e.Cluster = s.Cluster // Initialize Messenger. - s.Messenger.Index = s.Index - s.Messenger.Host = s.Host - s.Messenger.Cluster = s.Cluster s.Messenger.LogOutput = s.LogOutput // Initialize HTTP handler. @@ -236,6 +234,90 @@ func (s *Server) monitorMaxSlices() { } } +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +func (s *Server) LocalState() (proto.Message, error) { + if s.Index == nil { + return nil, errors.New("Messenger.Index is nil.") + } + return &internal.NodeState{ + Host: s.Host, + State: "OK", // TODO: make this work, pull from s.Cluster.Node + DBs: encodeDBs(s.Index.DBs()), + }, nil +} + +func (s *Server) ReceiveMessage(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.CreateSliceMessage: + d := s.Index.DB(obj.DB) + if d == nil { + return fmt.Errorf("Local DB not found: %s", obj.DB) + } + d.SetRemoteMaxSlice(obj.Slice) + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := s.Index.CreateDB(obj.DB, opt) + if err != nil { + return err + } + case *internal.DeleteDBMessage: + fmt.Println("DELETE:", obj.DB) + if err := s.Index.DeleteDB(obj.DB); err != nil { + return err + } + case *internal.CreateFrameMessage: + db := s.Index.DB(obj.DB) + opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + _, err := db.CreateFrame(obj.Frame, opt) + if err != nil { + return err + } + case *internal.DeleteFrameMessage: + db := s.Index.DB(obj.DB) + if err := db.DeleteFrame(obj.Frame); err != nil { + return err + } + } + return nil +} + +// HandleRemoteState receives incoming NodeState from remote nodes. +func (s *Server) HandleRemoteState(pb proto.Message) error { + return s.mergeRemoteState(pb.(*internal.NodeState)) +} + +func (s *Server) mergeRemoteState(ns *internal.NodeState) error { + // TODO: update some node state value in the cluster (it should be in cluster.node i guess) + + // Create databases that don't exist. + for _, db := range ns.DBs { + opt := DBOptions{ + ColumnLabel: db.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + } + d, err := s.Index.CreateDBIfNotExists(db.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range db.Frames { + opt := FrameOptions{ + RowLabel: f.Meta.RowLabel, + TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), + CacheSize: f.Meta.CacheSize, + } + _, err := d.CreateFrameIfNotExists(f.Name, opt) + if err != nil { + return err + } + } + } + + return nil +} + func checkMaxSlices(hostport string) (map[string]uint64, error) { // Create HTTP request. req, err := http.NewRequest("GET", (&url.URL{ diff --git a/server/server.go b/server/server.go index 11bb055a9..0b61bf405 100644 --- a/server/server.go +++ b/server/server.go @@ -94,7 +94,7 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.Messenger = PilosaMessenger(m.Config) + m.Server.Messenger = PilosaMessenger(m.Config, m.Server) m.Server.Cluster = PilosaCluster(m.Config) // Associate objects to the MessageBroker based on config. @@ -112,15 +112,13 @@ func (m *Command) Run(args ...string) (err error) { } // PilosaMessenger returns a new instance of Messenger based on the config. -func PilosaMessenger(c *pilosa.Config) *pilosa.Messenger { +func PilosaMessenger(c *pilosa.Config, server *pilosa.Server) *pilosa.Messenger { messenger := pilosa.NewMessenger() switch c.Cluster.MessengerType { case "broadcast": - n := pilosa.NewHTTPMessageBroker(messenger) - messenger.Broker = n + messenger.Broker = pilosa.NewHTTPMessageBroker(server) case "gossip": - n := pilosa.NewGossipMessageBroker(messenger) - messenger.Broker = n + messenger.Broker = pilosa.NewGossipMessageBroker(server) case "static": // nop } From 5079ee7a2286766aa9ff597baa2c2ef5bbc0fda5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 12:14:11 -0500 Subject: [PATCH 25/50] remove messenger "wrapper" around messagebroker all the functionality aside from the broker has been removed --- cluster.go | 6 +----- db.go | 4 ++-- frame.go | 2 +- gossip.go | 2 +- handler.go | 8 ++++---- handler_test.go | 5 ++--- index.go | 5 ++--- messenger.go | 33 +-------------------------------- server.go | 13 +++++-------- server/server.go | 17 ++++++++--------- 10 files changed, 27 insertions(+), 68 deletions(-) diff --git a/cluster.go b/cluster.go index 13f620b88..a435a20b7 100644 --- a/cluster.go +++ b/cluster.go @@ -248,7 +248,6 @@ func (h *HTTPNodeSet) Join(nodes []*Node) error { // StaticNodeSet represents a basic NodeSet for testing type StaticNodeSet struct { - Messenger nodes []*Node } @@ -264,9 +263,6 @@ func (s *StaticNodeSet) Open() error { return nil } -func (s *StaticNodeSet) SendMessage(pb proto.Message, method string) error { - return nil -} -func (s *StaticNodeSet) ReceiveMessage(pb proto.Message) error { +func (s *StaticNodeSet) Send(pb proto.Message, method string) error { return nil } diff --git a/db.go b/db.go index cc27e18c9..4eb61707e 100644 --- a/db.go +++ b/db.go @@ -43,7 +43,7 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - messenger *Messenger + msgbroker MessageBroker stats StatsClient LogOutput io.Writer @@ -417,7 +417,7 @@ func (db *DB) newFrame(path, name string) (*Frame, error) { } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) - f.messenger = db.messenger + f.msgbroker = db.msgbroker return f, nil } diff --git a/frame.go b/frame.go index c5a40b1d1..1da970efc 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,7 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - messenger *Messenger + msgbroker MessageBroker stats StatsClient // Frame settings. diff --git a/gossip.go b/gossip.go index 59abef70d..37b8c3b97 100644 --- a/gossip.go +++ b/gossip.go @@ -96,7 +96,7 @@ type GossipMessageBroker struct { LogOutput io.Writer } -// implementation of the messenger.Messenger interface +// implementation of the messenger.MessageBroker interface func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { msg, err := MarshalMessage(pb) if err != nil { diff --git a/handler.go b/handler.go index 7616fd2d8..5c015abde 100644 --- a/handler.go +++ b/handler.go @@ -26,7 +26,7 @@ import ( // Handler represents an HTTP handler. type Handler struct { Index *Index - Messenger *Messenger + MsgBroker MessageBroker Server *Server // Local hostname & cluster configuration. @@ -370,7 +370,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.Messenger.SendMessage( + err := h.MsgBroker.Send( &internal.DeleteDBMessage{ DB: req.DB, }, "direct") @@ -514,7 +514,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Send the create message to all nodes. - err = h.Messenger.SendMessage( + err = h.MsgBroker.Send( &internal.CreateFrameMessage{ DB: req.DB, Frame: req.Frame, @@ -586,7 +586,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.Messenger.SendMessage( + err := h.MsgBroker.Send( &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, diff --git a/handler_test.go b/handler_test.go index 92bcc7601..e95e5c0dd 100644 --- a/handler_test.go +++ b/handler_test.go @@ -794,7 +794,7 @@ func NewHandler() *Handler { h.Handler.LogOutput = ioutil.Discard // Handler test messages can no-op. - h.Messenger = pilosa.NewMessenger() + h.MsgBroker = pilosa.NopMessageBroker return h } @@ -828,8 +828,7 @@ func NewServer() *Server { s.Handler.Host = s.Host() // Handler test messages can no-op. - s.Handler.Messenger = pilosa.NewMessenger() - + s.Handler.MsgBroker = pilosa.NopMessageBroker // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() diff --git a/index.go b/index.go index 5e9f5187e..ab659ada1 100644 --- a/index.go +++ b/index.go @@ -23,8 +23,7 @@ type Index struct { // Databases by name. dbs map[string]*DB - Messenger *Messenger - + MsgBroker MessageBroker // Close management wg sync.WaitGroup closing chan struct{} @@ -247,7 +246,7 @@ func (i *Index) newDB(path, name string) (*DB, error) { } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - db.messenger = i.Messenger + db.msgbroker = i.MsgBroker return db, nil } diff --git a/messenger.go b/messenger.go index 3672a407a..fc0c55d77 100644 --- a/messenger.go +++ b/messenger.go @@ -4,11 +4,9 @@ import ( "bytes" "errors" "fmt" - "io" "io/ioutil" "net/http" "net/url" - "os" "reflect" "golang.org/x/sync/errgroup" @@ -17,40 +15,11 @@ import ( "github.com/pilosa/pilosa/internal" ) -// Messenger represents an internal message handler. -type Messenger struct { - - // Broker handles Send - Broker MessageBroker - - // The writer for any logging. - LogOutput io.Writer -} - -// NewMessenger returns a new instance of Messenger with a default logger. -func NewMessenger() *Messenger { - return &Messenger{ - Broker: NopMessageBroker, - LogOutput: os.Stderr, - } -} - -func (m *Messenger) SendMessage(pb proto.Message, method string) error { - if m.Broker == nil { - return errors.New("Messenger.Broker is not defined.") - } - return m.Broker.Send(pb, method) -} - -////////////////////////////////////////////////////////////////// - // MessageBroker is an interface for handling incoming/outgoing messages. type MessageBroker interface { Send(pb proto.Message, method string) error } -////////////////////////////////////////////////////////////////// - func init() { NopMessageBroker = &nopMessageBroker{} } @@ -61,7 +30,7 @@ var NopMessageBroker MessageBroker type nopMessageBroker struct{} func (c *nopMessageBroker) Send(pb proto.Message, method string) error { - fmt.Println("NOPMessageBroker: Send") + fmt.Println("NOPMessageBroker: Send") // TODO remove or log properly? return nil } diff --git a/server.go b/server.go index 7dd35fa8f..68f9a6921 100644 --- a/server.go +++ b/server.go @@ -35,7 +35,7 @@ type Server struct { // Data storage and HTTP interface. Index *Index Handler *Handler - Messenger *Messenger + MsgBroker MessageBroker // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -56,7 +56,7 @@ func NewServer() *Server { Index: NewIndex(), Handler: NewHandler(), - Messenger: NewMessenger(), + MsgBroker: NopMessageBroker, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -111,18 +111,15 @@ func (s *Server) Open() error { e.Host = s.Host e.Cluster = s.Cluster - // Initialize Messenger. - s.Messenger.LogOutput = s.LogOutput - // Initialize HTTP handler. - s.Handler.Messenger = s.Messenger + s.Handler.MsgBroker = s.MsgBroker s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput // Initialize Index. - s.Index.Messenger = s.Messenger + s.Index.MsgBroker = s.MsgBroker s.Index.LogOutput = s.LogOutput // Serve HTTP. @@ -239,7 +236,7 @@ func (s *Server) monitorMaxSlices() { // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalState() (proto.Message, error) { if s.Index == nil { - return nil, errors.New("Messenger.Index is nil.") + return nil, errors.New("Server.Index is nil.") } return &internal.NodeState{ Host: s.Host, diff --git a/server/server.go b/server/server.go index 0b61bf405..69041ef3a 100644 --- a/server/server.go +++ b/server/server.go @@ -94,7 +94,7 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.Messenger = PilosaMessenger(m.Config, m.Server) + m.Server.MsgBroker = PilosaMessageBroker(m.Config, m.Server) m.Server.Cluster = PilosaCluster(m.Config) // Associate objects to the MessageBroker based on config. @@ -111,18 +111,17 @@ func (m *Command) Run(args ...string) (err error) { return nil } -// PilosaMessenger returns a new instance of Messenger based on the config. -func PilosaMessenger(c *pilosa.Config, server *pilosa.Server) *pilosa.Messenger { - messenger := pilosa.NewMessenger() +// PilosaMessageBroker returns a new instance of MessageBroker based on the config. +func PilosaMessageBroker(c *pilosa.Config, server *pilosa.Server) (broker pilosa.MessageBroker) { switch c.Cluster.MessengerType { case "broadcast": - messenger.Broker = pilosa.NewHTTPMessageBroker(server) + broker = pilosa.NewHTTPMessageBroker(server) case "gossip": - messenger.Broker = pilosa.NewGossipMessageBroker(server) + broker = pilosa.NewGossipMessageBroker(server) case "static": - // nop + broker = pilosa.NopMessageBroker } - return messenger + return broker } // PilosaCluster returns a new instance of Cluster based on the config. @@ -174,7 +173,7 @@ func AssociateMessageBroker(s *pilosa.Server, c *pilosa.Config) { case "broadcast": // nop case "gossip": - s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.Messenger.Broker.(*pilosa.GossipMessageBroker)) + s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.MsgBroker.(*pilosa.GossipMessageBroker)) case "static": // nop } From 4b8c2630e3838039537cf9df5c7e47b4c8b911df Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 12:30:02 -0500 Subject: [PATCH 26/50] split MessageBroker.Send into SendSync and SendAsync --- cluster.go | 6 +++++- gossip.go | 46 +++++++++++++++++++++++++--------------------- handler.go | 12 ++++++------ messenger.go | 24 +++++++++++++++++++----- 4 files changed, 55 insertions(+), 33 deletions(-) diff --git a/cluster.go b/cluster.go index a435a20b7..ef67b7a4a 100644 --- a/cluster.go +++ b/cluster.go @@ -263,6 +263,10 @@ func (s *StaticNodeSet) Open() error { return nil } -func (s *StaticNodeSet) Send(pb proto.Message, method string) error { +func (s *StaticNodeSet) SendSync(pb proto.Message) error { + return nil +} + +func (s *StaticNodeSet) SendAsync(pb proto.Message) error { return nil } diff --git a/gossip.go b/gossip.go index 37b8c3b97..6482a333d 100644 --- a/gossip.go +++ b/gossip.go @@ -96,8 +96,8 @@ type GossipMessageBroker struct { LogOutput io.Writer } -// implementation of the messenger.MessageBroker interface -func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { +// SendSync implementation of the messenger.MessageBroker interface +func (g *GossipMessageBroker) SendSync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -110,28 +110,32 @@ func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { // // Gossip uses the gossip protocol to eventually deliver the message // to every node. - switch method { - case "direct": - var eg errgroup.Group - for _, n := range mlist.Members() { - // Don't send the message to the local node. - if n == mlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return mlist.SendToTCP(node, msg) - }) + var eg errgroup.Group + for _, n := range mlist.Members() { + // Don't send the message to the local node. + if n == mlist.LocalNode() { + continue } - return eg.Wait() - case "gossip": - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) + node := n + eg.Go(func() error { + return mlist.SendToTCP(node, msg) + }) + } + return eg.Wait() +} + +// SendAsync implementation of the messenger.MessageBroker interface +func (g *GossipMessageBroker) SendAsync(pb proto.Message) error { + msg, err := MarshalMessage(pb) + if err != nil { + return err } + b := &broadcast{ + msg: msg, + notify: nil, + } + g.broadcasts.QueueBroadcast(b) return nil } diff --git a/handler.go b/handler.go index 5c015abde..ed1418f9a 100644 --- a/handler.go +++ b/handler.go @@ -370,10 +370,10 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.Send( + err := h.MsgBroker.SendSync( &internal.DeleteDBMessage{ DB: req.DB, - }, "direct") + }) if err != nil { h.logger().Printf("problem sending DeleteDB message: %s", err) } @@ -514,7 +514,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Send the create message to all nodes. - err = h.MsgBroker.Send( + err = h.MsgBroker.SendSync( &internal.CreateFrameMessage{ DB: req.DB, Frame: req.Frame, @@ -522,7 +522,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { RowLabel: req.Options.RowLabel, TimeQuantum: string(req.Options.TimeQuantum), }, - }, "direct") + }) if err != nil { h.logger().Printf("problem sending CreateFrame message: %s", err) } @@ -586,11 +586,11 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.Send( + err := h.MsgBroker.SendSync( &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, - }, "direct") + }) if err != nil { h.logger().Printf("problem sending DeleteFrame message: %s", err) } diff --git a/messenger.go b/messenger.go index fc0c55d77..22ba8d61f 100644 --- a/messenger.go +++ b/messenger.go @@ -17,7 +17,8 @@ import ( // MessageBroker is an interface for handling incoming/outgoing messages. type MessageBroker interface { - Send(pb proto.Message, method string) error + SendSync(pb proto.Message) error + SendAsync(pb proto.Message) error } func init() { @@ -29,8 +30,15 @@ var NopMessageBroker MessageBroker // nopMessageBroker represents a MessageBroker that doesn't do anything. type nopMessageBroker struct{} -func (c *nopMessageBroker) Send(pb proto.Message, method string) error { - fmt.Println("NOPMessageBroker: Send") // TODO remove or log properly? +// SendSync A no-op implemenetation of MessageBroker SendSync method. +func (c *nopMessageBroker) SendSync(pb proto.Message) error { + fmt.Println("NOPMessageBroker: SendSync") // TODO remove or log properly? + return nil +} + +// SendAsync A no-op implemenetation of MessageBroker SendAsync method. +func (c *nopMessageBroker) SendAsync(pb proto.Message) error { + fmt.Println("NOPMessageBroker: SendAsync") // TODO remove or log properly? return nil } @@ -46,9 +54,9 @@ func NewHTTPMessageBroker(s *Server) *HTTPMessageBroker { return &HTTPMessageBroker{server: s} } -// Send sends a protobuf message to all nodes simultaneously. +// SendSync sends a protobuf message to all nodes simultaneously. // It waits for all nodes to respond before the function returns (and returns any errors). -func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { +func (h *HTTPMessageBroker) SendSync(pb proto.Message) error { // Marshal the pb to []byte buf, err := MarshalMessage(pb) if err != nil { @@ -74,6 +82,12 @@ func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { return g.Wait() } +// SendAsync exists to implement the MessageBroker interface, but just calls +// SendSync. +func (h *HTTPMessageBroker) SendAsync(pb proto.Message) error { + return h.SendSync(pb) +} + func (h *HTTPMessageBroker) nodes() ([]*Node, error) { if h.server == nil { return nil, errors.New("HTTPMessageBroker has no reference to Server.") From 32d07fc9e1c1fa06f02375e13311b8315c7e5921 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 12:47:26 -0500 Subject: [PATCH 27/50] rename MessageBroker -> Broadcaster --- cmd/server.go | 2 +- config.go | 12 ++++++------ db.go | 6 +++--- frame.go | 4 ++-- gossip.go | 38 +++++++++++++++++++------------------- handler.go | 12 ++++++------ handler_test.go | 4 ++-- index.go | 4 ++-- messenger.go | 46 +++++++++++++++++++++++----------------------- server.go | 16 ++++++++-------- server/server.go | 36 ++++++++++++++++++------------------ 11 files changed, 90 insertions(+), 90 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 27c238eb6..e9d7fcfdf 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -83,7 +83,7 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "static", "Type of Messenger to use for inter-host messaging. Choose from [static, broadcast, gossip]") + flags.StringVarP(&Server.Config.Cluster.BroadcasterType, "cluster.broadcaster-type", "", "static", "Type of Broadcaster to use for inter-host messaging. Choose from [static, http, gossip]") flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") diff --git a/config.go b/config.go index 150bd1ec9..8b718f4e1 100644 --- a/config.go +++ b/config.go @@ -4,10 +4,10 @@ import "time" const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultMessengerType = "static" - DefaultGossipPort = "14000" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultBroadcasterType = "static" + DefaultGossipPort = "14000" ) // Config represents the configuration for the command. @@ -17,7 +17,7 @@ type Config struct { Cluster struct { ReplicaN int `toml:"replicas"` - MessengerType string `toml:"messenger-type"` + BroadcasterType string `toml:"broadcaster-type"` Nodes []string `toml:"hosts"` PollingInterval Duration `toml:"polling-interval"` Gossip ConfigGossip `toml:"gossip"` @@ -45,7 +45,7 @@ func NewConfig() *Config { Host: DefaultHost + ":" + DefaultPort, } c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.MessengerType = DefaultMessengerType + c.Cluster.BroadcasterType = DefaultBroadcasterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) diff --git a/db.go b/db.go index 4eb61707e..d2ceaf13a 100644 --- a/db.go +++ b/db.go @@ -43,8 +43,8 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - msgbroker MessageBroker - stats StatsClient + broadcaster Broadcaster + stats StatsClient LogOutput io.Writer } @@ -417,7 +417,7 @@ func (db *DB) newFrame(path, name string) (*Frame, error) { } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) - f.msgbroker = db.msgbroker + f.broadcaster = db.broadcaster return f, nil } diff --git a/frame.go b/frame.go index 1da970efc..a497e6906 100644 --- a/frame.go +++ b/frame.go @@ -38,8 +38,8 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - msgbroker MessageBroker - stats StatsClient + broadcaster Broadcaster + stats StatsClient // Frame settings. rowLabel string diff --git a/gossip.go b/gossip.go index 6482a333d..da9c847f5 100644 --- a/gossip.go +++ b/gossip.go @@ -23,7 +23,7 @@ type GossipNodeSet struct { LogOutput io.Writer } -func (g *GossipNodeSet) AttachBroker(mb *GossipMessageBroker) { +func (g *GossipNodeSet) AttachBroadcaster(mb *GossipBroadcaster) { g.config.memberlistConfig.Delegate = mb } @@ -85,9 +85,9 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed //////////////////////////////////////////////////////////////// -// GossipMessageBroker represents a gossip implementation of pilosa.MessageBroker -// GossipMessageBroker also represents an implementation of memberlist.Delegate -type GossipMessageBroker struct { +// GossipBroadcaster represents a gossip implementation of pilosa.Broadcaster +// GossipBroadcaster also represents an implementation of memberlist.Delegate +type GossipBroadcaster struct { broadcasts *memberlist.TransmitLimitedQueue server *Server @@ -96,8 +96,8 @@ type GossipMessageBroker struct { LogOutput io.Writer } -// SendSync implementation of the messenger.MessageBroker interface -func (g *GossipMessageBroker) SendSync(pb proto.Message) error { +// SendSync implementation of the Broadcaster interface +func (g *GossipBroadcaster) SendSync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -124,8 +124,8 @@ func (g *GossipMessageBroker) SendSync(pb proto.Message) error { return eg.Wait() } -// SendAsync implementation of the messenger.MessageBroker interface -func (g *GossipMessageBroker) SendAsync(pb proto.Message) error { +// SendAsync implementation of the Broadcaster interface +func (g *GossipBroadcaster) SendAsync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -139,7 +139,7 @@ func (g *GossipMessageBroker) SendAsync(pb proto.Message) error { return nil } -func (g *GossipMessageBroker) Receive(pb proto.Message) error { +func (g *GossipBroadcaster) Receive(pb proto.Message) error { if err := g.server.ReceiveMessage(pb); err != nil { return err } @@ -147,11 +147,11 @@ func (g *GossipMessageBroker) Receive(pb proto.Message) error { } // implementation of the memberlist.Delegate interface -func (g *GossipMessageBroker) NodeMeta(limit int) []byte { +func (g *GossipBroadcaster) NodeMeta(limit int) []byte { return []byte{} } -func (g *GossipMessageBroker) NotifyMsg(b []byte) { +func (g *GossipBroadcaster) NotifyMsg(b []byte) { m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -163,11 +163,11 @@ func (g *GossipMessageBroker) NotifyMsg(b []byte) { } } -func (g *GossipMessageBroker) GetBroadcasts(overhead, limit int) [][]byte { +func (g *GossipBroadcaster) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } -func (g *GossipMessageBroker) LocalState(join bool) []byte { +func (g *GossipBroadcaster) LocalState(join bool) []byte { pb, err := g.server.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) @@ -183,7 +183,7 @@ func (g *GossipMessageBroker) LocalState(join bool) []byte { return buf } -func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { +func (g *GossipBroadcaster) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeState if err := proto.Unmarshal(buf, &pb); err != nil { @@ -196,16 +196,16 @@ func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { } } -// logger returns a logger for the GossipMessageBroker. -func (g *GossipMessageBroker) logger() *log.Logger { +// logger returns a logger for the GossipBroadcaster +func (g *GossipBroadcaster) logger() *log.Logger { return log.New(g.LogOutput, "", log.LstdFlags) } //////////////////////////////////////////////////////////////// -// NewGossipMessageBroker returns a new instance of GossipMessageBroker. -func NewGossipMessageBroker(s *Server) *GossipMessageBroker { - g := &GossipMessageBroker{ +// NewGossipBroadcaster returns a new instance of GossipBroadcaster. +func NewGossipBroadcaster(s *Server) *GossipBroadcaster { + g := &GossipBroadcaster{ LogOutput: os.Stderr, server: s, } diff --git a/handler.go b/handler.go index ed1418f9a..83f969d22 100644 --- a/handler.go +++ b/handler.go @@ -25,9 +25,9 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Index *Index - MsgBroker MessageBroker - Server *Server + Index *Index + Broadcaster Broadcaster + Server *Server // Local hostname & cluster configuration. Host string @@ -370,7 +370,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.SendSync( + err := h.Broadcaster.SendSync( &internal.DeleteDBMessage{ DB: req.DB, }) @@ -514,7 +514,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Send the create message to all nodes. - err = h.MsgBroker.SendSync( + err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ DB: req.DB, Frame: req.Frame, @@ -586,7 +586,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.SendSync( + err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, diff --git a/handler_test.go b/handler_test.go index e95e5c0dd..51f48fef7 100644 --- a/handler_test.go +++ b/handler_test.go @@ -794,7 +794,7 @@ func NewHandler() *Handler { h.Handler.LogOutput = ioutil.Discard // Handler test messages can no-op. - h.MsgBroker = pilosa.NopMessageBroker + h.Broadcaster = pilosa.NopBroadcaster return h } @@ -828,7 +828,7 @@ func NewServer() *Server { s.Handler.Host = s.Host() // Handler test messages can no-op. - s.Handler.MsgBroker = pilosa.NopMessageBroker + s.Handler.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() diff --git a/index.go b/index.go index ab659ada1..eb8f4b204 100644 --- a/index.go +++ b/index.go @@ -23,7 +23,7 @@ type Index struct { // Databases by name. dbs map[string]*DB - MsgBroker MessageBroker + Broadcaster Broadcaster // Close management wg sync.WaitGroup closing chan struct{} @@ -246,7 +246,7 @@ func (i *Index) newDB(path, name string) (*DB, error) { } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - db.msgbroker = i.MsgBroker + db.broadcaster = i.Broadcaster return db, nil } diff --git a/messenger.go b/messenger.go index 22ba8d61f..0549e35fc 100644 --- a/messenger.go +++ b/messenger.go @@ -15,48 +15,48 @@ import ( "github.com/pilosa/pilosa/internal" ) -// MessageBroker is an interface for handling incoming/outgoing messages. -type MessageBroker interface { +// Broadcaster is an interface for handling incoming/outgoing messages. +type Broadcaster interface { SendSync(pb proto.Message) error SendAsync(pb proto.Message) error } func init() { - NopMessageBroker = &nopMessageBroker{} + NopBroadcaster = &nopBroadcaster{} } -var NopMessageBroker MessageBroker +var NopBroadcaster Broadcaster -// nopMessageBroker represents a MessageBroker that doesn't do anything. -type nopMessageBroker struct{} +// nopBroadcaster represents a Broadcaster that doesn't do anything. +type nopBroadcaster struct{} -// SendSync A no-op implemenetation of MessageBroker SendSync method. -func (c *nopMessageBroker) SendSync(pb proto.Message) error { - fmt.Println("NOPMessageBroker: SendSync") // TODO remove or log properly? +// SendSync A no-op implemenetation of Broadcaster SendSync method. +func (c *nopBroadcaster) SendSync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? return nil } -// SendAsync A no-op implemenetation of MessageBroker SendAsync method. -func (c *nopMessageBroker) SendAsync(pb proto.Message) error { - fmt.Println("NOPMessageBroker: SendAsync") // TODO remove or log properly? +// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +func (c *nopBroadcaster) SendAsync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? return nil } ////////////////////////////////////////////////////////////////// -// HTTPMessageBroker represents a NodeSet that broadcasts messages over HTTP. -type HTTPMessageBroker struct { +// HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. +type HTTPBroadcaster struct { server *Server } -// NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. -func NewHTTPMessageBroker(s *Server) *HTTPMessageBroker { - return &HTTPMessageBroker{server: s} +// NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. +func NewHTTPBroadcaster(s *Server) *HTTPBroadcaster { + return &HTTPBroadcaster{server: s} } // SendSync sends a protobuf message to all nodes simultaneously. // It waits for all nodes to respond before the function returns (and returns any errors). -func (h *HTTPMessageBroker) SendSync(pb proto.Message) error { +func (h *HTTPBroadcaster) SendSync(pb proto.Message) error { // Marshal the pb to []byte buf, err := MarshalMessage(pb) if err != nil { @@ -82,15 +82,15 @@ func (h *HTTPMessageBroker) SendSync(pb proto.Message) error { return g.Wait() } -// SendAsync exists to implement the MessageBroker interface, but just calls +// SendAsync exists to implement the Broadcaster interface, but just calls // SendSync. -func (h *HTTPMessageBroker) SendAsync(pb proto.Message) error { +func (h *HTTPBroadcaster) SendAsync(pb proto.Message) error { return h.SendSync(pb) } -func (h *HTTPMessageBroker) nodes() ([]*Node, error) { +func (h *HTTPBroadcaster) nodes() ([]*Node, error) { if h.server == nil { - return nil, errors.New("HTTPMessageBroker has no reference to Server.") + return nil, errors.New("HTTPBroadcaster has no reference to Server.") } nodeset, ok := h.server.Cluster.NodeSet.(*HTTPNodeSet) if !ok { @@ -99,7 +99,7 @@ func (h *HTTPMessageBroker) nodes() ([]*Node, error) { return nodeset.Nodes(), nil } -func (h *HTTPMessageBroker) sendNodeMessage(node *Node, msg []byte) error { +func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { var client *http.Client client = http.DefaultClient diff --git a/server.go b/server.go index 68f9a6921..2f0ace9ce 100644 --- a/server.go +++ b/server.go @@ -33,9 +33,9 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index - Handler *Handler - MsgBroker MessageBroker + Index *Index + Handler *Handler + Broadcaster Broadcaster // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -54,9 +54,9 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), - Handler: NewHandler(), - MsgBroker: NopMessageBroker, + Index: NewIndex(), + Handler: NewHandler(), + Broadcaster: NopBroadcaster, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -112,14 +112,14 @@ func (s *Server) Open() error { e.Cluster = s.Cluster // Initialize HTTP handler. - s.Handler.MsgBroker = s.MsgBroker + s.Handler.Broadcaster = s.Broadcaster s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput // Initialize Index. - s.Index.MsgBroker = s.MsgBroker + s.Index.Broadcaster = s.Broadcaster s.Index.LogOutput = s.LogOutput // Serve HTTP. diff --git a/server/server.go b/server/server.go index 69041ef3a..e880a1362 100644 --- a/server/server.go +++ b/server/server.go @@ -94,11 +94,11 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.MsgBroker = PilosaMessageBroker(m.Config, m.Server) + m.Server.Broadcaster = PilosaBroadcaster(m.Config, m.Server) m.Server.Cluster = PilosaCluster(m.Config) - // Associate objects to the MessageBroker based on config. - AssociateMessageBroker(m.Server, m.Config) + // Associate objects to the Broadcaster based on config. + AssociateBroadcaster(m.Server, m.Config) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) @@ -111,17 +111,17 @@ func (m *Command) Run(args ...string) (err error) { return nil } -// PilosaMessageBroker returns a new instance of MessageBroker based on the config. -func PilosaMessageBroker(c *pilosa.Config, server *pilosa.Server) (broker pilosa.MessageBroker) { - switch c.Cluster.MessengerType { - case "broadcast": - broker = pilosa.NewHTTPMessageBroker(server) +// PilosaBroadcaster returns a new instance of Broadcaster based on the config. +func PilosaBroadcaster(c *pilosa.Config, server *pilosa.Server) (broadcaster pilosa.Broadcaster) { + switch c.Cluster.BroadcasterType { + case "http": + broadcaster = pilosa.NewHTTPBroadcaster(server) case "gossip": - broker = pilosa.NewGossipMessageBroker(server) + broadcaster = pilosa.NewGossipBroadcaster(server) case "static": - broker = pilosa.NopMessageBroker + broadcaster = pilosa.NopBroadcaster } - return broker + return broadcaster } // PilosaCluster returns a new instance of Cluster based on the config. @@ -134,8 +134,8 @@ func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { } // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - switch c.Cluster.MessengerType { - case "broadcast": + switch c.Cluster.BroadcasterType { + case "http": cluster.NodeSet = pilosa.NewHTTPNodeSet() cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(cluster.Nodes) case "gossip": @@ -166,14 +166,14 @@ func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { return cluster } -// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker +// AssociateBroadcaster allows an implementation to associate objects to the Broadcaster // after cluster configuration. -func AssociateMessageBroker(s *pilosa.Server, c *pilosa.Config) { - switch c.Cluster.MessengerType { - case "broadcast": +func AssociateBroadcaster(s *pilosa.Server, c *pilosa.Config) { + switch c.Cluster.BroadcasterType { + case "http": // nop case "gossip": - s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.MsgBroker.(*pilosa.GossipMessageBroker)) + s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroadcaster(s.Broadcaster.(*pilosa.GossipBroadcaster)) case "static": // nop } From 4fc2494cc0ce6d55a4da5ec3a037777458429fc6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 13:51:27 -0500 Subject: [PATCH 28/50] add panic if Atoi on DefaultGossipPort fails --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index e880a1362..251bf77a5 100644 --- a/server/server.go +++ b/server/server.go @@ -141,7 +141,7 @@ func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { case "gossip": gport, err := strconv.Atoi(pilosa.DefaultGossipPort) if err != nil { - // what? + panic(err) // Atoi on a compile-time constant should never fail. } gossipPort := gport gossipSeed := pilosa.DefaultHost From bc8f7f702f10b89cfb37b3d2072b1e857614127a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 17:09:52 -0500 Subject: [PATCH 29/50] add BroadcastReceiver and collapse Gossip structs into one GossipNodeSet and GossipBroadcaster had a lot of cross dependency - I made them the same object and had it implement all three interfaces (NodeSet, Broadcaster, BroadcastReceiver). I implmented an HTTPBroadcastReceiver which runs as as separate server. BroadcastReceiver is a separate entity on the pilosa.Server object and is started separately from broadcaster and nodeset. In the case of GossipNodeSet, the broadcast receiver must be started before Open()ing the Nodeset, because it doesn't actually start listening until Open() is called, but it needs the handler set up before then. server/server.go was heavily refactored to configure the new structs and interfaces on the pilosa.Server object. --- broadcast.go | 83 +++++++++++++++++++++++++++++++++ gossip.go | 87 ++++++++++++++--------------------- handler.go | 30 ------------ messenger.go | 12 +++-- server.go | 18 +++++--- server/server.go | 117 ++++++++++++++++++++--------------------------- 6 files changed, 186 insertions(+), 161 deletions(-) create mode 100644 broadcast.go diff --git a/broadcast.go b/broadcast.go new file mode 100644 index 000000000..feaf7f55f --- /dev/null +++ b/broadcast.go @@ -0,0 +1,83 @@ +package pilosa + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/gogo/protobuf/proto" +) + +// BroadcastHandler is the interface for the pilosa object which knows how to +// handle broadcast messages. (Hint: this is implemented by pilosa.Server) +type BroadcastHandler interface { + ReceiveMessage(pb proto.Message) error +} + +// BroadcastReceiver is the interface for the object which will listen for and +// decode broadcast messages before passing them to pilosa to handle. The +// implementation of this could be an http server which listens for messages, +// gets the protobuf payload, and then passes it to +// BroadcastHandler.ReceiveMessage. +type BroadcastReceiver interface { + // Start starts listening for broadcast messages - it should return + // immediately, spawning a goroutine if necessary. + Start(BroadcastHandler) error +} + +type nopBroadcastReceiver struct{} + +func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil } + +var NopBroadcastReceiver = &nopBroadcastReceiver{} + +type HTTPBroadcastReceiver struct { + port string + handler BroadcastHandler + logOutput io.Writer +} + +func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { + return &HTTPBroadcastReceiver{ + port: port, + logOutput: logOutput, + } +} + +func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { + rec.handler = b + go func() { + err := http.ListenAndServe(":"+rec.port, rec) + if err != nil { + fmt.Fprintf(rec.logOutput, "Error listening on %v for HTTPBroadcastReceiver: %v\n", ":"+rec.port, err) + } + }() + return nil +} + +func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal message to specific proto type. + m, err := UnmarshalMessage(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := rec.handler.ReceiveMessage(m); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} diff --git a/gossip.go b/gossip.go index da9c847f5..1224cf69b 100644 --- a/gossip.go +++ b/gossip.go @@ -1,6 +1,7 @@ package pilosa import ( + "fmt" "io" "log" "os" @@ -13,9 +14,15 @@ import ( ) // GossipNodeSet represents a gossip implementation of NodeSet using memberlist +// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { memberlist *memberlist.Memberlist + handler BroadcastHandler + + broadcasts *memberlist.TransmitLimitedQueue + + server *Server config *GossipConfig @@ -23,10 +30,6 @@ type GossipNodeSet struct { LogOutput io.Writer } -func (g *GossipNodeSet) AttachBroadcaster(mb *GossipBroadcaster) { - g.config.memberlistConfig.Delegate = mb -} - func (g *GossipNodeSet) Nodes() []*Node { a := make([]*Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { @@ -35,7 +38,15 @@ func (g *GossipNodeSet) Nodes() []*Node { return a } +func (g *GossipNodeSet) Start(h BroadcastHandler) error { + g.handler = h + return nil +} + func (g *GossipNodeSet) Open() error { + if g.handler == nil { + return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") + } ml, err := memberlist.Create(g.config.memberlistConfig) if err != nil { return err @@ -48,6 +59,12 @@ func (g *GossipNodeSet) Open() error { if err != nil { return err } + g.broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return ml.NumMembers() + }, + RetransmitMult: 3, + } return nil } @@ -64,7 +81,7 @@ type GossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, s *Server) *GossipNodeSet { g := &GossipNodeSet{ LogOutput: os.Stderr, } @@ -79,25 +96,15 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = gossipHost g.config.memberlistConfig.AdvertisePort = gossipPort + g.config.memberlistConfig.Delegate = g + + g.server = s return g } -//////////////////////////////////////////////////////////////// - -// GossipBroadcaster represents a gossip implementation of pilosa.Broadcaster -// GossipBroadcaster also represents an implementation of memberlist.Delegate -type GossipBroadcaster struct { - broadcasts *memberlist.TransmitLimitedQueue - - server *Server - - // The writer for any logging. - LogOutput io.Writer -} - // SendSync implementation of the Broadcaster interface -func (g *GossipBroadcaster) SendSync(pb proto.Message) error { +func (g *GossipNodeSet) SendSync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -125,7 +132,7 @@ func (g *GossipBroadcaster) SendSync(pb proto.Message) error { } // SendAsync implementation of the Broadcaster interface -func (g *GossipBroadcaster) SendAsync(pb proto.Message) error { +func (g *GossipNodeSet) SendAsync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -139,19 +146,19 @@ func (g *GossipBroadcaster) SendAsync(pb proto.Message) error { return nil } -func (g *GossipBroadcaster) Receive(pb proto.Message) error { - if err := g.server.ReceiveMessage(pb); err != nil { +func (g *GossipNodeSet) Receive(pb proto.Message) error { + if err := g.handler.ReceiveMessage(pb); err != nil { return err } return nil } // implementation of the memberlist.Delegate interface -func (g *GossipBroadcaster) NodeMeta(limit int) []byte { +func (g *GossipNodeSet) NodeMeta(limit int) []byte { return []byte{} } -func (g *GossipBroadcaster) NotifyMsg(b []byte) { +func (g *GossipNodeSet) NotifyMsg(b []byte) { m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -163,11 +170,11 @@ func (g *GossipBroadcaster) NotifyMsg(b []byte) { } } -func (g *GossipBroadcaster) GetBroadcasts(overhead, limit int) [][]byte { +func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } -func (g *GossipBroadcaster) LocalState(join bool) []byte { +func (g *GossipNodeSet) LocalState(join bool) []byte { pb, err := g.server.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) @@ -183,7 +190,7 @@ func (g *GossipBroadcaster) LocalState(join bool) []byte { return buf } -func (g *GossipBroadcaster) MergeRemoteState(buf []byte, join bool) { +func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeState if err := proto.Unmarshal(buf, &pb); err != nil { @@ -196,32 +203,6 @@ func (g *GossipBroadcaster) MergeRemoteState(buf []byte, join bool) { } } -// logger returns a logger for the GossipBroadcaster -func (g *GossipBroadcaster) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - -//////////////////////////////////////////////////////////////// - -// NewGossipBroadcaster returns a new instance of GossipBroadcaster. -func NewGossipBroadcaster(s *Server) *GossipBroadcaster { - g := &GossipBroadcaster{ - LogOutput: os.Stderr, - server: s, - } - - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - return g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() - }, - RetransmitMult: 3, - } - - return g -} - -//////////////////////////////////////////////////////////////// - // broadcast represents an implementation of memberlist.Broadcast type broadcast struct { msg []byte diff --git a/handler.go b/handler.go index 83f969d22..94b99bbe0 100644 --- a/handler.go +++ b/handler.go @@ -192,36 +192,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handlePostMessage handles /message requests. -func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Unmarshal message to specific proto type. - m, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.Server.ReceiveMessage(m); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - return -} - func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { var ms map[string]uint64 if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { diff --git a/messenger.go b/messenger.go index 0549e35fc..f0b1e76a6 100644 --- a/messenger.go +++ b/messenger.go @@ -11,6 +11,8 @@ import ( "golang.org/x/sync/errgroup" + "net" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -46,11 +48,12 @@ func (c *nopBroadcaster) SendAsync(pb proto.Message) error { // HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. type HTTPBroadcaster struct { - server *Server + server *Server + internalPort string } // NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. -func NewHTTPBroadcaster(s *Server) *HTTPBroadcaster { +func NewHTTPBroadcaster(s *Server, internalPort string) *HTTPBroadcaster { return &HTTPBroadcaster{server: s} } @@ -103,11 +106,12 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { var client *http.Client client = http.DefaultClient + host, _, err := net.SplitHostPort(node.Host) + // Create HTTP request. req, err := http.NewRequest("POST", (&url.URL{ Scheme: "http", - Host: node.Host, - Path: "/message", + Host: host + ":" + h.internalPort, }).String(), bytes.NewReader(msg)) if err != nil { return err diff --git a/server.go b/server.go index 2f0ace9ce..111e94ee7 100644 --- a/server.go +++ b/server.go @@ -33,9 +33,10 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index - Handler *Handler - Broadcaster Broadcaster + Index *Index + Handler *Handler + Broadcaster Broadcaster + BroadcastReceiver BroadcastReceiver // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -54,9 +55,10 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), - Handler: NewHandler(), - Broadcaster: NopBroadcaster, + Index: NewIndex(), + Handler: NewHandler(), + Broadcaster: NopBroadcaster, + BroadcastReceiver: NopBroadcastReceiver, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -100,6 +102,10 @@ func (s *Server) Open() error { return err } + if err := s.BroadcastReceiver.Start(s); err != nil { + return err + } + // Open NodeSet communication if err := s.Cluster.NodeSet.Open(); err != nil { return err diff --git a/server/server.go b/server/server.go index 251bf77a5..9d00eabfb 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,29 @@ func (m *Command) Run(args ...string) (err error) { m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) } + // SetupServer + err = m.SetupServer() + if err != nil { + return err + } + + // Initialize server. + if err = m.Server.Open(); err != nil { + return fmt.Errorf("server.Open: %v", err) + } + fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) + return nil +} + +func (m *Command) SetupServer() error { + cluster := pilosa.NewCluster() + cluster.ReplicaN = m.Config.Cluster.ReplicaN + + for _, hostport := range m.Config.Cluster.Nodes { + cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) + } + m.Server.Cluster = cluster + // Setup logging output. if m.Config.LogPath == "" { m.Server.LogOutput = m.Stderr @@ -90,93 +113,51 @@ func (m *Command) Run(args ...string) (err error) { m.Server.Index.Stats = pilosa.NewExpvarStatsClient() // Build cluster from config file. + var err error m.Server.Host, err = normalizeHost(m.Config.Host) if err != nil { return err } - m.Server.Broadcaster = PilosaBroadcaster(m.Config, m.Server) - m.Server.Cluster = PilosaCluster(m.Config) - // Associate objects to the Broadcaster based on config. - AssociateBroadcaster(m.Server, m.Config) - - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - - // Initialize server. - if err = m.Server.Open(); err != nil { - return fmt.Errorf("server.Open: %v", err) - } - fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) - return nil -} - -// PilosaBroadcaster returns a new instance of Broadcaster based on the config. -func PilosaBroadcaster(c *pilosa.Config, server *pilosa.Server) (broadcaster pilosa.Broadcaster) { - switch c.Cluster.BroadcasterType { + switch m.Config.Cluster.BroadcasterType { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": - broadcaster = pilosa.NewHTTPBroadcaster(server) + port := strconv.Itoa(m.Config.Cluster.Gossip.Port) + m.Server.Broadcaster = pilosa.NewHTTPBroadcaster(m.Server, port) + m.Server.BroadcastReceiver = pilosa.NewHTTPBroadcastReceiver(port, m.Stderr) + m.Server.Cluster.NodeSet = pilosa.NewHTTPNodeSet() + m.Server.Cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(m.Server.Cluster.Nodes) case "gossip": - broadcaster = pilosa.NewGossipBroadcaster(server) - case "static": - broadcaster = pilosa.NopBroadcaster - } - return broadcaster -} - -// PilosaCluster returns a new instance of Cluster based on the config. -func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { - cluster := pilosa.NewCluster() - cluster.ReplicaN = c.Cluster.ReplicaN - - for _, hostport := range c.Cluster.Nodes { - cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) - } - - // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - switch c.Cluster.BroadcasterType { - case "http": - cluster.NodeSet = pilosa.NewHTTPNodeSet() - cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(cluster.Nodes) - case "gossip": - gport, err := strconv.Atoi(pilosa.DefaultGossipPort) + gossipPort, err := strconv.Atoi(pilosa.DefaultGossipPort) if err != nil { panic(err) // Atoi on a compile-time constant should never fail. } - gossipPort := gport gossipSeed := pilosa.DefaultHost - if c.Cluster.Gossip.Port != 0 { - gossipPort = c.Cluster.Gossip.Port + if m.Config.Cluster.Gossip.Port != 0 { + gossipPort = m.Config.Cluster.Gossip.Port } - if c.Cluster.Gossip.Seed != "" { - gossipSeed = c.Cluster.Gossip.Seed + if m.Config.Cluster.Gossip.Seed != "" { + gossipSeed = m.Config.Cluster.Gossip.Seed } // get the host portion of addr to use for binding - gossipHost, _, err := net.SplitHostPort(c.Host) + gossipHost, _, err := net.SplitHostPort(m.Config.Host) if err != nil { - gossipHost = c.Host + gossipHost = m.Config.Host } - cluster.NodeSet = pilosa.NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) - case "static": - cluster.NodeSet = pilosa.NewStaticNodeSet() + gossipNodeSet := pilosa.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) + m.Server.Cluster.NodeSet = gossipNodeSet + m.Server.Broadcaster = gossipNodeSet + m.Server.BroadcastReceiver = gossipNodeSet + case "static", "": + m.Server.Broadcaster = pilosa.NopBroadcaster + m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() + m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver default: - cluster.NodeSet = pilosa.NewStaticNodeSet() + return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.BroadcasterType) } - return cluster -} - -// AssociateBroadcaster allows an implementation to associate objects to the Broadcaster -// after cluster configuration. -func AssociateBroadcaster(s *pilosa.Server, c *pilosa.Config) { - switch c.Cluster.BroadcasterType { - case "http": - // nop - case "gossip": - s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroadcaster(s.Broadcaster.(*pilosa.GossipBroadcaster)) - case "static": - // nop - } + // Set configuration options. + m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) + return nil } func normalizeHost(host string) (string, error) { From 2d139d299c123426431ddec54b85eab4c5f55864 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 22:05:27 -0500 Subject: [PATCH 30/50] remove server from handler - not used anymore --- handler.go | 1 - server.go | 1 - 2 files changed, 2 deletions(-) diff --git a/handler.go b/handler.go index 94b99bbe0..c8dda53c2 100644 --- a/handler.go +++ b/handler.go @@ -27,7 +27,6 @@ import ( type Handler struct { Index *Index Broadcaster Broadcaster - Server *Server // Local hostname & cluster configuration. Host string diff --git a/server.go b/server.go index 111e94ee7..229214805 100644 --- a/server.go +++ b/server.go @@ -67,7 +67,6 @@ func NewServer() *Server { } s.Handler.Index = s.Index - s.Handler.Server = s // TODO remove return s } From 059403e6731d18d687d2ac099f721a6ab1219b2f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 22:31:32 -0500 Subject: [PATCH 31/50] add stateHandler interface to gossip now it doesn't need to directly reference pilosa.Server --- gossip.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gossip.go b/gossip.go index 1224cf69b..664b91aa7 100644 --- a/gossip.go +++ b/gossip.go @@ -13,6 +13,14 @@ import ( "github.com/pilosa/pilosa/internal" ) +// StateHandler specifies two methods which an object must implement to share +// state in the cluster. These are used by the GossipNodeSet to implement the +// LocalState and MergeRemoteState methods of memberlist.Delegate +type StateHandler interface { + LocalState() (proto.Message, error) + HandleRemoteState(proto.Message) error +} + // GossipNodeSet represents a gossip implementation of NodeSet using memberlist // GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster // GossipNodeSet also represents an implementation of memberlist.Delegate @@ -22,9 +30,8 @@ type GossipNodeSet struct { broadcasts *memberlist.TransmitLimitedQueue - server *Server - - config *GossipConfig + stateHandler StateHandler + config *GossipConfig // The writer for any logging. LogOutput io.Writer @@ -81,7 +88,7 @@ type GossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, s *Server) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh StateHandler) *GossipNodeSet { g := &GossipNodeSet{ LogOutput: os.Stderr, } @@ -98,7 +105,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.AdvertisePort = gossipPort g.config.memberlistConfig.Delegate = g - g.server = s + g.stateHandler = sh return g } @@ -110,7 +117,7 @@ func (g *GossipNodeSet) SendSync(pb proto.Message) error { return err } - mlist := g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist + mlist := g.memberlist // Direct sends the message directly to every node. // An error from any node raises an error on the entire operation. @@ -175,7 +182,7 @@ func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { } func (g *GossipNodeSet) LocalState(join bool) []byte { - pb, err := g.server.LocalState() + pb, err := g.stateHandler.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) return []byte{} @@ -197,7 +204,7 @@ func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { g.logger().Printf("error unmarshalling nodestate data, err=%s", err) return } - err := g.server.HandleRemoteState(&pb) + err := g.stateHandler.HandleRemoteState(&pb) if err != nil { g.logger().Printf("merge state error: %s", err) } From c4d3bde50198802c3df44369c165eacd62b0b14a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 08:41:23 -0500 Subject: [PATCH 32/50] move gossip code to separate package --- gossip.go => gossip/gossip.go | 23 ++++++++++++----------- server/server.go | 3 ++- 2 files changed, 14 insertions(+), 12 deletions(-) rename gossip.go => gossip/gossip.go (90%) diff --git a/gossip.go b/gossip/gossip.go similarity index 90% rename from gossip.go rename to gossip/gossip.go index 664b91aa7..050435441 100644 --- a/gossip.go +++ b/gossip/gossip.go @@ -1,4 +1,4 @@ -package pilosa +package gossip import ( "fmt" @@ -10,6 +10,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" ) @@ -26,7 +27,7 @@ type StateHandler interface { // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { memberlist *memberlist.Memberlist - handler BroadcastHandler + handler pilosa.BroadcastHandler broadcasts *memberlist.TransmitLimitedQueue @@ -37,15 +38,15 @@ type GossipNodeSet struct { LogOutput io.Writer } -func (g *GossipNodeSet) Nodes() []*Node { - a := make([]*Node, 0, g.memberlist.NumMembers()) +func (g *GossipNodeSet) Nodes() []*pilosa.Node { + a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { - a = append(a, &Node{Host: n.Name}) + a = append(a, &pilosa.Node{Host: n.Name}) } return a } -func (g *GossipNodeSet) Start(h BroadcastHandler) error { +func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error { g.handler = h return nil } @@ -61,8 +62,8 @@ func (g *GossipNodeSet) Open() error { g.memberlist = ml // attach to gossip seed node - nodes := []*Node{&Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds - _, err = g.memberlist.Join(Nodes(nodes).Hosts()) + nodes := []*pilosa.Node{&pilosa.Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds + _, err = g.memberlist.Join(pilosa.Nodes(nodes).Hosts()) if err != nil { return err } @@ -112,7 +113,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed // SendSync implementation of the Broadcaster interface func (g *GossipNodeSet) SendSync(pb proto.Message) error { - msg, err := MarshalMessage(pb) + msg, err := pilosa.MarshalMessage(pb) if err != nil { return err } @@ -140,7 +141,7 @@ func (g *GossipNodeSet) SendSync(pb proto.Message) error { // SendAsync implementation of the Broadcaster interface func (g *GossipNodeSet) SendAsync(pb proto.Message) error { - msg, err := MarshalMessage(pb) + msg, err := pilosa.MarshalMessage(pb) if err != nil { return err } @@ -166,7 +167,7 @@ func (g *GossipNodeSet) NodeMeta(limit int) []byte { } func (g *GossipNodeSet) NotifyMsg(b []byte) { - m, err := UnmarshalMessage(b) + m, err := pilosa.UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) return diff --git a/server/server.go b/server/server.go index 9d00eabfb..80c0457f8 100644 --- a/server/server.go +++ b/server/server.go @@ -17,6 +17,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gossip" ) func init() { @@ -143,7 +144,7 @@ func (m *Command) SetupServer() error { if err != nil { gossipHost = m.Config.Host } - gossipNodeSet := pilosa.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) + gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet From 802fd32988c66927b43401f911ec090b474c1519 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 08:59:46 -0500 Subject: [PATCH 33/50] separate broadcast interfaces from implementations interfaces are in broadcast.go and http implementations in messenger.go (gossip is already in a separate package). Next changes will bring in the HTTP nodestate implementation, and separate the http implementation into a separate package. --- broadcast.go | 126 +++++++++++++++++++++++++++++++------------------- messenger.go | 128 ++++++++++++++++++--------------------------------- 2 files changed, 124 insertions(+), 130 deletions(-) diff --git a/broadcast.go b/broadcast.go index feaf7f55f..d56576b2c 100644 --- a/broadcast.go +++ b/broadcast.go @@ -2,13 +2,39 @@ package pilosa import ( "fmt" - "io" - "io/ioutil" - "net/http" + "reflect" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) +// Broadcaster is an interface for broadcasting messages. +type Broadcaster interface { + SendSync(pb proto.Message) error + SendAsync(pb proto.Message) error +} + +func init() { + NopBroadcaster = &nopBroadcaster{} +} + +var NopBroadcaster Broadcaster + +// nopBroadcaster represents a Broadcaster that doesn't do anything. +type nopBroadcaster struct{} + +// SendSync A no-op implemenetation of Broadcaster SendSync method. +func (c *nopBroadcaster) SendSync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? + return nil +} + +// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +func (c *nopBroadcaster) SendAsync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? + return nil +} + // BroadcastHandler is the interface for the pilosa object which knows how to // handle broadcast messages. (Hint: this is implemented by pilosa.Server) type BroadcastHandler interface { @@ -32,52 +58,58 @@ func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil } var NopBroadcastReceiver = &nopBroadcastReceiver{} -type HTTPBroadcastReceiver struct { - port string - handler BroadcastHandler - logOutput io.Writer -} +const ( + MessageTypeCreateSlice = 1 + MessageTypeCreateDB = 2 + MessageTypeDeleteDB = 3 + MessageTypeCreateFrame = 4 + MessageTypeDeleteFrame = 5 +) -func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { - return &HTTPBroadcastReceiver{ - port: port, - logOutput: logOutput, +func MarshalMessage(m proto.Message) ([]byte, error) { + var typ uint8 + switch obj := m.(type) { + case *internal.CreateSliceMessage: + typ = MessageTypeCreateSlice + case *internal.CreateDBMessage: + typ = MessageTypeCreateDB + case *internal.DeleteDBMessage: + typ = MessageTypeDeleteDB + case *internal.CreateFrameMessage: + typ = MessageTypeCreateFrame + case *internal.DeleteFrameMessage: + typ = MessageTypeDeleteFrame + default: + return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } -} - -func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { - rec.handler = b - go func() { - err := http.ListenAndServe(":"+rec.port, rec) - if err != nil { - fmt.Fprintf(rec.logOutput, "Error listening on %v for HTTPBroadcastReceiver: %v\n", ":"+rec.port, err) - } - }() - return nil -} - -func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) + buf, err := proto.Marshal(m) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Unmarshal message to specific proto type. - m, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := rec.handler.ReceiveMessage(m); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + return nil, err } + return append([]byte{typ}, buf...), nil +} + +func UnmarshalMessage(buf []byte) (proto.Message, error) { + typ, buf := buf[0], buf[1:] + + var m proto.Message + switch typ { + case MessageTypeCreateSlice: + m = &internal.CreateSliceMessage{} + case MessageTypeCreateDB: + m = &internal.CreateDBMessage{} + case MessageTypeDeleteDB: + m = &internal.DeleteDBMessage{} + case MessageTypeCreateFrame: + m = &internal.CreateFrameMessage{} + case MessageTypeDeleteFrame: + m = &internal.DeleteFrameMessage{} + default: + return nil, fmt.Errorf("invalid message type: %d", typ) + } + + if err := proto.Unmarshal(buf, m); err != nil { + return nil, err + } + return m, nil } diff --git a/messenger.go b/messenger.go index f0b1e76a6..78a1d1696 100644 --- a/messenger.go +++ b/messenger.go @@ -4,48 +4,18 @@ import ( "bytes" "errors" "fmt" + "io" "io/ioutil" "net/http" "net/url" - "reflect" "golang.org/x/sync/errgroup" "net" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) -// Broadcaster is an interface for handling incoming/outgoing messages. -type Broadcaster interface { - SendSync(pb proto.Message) error - SendAsync(pb proto.Message) error -} - -func init() { - NopBroadcaster = &nopBroadcaster{} -} - -var NopBroadcaster Broadcaster - -// nopBroadcaster represents a Broadcaster that doesn't do anything. -type nopBroadcaster struct{} - -// SendSync A no-op implemenetation of Broadcaster SendSync method. -func (c *nopBroadcaster) SendSync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? - return nil -} - -// SendAsync A no-op implemenetation of Broadcaster SendAsync method. -func (c *nopBroadcaster) SendAsync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? - return nil -} - -////////////////////////////////////////////////////////////////// - // HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. type HTTPBroadcaster struct { server *Server @@ -142,60 +112,52 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { return nil } -////////////////////////////////////////////////////////////////// +type HTTPBroadcastReceiver struct { + port string + handler BroadcastHandler + logOutput io.Writer +} -const ( - MessageTypeCreateSlice = 1 - MessageTypeCreateDB = 2 - MessageTypeDeleteDB = 3 - MessageTypeCreateFrame = 4 - MessageTypeDeleteFrame = 5 -) - -func MarshalMessage(m proto.Message) ([]byte, error) { - var typ uint8 - switch obj := m.(type) { - case *internal.CreateSliceMessage: - typ = MessageTypeCreateSlice - case *internal.CreateDBMessage: - typ = MessageTypeCreateDB - case *internal.DeleteDBMessage: - typ = MessageTypeDeleteDB - case *internal.CreateFrameMessage: - typ = MessageTypeCreateFrame - case *internal.DeleteFrameMessage: - typ = MessageTypeDeleteFrame - default: - return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) +func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { + return &HTTPBroadcastReceiver{ + port: port, + logOutput: logOutput, } - buf, err := proto.Marshal(m) +} + +func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { + rec.handler = b + go func() { + err := http.ListenAndServe(":"+rec.port, rec) + if err != nil { + fmt.Fprintf(rec.logOutput, "Error listening on %v for HTTPBroadcastReceiver: %v\n", ":"+rec.port, err) + } + }() + return nil +} + +func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) if err != nil { - return nil, err + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal message to specific proto type. + m, err := UnmarshalMessage(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := rec.handler.ReceiveMessage(m); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } - return append([]byte{typ}, buf...), nil -} - -func UnmarshalMessage(buf []byte) (proto.Message, error) { - typ, buf := buf[0], buf[1:] - - var m proto.Message - switch typ { - case MessageTypeCreateSlice: - m = &internal.CreateSliceMessage{} - case MessageTypeCreateDB: - m = &internal.CreateDBMessage{} - case MessageTypeDeleteDB: - m = &internal.DeleteDBMessage{} - case MessageTypeCreateFrame: - m = &internal.CreateFrameMessage{} - case MessageTypeDeleteFrame: - m = &internal.DeleteFrameMessage{} - default: - return nil, fmt.Errorf("invalid message type: %d", typ) - } - - if err := proto.Unmarshal(buf, m); err != nil { - return nil, err - } - return m, nil } From d2621709f13a1485c36b2edc752c244ecf9a2c09 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 09:19:34 -0500 Subject: [PATCH 34/50] move NodeSet out of cluster.go into broadcast and messenger HTTP implementation will be split into separate package. --- broadcast.go | 26 +++++++++++++++++++++++ cluster.go | 59 ---------------------------------------------------- messenger.go | 23 ++++++++++++++++++++ 3 files changed, 49 insertions(+), 59 deletions(-) diff --git a/broadcast.go b/broadcast.go index d56576b2c..6eaf834cc 100644 --- a/broadcast.go +++ b/broadcast.go @@ -8,6 +8,32 @@ import ( "github.com/pilosa/pilosa/internal" ) +// NodeSet represents an interface for Node membership and inter-node communication. +type NodeSet interface { + // Returns a list of all Nodes in the cluster + Nodes() []*Node + + // Open starts any network activity implemented by the NodeSet + Open() error +} + +// StaticNodeSet represents a basic NodeSet for testing +type StaticNodeSet struct { + nodes []*Node +} + +func NewStaticNodeSet() *StaticNodeSet { + return &StaticNodeSet{} +} + +func (s *StaticNodeSet) Nodes() []*Node { + return s.nodes +} + +func (s *StaticNodeSet) Open() error { + return nil +} + // Broadcaster is an interface for broadcasting messages. type Broadcaster interface { SendSync(pb proto.Message) error diff --git a/cluster.go b/cluster.go index ef67b7a4a..d8f4401e3 100644 --- a/cluster.go +++ b/cluster.go @@ -3,8 +3,6 @@ package pilosa import ( "encoding/binary" "hash/fnv" - - "github.com/gogo/protobuf/proto" ) const ( @@ -191,15 +189,6 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// NodeSet represents an interface for Node membership and inter-node communication. -type NodeSet interface { - // Returns a list of all Nodes in the cluster - Nodes() []*Node - - // Open starts any network activity implemented by the NodeSet - Open() error -} - // Hasher represents an interface to hash integers into buckets. type Hasher interface { // Hashes the key into a number between [0,N). @@ -222,51 +211,3 @@ func (h *jmphasher) Hash(key uint64, n int) int { } return int(b) } - -// HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. -type HTTPNodeSet struct { - nodes []*Node -} - -// NewHTTPNodeSet returns a new instance of HTTPNodeSet. -func NewHTTPNodeSet() *HTTPNodeSet { - return &HTTPNodeSet{} -} - -func (h *HTTPNodeSet) Nodes() []*Node { - return h.nodes -} - -func (h *HTTPNodeSet) Open() error { - return nil -} - -func (h *HTTPNodeSet) Join(nodes []*Node) error { - h.nodes = nodes - return nil -} - -// StaticNodeSet represents a basic NodeSet for testing -type StaticNodeSet struct { - nodes []*Node -} - -func NewStaticNodeSet() *StaticNodeSet { - return &StaticNodeSet{} -} - -func (s *StaticNodeSet) Nodes() []*Node { - return s.nodes -} - -func (s *StaticNodeSet) Open() error { - return nil -} - -func (s *StaticNodeSet) SendSync(pb proto.Message) error { - return nil -} - -func (s *StaticNodeSet) SendAsync(pb proto.Message) error { - return nil -} diff --git a/messenger.go b/messenger.go index 78a1d1696..37be45c0c 100644 --- a/messenger.go +++ b/messenger.go @@ -161,3 +161,26 @@ func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Reque return } } + +// HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. +type HTTPNodeSet struct { + nodes []*Node +} + +// NewHTTPNodeSet returns a new instance of HTTPNodeSet. +func NewHTTPNodeSet() *HTTPNodeSet { + return &HTTPNodeSet{} +} + +func (h *HTTPNodeSet) Nodes() []*Node { + return h.nodes +} + +func (h *HTTPNodeSet) Open() error { + return nil +} + +func (h *HTTPNodeSet) Join(nodes []*Node) error { + h.nodes = nodes + return nil +} From 6e8bf5dcc16f78fb6a68ac9a401d7d6659d51209 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 09:43:50 -0500 Subject: [PATCH 35/50] move http messenger stuff to separate package --- cluster_test.go | 5 +++-- messenger.go => httpbroadcast/messenger.go | 25 +++++++++++----------- server/server.go | 12 +++++++---- 3 files changed, 24 insertions(+), 18 deletions(-) rename messenger.go => httpbroadcast/messenger.go (85%) diff --git a/cluster_test.go b/cluster_test.go index 1dd6bb486..9dbaa420d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -9,6 +9,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/httpbroadcast" ) // Ensure the cluster can fairly distribute partitions across the nodes. @@ -95,10 +96,10 @@ func TestCluster_Health(t *testing.T) { {Host: "serverB:1000"}, {Host: "serverC:1000"}, }, - NodeSet: &pilosa.HTTPNodeSet{}, + NodeSet: &httpbroadcast.HTTPNodeSet{}, } - err := c.NodeSet.(*pilosa.HTTPNodeSet).Join([]*pilosa.Node{ + err := c.NodeSet.(*httpbroadcast.HTTPNodeSet).Join([]*pilosa.Node{ &pilosa.Node{Host: "serverA:1000"}, &pilosa.Node{Host: "serverC:1000"}, &pilosa.Node{Host: "serverD:1000"}, diff --git a/messenger.go b/httpbroadcast/messenger.go similarity index 85% rename from messenger.go rename to httpbroadcast/messenger.go index 37be45c0c..1545e3328 100644 --- a/messenger.go +++ b/httpbroadcast/messenger.go @@ -1,4 +1,4 @@ -package pilosa +package httpbroadcast import ( "bytes" @@ -14,16 +14,17 @@ import ( "net" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" ) // HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. type HTTPBroadcaster struct { - server *Server + server *pilosa.Server internalPort string } // NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. -func NewHTTPBroadcaster(s *Server, internalPort string) *HTTPBroadcaster { +func NewHTTPBroadcaster(s *pilosa.Server, internalPort string) *HTTPBroadcaster { return &HTTPBroadcaster{server: s} } @@ -31,7 +32,7 @@ func NewHTTPBroadcaster(s *Server, internalPort string) *HTTPBroadcaster { // It waits for all nodes to respond before the function returns (and returns any errors). func (h *HTTPBroadcaster) SendSync(pb proto.Message) error { // Marshal the pb to []byte - buf, err := MarshalMessage(pb) + buf, err := pilosa.MarshalMessage(pb) if err != nil { return err } @@ -61,7 +62,7 @@ func (h *HTTPBroadcaster) SendAsync(pb proto.Message) error { return h.SendSync(pb) } -func (h *HTTPBroadcaster) nodes() ([]*Node, error) { +func (h *HTTPBroadcaster) nodes() ([]*pilosa.Node, error) { if h.server == nil { return nil, errors.New("HTTPBroadcaster has no reference to Server.") } @@ -72,7 +73,7 @@ func (h *HTTPBroadcaster) nodes() ([]*Node, error) { return nodeset.Nodes(), nil } -func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { +func (h *HTTPBroadcaster) sendNodeMessage(node *pilosa.Node, msg []byte) error { var client *http.Client client = http.DefaultClient @@ -114,7 +115,7 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { type HTTPBroadcastReceiver struct { port string - handler BroadcastHandler + handler pilosa.BroadcastHandler logOutput io.Writer } @@ -125,7 +126,7 @@ func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastRe } } -func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { +func (rec *HTTPBroadcastReceiver) Start(b pilosa.BroadcastHandler) error { rec.handler = b go func() { err := http.ListenAndServe(":"+rec.port, rec) @@ -150,7 +151,7 @@ func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Reque } // Unmarshal message to specific proto type. - m, err := UnmarshalMessage(body) + m, err := pilosa.UnmarshalMessage(body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -164,7 +165,7 @@ func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Reque // HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. type HTTPNodeSet struct { - nodes []*Node + nodes []*pilosa.Node } // NewHTTPNodeSet returns a new instance of HTTPNodeSet. @@ -172,7 +173,7 @@ func NewHTTPNodeSet() *HTTPNodeSet { return &HTTPNodeSet{} } -func (h *HTTPNodeSet) Nodes() []*Node { +func (h *HTTPNodeSet) Nodes() []*pilosa.Node { return h.nodes } @@ -180,7 +181,7 @@ func (h *HTTPNodeSet) Open() error { return nil } -func (h *HTTPNodeSet) Join(nodes []*Node) error { +func (h *HTTPNodeSet) Join(nodes []*pilosa.Node) error { h.nodes = nodes return nil } diff --git a/server/server.go b/server/server.go index 80c0457f8..3571dcf3c 100644 --- a/server/server.go +++ b/server/server.go @@ -18,6 +18,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/httpbroadcast" ) func init() { @@ -123,10 +124,13 @@ func (m *Command) SetupServer() error { switch m.Config.Cluster.BroadcasterType { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": port := strconv.Itoa(m.Config.Cluster.Gossip.Port) - m.Server.Broadcaster = pilosa.NewHTTPBroadcaster(m.Server, port) - m.Server.BroadcastReceiver = pilosa.NewHTTPBroadcastReceiver(port, m.Stderr) - m.Server.Cluster.NodeSet = pilosa.NewHTTPNodeSet() - m.Server.Cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(m.Server.Cluster.Nodes) + m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, port) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(port, m.Stderr) + m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() + err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) + if err != nil { + return err + } case "gossip": gossipPort, err := strconv.Atoi(pilosa.DefaultGossipPort) if err != nil { From 0da580d0e902e34671ffcff2de613bced9d7e749 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 11:18:43 -0500 Subject: [PATCH 36/50] change broadcaster-type to type It is nested under "cluster" in the config, and it controls cluster membership as well as broadcasting, so I think type is more appropriate. Also, brevity. --- cmd/server.go | 2 +- config.go | 12 ++++++------ server/server.go | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index e9d7fcfdf..2dd662bbe 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -83,7 +83,7 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.BroadcasterType, "cluster.broadcaster-type", "", "static", "Type of Broadcaster to use for inter-host messaging. Choose from [static, http, gossip]") + flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") diff --git a/config.go b/config.go index 8b718f4e1..094bb9d65 100644 --- a/config.go +++ b/config.go @@ -4,10 +4,10 @@ import "time" const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultBroadcasterType = "static" - DefaultGossipPort = "14000" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultClusterType = "static" + DefaultGossipPort = "14000" ) // Config represents the configuration for the command. @@ -17,7 +17,7 @@ type Config struct { Cluster struct { ReplicaN int `toml:"replicas"` - BroadcasterType string `toml:"broadcaster-type"` + Type string `toml:"type"` Nodes []string `toml:"hosts"` PollingInterval Duration `toml:"polling-interval"` Gossip ConfigGossip `toml:"gossip"` @@ -45,7 +45,7 @@ func NewConfig() *Config { Host: DefaultHost + ":" + DefaultPort, } c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.BroadcasterType = DefaultBroadcasterType + c.Cluster.Type = DefaultClusterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) diff --git a/server/server.go b/server/server.go index 3571dcf3c..16d17407f 100644 --- a/server/server.go +++ b/server/server.go @@ -114,14 +114,13 @@ func (m *Command) SetupServer() error { m.Server.Index.Path = m.Config.DataDir m.Server.Index.Stats = pilosa.NewExpvarStatsClient() - // Build cluster from config file. var err error m.Server.Host, err = normalizeHost(m.Config.Host) if err != nil { return err } - switch m.Config.Cluster.BroadcasterType { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership + switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": port := strconv.Itoa(m.Config.Cluster.Gossip.Port) m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, port) @@ -157,7 +156,7 @@ func (m *Command) SetupServer() error { m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver default: - return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.BroadcasterType) + return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type) } // Set configuration options. From d10d8d94e1ee57b1555478c17f1d71f5f8567443 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 11:31:17 -0500 Subject: [PATCH 37/50] simplify config structure - remove gossip sub-struct --- cmd/server.go | 4 ++-- config.go | 16 ++++++---------- server/server.go | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 2dd662bbe..51d92824a 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -84,8 +84,8 @@ on the configured port.`, flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") - flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") - flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") + flags.StringVarP(&Server.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.") + flags.StringVarP(&Server.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") return serveCmd } diff --git a/config.go b/config.go index 094bb9d65..22750d72d 100644 --- a/config.go +++ b/config.go @@ -16,11 +16,12 @@ type Config struct { Host string `toml:"host"` Cluster struct { - ReplicaN int `toml:"replicas"` - Type string `toml:"type"` - Nodes []string `toml:"hosts"` - PollingInterval Duration `toml:"polling-interval"` - Gossip ConfigGossip `toml:"gossip"` + ReplicaN int `toml:"replicas"` + Type string `toml:"type"` + Nodes []string `toml:"hosts"` + PollingInterval Duration `toml:"polling-interval"` + InternalPort string `toml:"internal-port"` + GossipSeed string `toml:"gossip-seed"` } `toml:"cluster"` Plugins struct { @@ -34,11 +35,6 @@ type Config struct { LogPath string `toml:"log-path"` } -type ConfigGossip struct { - Port int `toml:"port"` - Seed string `toml:"seed"` -} - // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ diff --git a/server/server.go b/server/server.go index 16d17407f..a627daade 100644 --- a/server/server.go +++ b/server/server.go @@ -122,25 +122,25 @@ func (m *Command) SetupServer() error { switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": - port := strconv.Itoa(m.Config.Cluster.Gossip.Port) - m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, port) - m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(port, m.Stderr) + m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr) m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) if err != nil { return err } case "gossip": - gossipPort, err := strconv.Atoi(pilosa.DefaultGossipPort) + gossipPortStr := pilosa.DefaultGossipPort + if m.Config.Cluster.InternalPort != "" { + gossipPortStr = m.Config.Cluster.InternalPort + } + gossipPort, err := strconv.Atoi(gossipPortStr) if err != nil { - panic(err) // Atoi on a compile-time constant should never fail. + return err } gossipSeed := pilosa.DefaultHost - if m.Config.Cluster.Gossip.Port != 0 { - gossipPort = m.Config.Cluster.Gossip.Port - } - if m.Config.Cluster.Gossip.Seed != "" { - gossipSeed = m.Config.Cluster.Gossip.Seed + if m.Config.Cluster.GossipSeed != "" { + gossipSeed = m.Config.Cluster.GossipSeed } // get the host portion of addr to use for binding gossipHost, _, err := net.SplitHostPort(m.Config.Host) From a11daecfcbdecb0e409b3d24a5268f4095eaf0d0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 11:34:27 -0500 Subject: [PATCH 38/50] remove done TODO --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index a627daade..1bf3fe5ce 100644 --- a/server/server.go +++ b/server/server.go @@ -120,7 +120,7 @@ func (m *Command) SetupServer() error { return err } - switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership + switch m.Config.Cluster.Type { case "http": m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort) m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr) From b4a69be751f20f8e5c076e2b58af0088eb051699 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 09:09:21 -0500 Subject: [PATCH 39/50] Add support for Node.InternalHost. This was required for the HTTPBroadcaster to run locally on different InternalPorts. --- broadcast.go | 2 -- cluster.go | 3 ++- cmd/server.go | 1 + config.go | 20 ++++++-------------- handler_test.go | 2 +- httpbroadcast/messenger.go | 11 ++--------- server/server.go | 19 ++++++++++++------- 7 files changed, 24 insertions(+), 34 deletions(-) diff --git a/broadcast.go b/broadcast.go index 6eaf834cc..a3f3cb6c9 100644 --- a/broadcast.go +++ b/broadcast.go @@ -51,13 +51,11 @@ type nopBroadcaster struct{} // SendSync A no-op implemenetation of Broadcaster SendSync method. func (c *nopBroadcaster) SendSync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? return nil } // SendAsync A no-op implemenetation of Broadcaster SendAsync method. func (c *nopBroadcaster) SendAsync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? return nil } diff --git a/cluster.go b/cluster.go index d8f4401e3..6290f0050 100644 --- a/cluster.go +++ b/cluster.go @@ -19,7 +19,8 @@ const ( // Node represents a node in the cluster. type Node struct { - Host string `json:"host"` + Host string `json:"host"` + InternalHost string `json:"internal_host"` } // Nodes represents a list of nodes. diff --git a/cmd/server.go b/cmd/server.go index 51d92824a..d4e72a411 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -77,6 +77,7 @@ on the configured port.`, flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&Server.Config.Cluster.Nodes, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") + flags.StringSliceVarP(&Server.Config.Cluster.InternalNodes, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path") diff --git a/config.go b/config.go index 22750d72d..b577f6281 100644 --- a/config.go +++ b/config.go @@ -4,10 +4,10 @@ import "time" const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultClusterType = "static" - DefaultGossipPort = "14000" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultClusterType = "static" + DefaultInternalPort = "14000" ) // Config represents the configuration for the command. @@ -19,6 +19,7 @@ type Config struct { ReplicaN int `toml:"replicas"` Type string `toml:"type"` Nodes []string `toml:"hosts"` + InternalNodes []string `toml:"internal-hosts"` PollingInterval Duration `toml:"polling-interval"` InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` @@ -44,20 +45,11 @@ func NewConfig() *Config { c.Cluster.Type = DefaultClusterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} + c.Cluster.InternalNodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) return c } -// NewConfigForHosts returns a Config object with Config.Cluster.Nodes already -// set up. -func NewConfigForHosts(hosts []string) *Config { - conf := NewConfig() - for _, hostport := range hosts { - conf.Cluster.Nodes = append(conf.Cluster.Nodes, hostport) - } - return conf -} - // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/handler_test.go b/handler_test.go index 51f48fef7..2611da50e 100644 --- a/handler_test.go +++ b/handler_test.go @@ -763,7 +763,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"host":"host1"},{"host":"host2"}]`+"\n" { + } else if w.Body.String() != `[{"host":"host1","internal_host":""},{"host":"host2","internal_host":""}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } diff --git a/httpbroadcast/messenger.go b/httpbroadcast/messenger.go index 1545e3328..ebd4b2e6a 100644 --- a/httpbroadcast/messenger.go +++ b/httpbroadcast/messenger.go @@ -11,8 +11,6 @@ import ( "golang.org/x/sync/errgroup" - "net" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" ) @@ -25,7 +23,7 @@ type HTTPBroadcaster struct { // NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. func NewHTTPBroadcaster(s *pilosa.Server, internalPort string) *HTTPBroadcaster { - return &HTTPBroadcaster{server: s} + return &HTTPBroadcaster{server: s, internalPort: internalPort} } // SendSync sends a protobuf message to all nodes simultaneously. @@ -77,16 +75,11 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *pilosa.Node, msg []byte) error { var client *http.Client client = http.DefaultClient - host, _, err := net.SplitHostPort(node.Host) - // Create HTTP request. req, err := http.NewRequest("POST", (&url.URL{ Scheme: "http", - Host: host + ":" + h.internalPort, + Host: node.InternalHost, }).String(), bytes.NewReader(msg)) - if err != nil { - return err - } // Require protobuf encoding. req.Header.Set("Content-Type", "application/x-protobuf") diff --git a/server/server.go b/server/server.go index 1bf3fe5ce..574941c42 100644 --- a/server/server.go +++ b/server/server.go @@ -96,6 +96,9 @@ func (m *Command) SetupServer() error { for _, hostport := range m.Config.Cluster.Nodes { cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) } + for i, internalhostport := range m.Config.Cluster.InternalNodes { + cluster.Nodes[i].InternalHost = internalhostport + } m.Server.Cluster = cluster // Setup logging output. @@ -120,21 +123,23 @@ func (m *Command) SetupServer() error { return err } + // Set internal port (string). + internalPortStr := pilosa.DefaultInternalPort + if m.Config.Cluster.InternalPort != "" { + internalPortStr = m.Config.Cluster.InternalPort + } + switch m.Config.Cluster.Type { case "http": - m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort) - m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr) + m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, internalPortStr) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Stderr) m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) if err != nil { return err } case "gossip": - gossipPortStr := pilosa.DefaultGossipPort - if m.Config.Cluster.InternalPort != "" { - gossipPortStr = m.Config.Cluster.InternalPort - } - gossipPort, err := strconv.Atoi(gossipPortStr) + gossipPort, err := strconv.Atoi(internalPortStr) if err != nil { return err } From 4c8e46e17fbcfaf0483eae45a92787df6e572176 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 09:39:16 -0500 Subject: [PATCH 40/50] change Config.Nodes to Config.Hosts --- cmd/server.go | 4 ++-- cmd/server_test.go | 6 +++--- config.go | 8 ++++---- server/server.go | 7 +++++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index d4e72a411..358793af3 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -76,8 +76,8 @@ on the configured port.`, flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") - flags.StringSliceVarP(&Server.Config.Cluster.Nodes, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.StringSliceVarP(&Server.Config.Cluster.InternalNodes, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") + flags.StringSliceVarP(&Server.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") + flags.StringSliceVarP(&Server.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path") diff --git a/cmd/server_test.go b/cmd/server_test.go index f1310abc2..e51825e60 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -47,7 +47,7 @@ bind = "localhost:0" v.Check(cmd.Server.Config.DataDir, actualDataDir) v.Check(cmd.Server.Config.Host, "localhost:0") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) - v.Check(cmd.Server.Config.Cluster.Nodes, []string{"example.com:10101", "example.com:10110"}) + v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:10101", "example.com:10110"}) v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Second*182)) return v.Error() }, @@ -68,7 +68,7 @@ data-dir = "` + actualDataDir + `" `, validation: func() error { v := validator{} - v.Check(cmd.Server.Config.Cluster.Nodes, []string{"example.com:1110", "example.com:1111"}) + v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:1110", "example.com:1111"}) v.Check(cmd.Server.Config.Plugins.Path, "/var/sloth") v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) return v.Error() @@ -94,7 +94,7 @@ data-dir = "` + actualDataDir + `" `, validation: func() error { v := validator{} - v.Check(cmd.Server.Config.Cluster.Nodes, []string{"localhost:19444"}) + v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Minute*2)) v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11)) v.Check(cmd.Server.CPUProfile, profFile.Name()) diff --git a/config.go b/config.go index b577f6281..d5d725ece 100644 --- a/config.go +++ b/config.go @@ -18,8 +18,8 @@ type Config struct { Cluster struct { ReplicaN int `toml:"replicas"` Type string `toml:"type"` - Nodes []string `toml:"hosts"` - InternalNodes []string `toml:"internal-hosts"` + Hosts []string `toml:"hosts"` + InternalHosts []string `toml:"internal-hosts"` PollingInterval Duration `toml:"polling-interval"` InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` @@ -44,8 +44,8 @@ func NewConfig() *Config { c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Type = DefaultClusterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) - c.Cluster.Nodes = []string{} - c.Cluster.InternalNodes = []string{} + c.Cluster.Hosts = []string{} + c.Cluster.InternalHosts = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) return c } diff --git a/server/server.go b/server/server.go index 574941c42..ec2b55c7c 100644 --- a/server/server.go +++ b/server/server.go @@ -93,10 +93,13 @@ func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN - for _, hostport := range m.Config.Cluster.Nodes { + for _, hostport := range m.Config.Cluster.Hosts { cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) } - for i, internalhostport := range m.Config.Cluster.InternalNodes { + // TODO: if InternalHosts is not provided then pilosa.Node.InternalHost is empty. + // This will throw an error when trying to Broadcast messages over HTTP. + // One option may be to fall back to using host from hostport + config.InternalPort. + for i, internalhostport := range m.Config.Cluster.InternalHosts { cluster.Nodes[i].InternalHost = internalhostport } m.Server.Cluster = cluster From 26b524a60e6bcd35f3fc9853211728ea18d6e8fb Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 11:35:02 -0500 Subject: [PATCH 41/50] fix some tests and update glide.lock --- fragment_test.go | 33 +- glide.lock | 4 +- glide.yaml | 2 - handler.go | 12 +- internal/private.pb.go | 1744 +++++++++++++++++++++++++++++++++++++--- internal/private.proto | 55 +- 6 files changed, 1726 insertions(+), 124 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index e54b11f82..e03d5111c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -280,15 +280,34 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) - cacheLimit := uint32(3) - file, err := ioutil.TempFile("", "pilosa-fragment-") + cacheSize := uint32(3) + + // Create DB. + db := MustOpenDB() + defer db.Close() + + // Create frame. + frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) if err != nil { - panic(err) + t.Fatal(err) } - file.Close() + + // Create view. + view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + if err != nil { + t.Fatal(err) + } + + // Create fragment. + frag, err := view.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } + // Close the storage so we can re-open it without encountering a flock. + frag.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", pilosa.ViewStandard, slice, cacheLimit), + Fragment: frag, BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -316,8 +335,8 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) - } else if len(pairs) > int(cacheLimit) { - t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) + } else if len(pairs) > int(cacheSize) { + t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { diff --git a/glide.lock b/glide.lock index ae504264c..c142737ff 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 743e8f978eb4ad8f80a2ab71b05caebbf50b6769b71aa457bc4f144fef8c6595 -updated: 2017-04-18T15:33:39.035615802-05:00 +hash: 4bdea17c62dcd469584382515052e7a246fae6deefc2d62da70bf768e18e1f0c +updated: 2017-04-19T10:51:10.409094081-05:00 imports: - name: github.com/armon/go-metrics version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 diff --git a/glide.yaml b/glide.yaml index d81cbaf28..a7c0e98eb 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,7 +31,5 @@ import: - package: github.com/spf13/viper - package: github.com/gorilla/mux version: ^1.3.0 -- package: github.com/aws/aws-sdk-go - version: ^1.6.10 - package: github.com/hashicorp/memberlist - package: golang.org/x/sync diff --git a/handler.go b/handler.go index c8dda53c2..c9140867b 100644 --- a/handler.go +++ b/handler.go @@ -339,9 +339,9 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.Broadcaster.SendSync( + err = h.Broadcaster.SendSync( &internal.DeleteDBMessage{ - DB: req.DB, + DB: dbName, }) if err != nil { h.logger().Printf("problem sending DeleteDB message: %s", err) @@ -485,8 +485,8 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { // Send the create message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ - DB: req.DB, - Frame: req.Frame, + DB: dbName, + Frame: frameName, Meta: &internal.FrameMeta{ RowLabel: req.Options.RowLabel, TimeQuantum: string(req.Options.TimeQuantum), @@ -557,8 +557,8 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { // Send the delete message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ - DB: req.DB, - Frame: req.Frame, + DB: dbName, + Frame: frameName, }) if err != nil { h.logger().Printf("problem sending DeleteFrame message: %s", err) diff --git a/internal/private.pb.go b/internal/private.pb.go index 1ffa98885..5ceb800c2 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -9,13 +9,21 @@ private.proto It has these top-level messages: - DB - Frame + DBMeta + FrameMeta ImportResponse BlockDataRequest BlockDataResponse Cache MaxSlicesResponse + CreateSliceMessage + DeleteDBMessage + CreateDBMessage + CreateFrameMessage + DeleteFrameMessage + Frame + DB + NodeState */ package internal @@ -36,27 +44,28 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type DB struct { +type DBMeta struct { TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` } -func (m *DB) Reset() { *m = DB{} } -func (m *DB) String() string { return proto.CompactTextString(m) } -func (*DB) ProtoMessage() {} -func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *DBMeta) Reset() { *m = DBMeta{} } +func (m *DBMeta) String() string { return proto.CompactTextString(m) } +func (*DBMeta) ProtoMessage() {} +func (*DBMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -type Frame struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - RowLabel string `protobuf:"bytes,2,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` - InverseEnabled bool `protobuf:"varint,3,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` - CacheType string `protobuf:"bytes,4,opt,name=CacheType,proto3" json:"CacheType,omitempty"` +type FrameMeta struct { + RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` + InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } -func (m *Frame) Reset() { *m = Frame{} } -func (m *Frame) String() string { return proto.CompactTextString(m) } -func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +func (m *FrameMeta) Reset() { *m = FrameMeta{} } +func (m *FrameMeta) String() string { return proto.CompactTextString(m) } +func (*FrameMeta) ProtoMessage() {} +func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` @@ -115,16 +124,149 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { return nil } +type CreateSliceMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` +} + +func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } +func (m *CreateSliceMessage) String() string { return proto.CompactTextString(m) } +func (*CreateSliceMessage) ProtoMessage() {} +func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } + +type DeleteDBMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` +} + +func (m *DeleteDBMessage) Reset() { *m = DeleteDBMessage{} } +func (m *DeleteDBMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteDBMessage) ProtoMessage() {} +func (*DeleteDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } + +type CreateDBMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *CreateDBMessage) Reset() { *m = CreateDBMessage{} } +func (m *CreateDBMessage) String() string { return proto.CompactTextString(m) } +func (*CreateDBMessage) ProtoMessage() {} +func (*CreateDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } + +func (m *CreateDBMessage) GetMeta() *DBMeta { + if m != nil { + return m.Meta + } + return nil +} + +type CreateFrameMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *CreateFrameMessage) Reset() { *m = CreateFrameMessage{} } +func (m *CreateFrameMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFrameMessage) ProtoMessage() {} +func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } + +func (m *CreateFrameMessage) GetMeta() *FrameMeta { + if m != nil { + return m.Meta + } + return nil +} + +type DeleteFrameMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` +} + +func (m *DeleteFrameMessage) Reset() { *m = DeleteFrameMessage{} } +func (m *DeleteFrameMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFrameMessage) ProtoMessage() {} +func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } + +type Frame struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *Frame) Reset() { *m = Frame{} } +func (m *Frame) String() string { return proto.CompactTextString(m) } +func (*Frame) ProtoMessage() {} +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } + +func (m *Frame) GetMeta() *FrameMeta { + if m != nil { + return m.Meta + } + return nil +} + +type DB struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` + Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` +} + +func (m *DB) Reset() { *m = DB{} } +func (m *DB) String() string { return proto.CompactTextString(m) } +func (*DB) ProtoMessage() {} +func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } + +func (m *DB) GetMeta() *DBMeta { + if m != nil { + return m.Meta + } + return nil +} + +func (m *DB) GetFrames() []*Frame { + if m != nil { + return m.Frames + } + return nil +} + +type NodeState struct { + Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + DBs []*DB `protobuf:"bytes,3,rep,name=DBs" json:"DBs,omitempty"` +} + +func (m *NodeState) Reset() { *m = NodeState{} } +func (m *NodeState) String() string { return proto.CompactTextString(m) } +func (*NodeState) ProtoMessage() {} +func (*NodeState) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } + +func (m *NodeState) GetDBs() []*DB { + if m != nil { + return m.DBs + } + return nil +} + func init() { - proto.RegisterType((*DB)(nil), "internal.DB") - proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*DBMeta)(nil), "internal.DBMeta") + proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") + proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") + proto.RegisterType((*DeleteDBMessage)(nil), "internal.DeleteDBMessage") + proto.RegisterType((*CreateDBMessage)(nil), "internal.CreateDBMessage") + proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") + proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") + proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*DB)(nil), "internal.DB") + proto.RegisterType((*NodeState)(nil), "internal.NodeState") } -func (m *DB) Marshal() (dAtA []byte, err error) { +func (m *DBMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -134,7 +276,7 @@ func (m *DB) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DB) MarshalTo(dAtA []byte) (int, error) { +func (m *DBMeta) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -154,7 +296,7 @@ func (m *DB) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Frame) Marshal() (dAtA []byte, err error) { +func (m *FrameMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -164,25 +306,19 @@ func (m *Frame) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Frame) MarshalTo(dAtA []byte) (int, error) { +func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } if len(m.RowLabel) > 0 { - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.RowLabel))) i += copy(dAtA[i:], m.RowLabel) } if m.InverseEnabled { - dAtA[i] = 0x18 + dAtA[i] = 0x10 i++ if m.InverseEnabled { dAtA[i] = 1 @@ -192,11 +328,22 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { i++ } if len(m.CacheType) > 0 { - dAtA[i] = 0x22 + dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType))) i += copy(dAtA[i:], m.CacheType) } + if m.CacheSize != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.CacheSize)) + } + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } return i, nil } @@ -391,6 +538,290 @@ func (m *MaxSlicesResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if m.Slice != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + } + return i, nil +} + +func (m *DeleteDBMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteDBMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + return i, nil +} + +func (m *CreateDBMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateDBMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n7, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + } + return i, nil +} + +func (m *CreateFrameMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Meta != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n8, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n8 + } + return i, nil +} + +func (m *DeleteFrameMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + return i, nil +} + +func (m *Frame) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frame) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n9, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 + } + return i, nil +} + +func (m *DB) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DB) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n10, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n10 + } + if m.MaxSlice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlice)) + } + if len(m.Frames) > 0 { + for _, msg := range m.Frames { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *NodeState) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeState) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Host) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + if len(m.State) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } + if len(m.DBs) > 0 { + for _, msg := range m.DBs { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -418,7 +849,7 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *DB) Size() (n int) { +func (m *DBMeta) Size() (n int) { var l int _ = l l = len(m.TimeQuantum) @@ -432,13 +863,9 @@ func (m *DB) Size() (n int) { return n } -func (m *Frame) Size() (n int) { +func (m *FrameMeta) Size() (n int) { var l int _ = l - l = len(m.TimeQuantum) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } l = len(m.RowLabel) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -450,6 +877,13 @@ func (m *Frame) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.CacheSize != 0 { + n += 1 + sovPrivate(uint64(m.CacheSize)) + } + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -534,6 +968,132 @@ func (m *MaxSlicesResponse) Size() (n int) { return n } +func (m *CreateSliceMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovPrivate(uint64(m.Slice)) + } + return n +} + +func (m *DeleteDBMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *CreateDBMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *CreateFrameMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *DeleteFrameMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *Frame) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *DB) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.MaxSlice != 0 { + n += 1 + sovPrivate(uint64(m.MaxSlice)) + } + if len(m.Frames) > 0 { + for _, e := range m.Frames { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *NodeState) Size() (n int) { + var l int + _ = l + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.DBs) > 0 { + for _, e := range m.DBs { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -547,7 +1107,7 @@ func sovPrivate(x uint64) (n int) { func sozPrivate(x uint64) (n int) { return sovPrivate(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *DB) Unmarshal(dAtA []byte) error { +func (m *DBMeta) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -570,10 +1130,10 @@ func (m *DB) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DB: wiretype end group for non-group") + return fmt.Errorf("proto: DBMeta: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: DBMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -655,7 +1215,7 @@ func (m *DB) Unmarshal(dAtA []byte) error { } return nil } -func (m *Frame) Unmarshal(dAtA []byte) error { +func (m *FrameMeta) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -678,42 +1238,13 @@ func (m *Frame) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Frame: wiretype end group for non-group") + return fmt.Errorf("proto: FrameMeta: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType) } @@ -742,7 +1273,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } m.RowLabel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) } @@ -762,7 +1293,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } } m.InverseEnabled = bool(v != 0) - case 4: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType) } @@ -791,6 +1322,54 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } m.CacheType = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CacheSize", wireType) + } + m.CacheSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CacheSize |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -1508,6 +2087,957 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateSliceMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateSliceMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteDBMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteDBMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CreateDBMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateDBMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateDBMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &DBMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateFrameMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &FrameMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteFrameMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frame) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Frame: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &FrameMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DB) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DB: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &DBMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxSlice", wireType) + } + m.MaxSlice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxSlice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frames = append(m.Frames, &Frame{}) + if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NodeState) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeState: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeState: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DBs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DBs = append(m.DBs, &DB{}) + if err := m.DBs[len(m.DBs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -1616,31 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 406 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x92, 0xd1, 0x8a, 0xd3, 0x40, - 0x14, 0x86, 0x9d, 0x36, 0x95, 0xf6, 0x2c, 0x86, 0xee, 0xb0, 0x17, 0x61, 0x59, 0x42, 0x18, 0x50, - 0x8a, 0x17, 0xbd, 0xd0, 0x1b, 0x11, 0xaf, 0xb2, 0xad, 0x6c, 0x41, 0xc1, 0x1d, 0x17, 0xef, 0xa7, - 0xf5, 0x88, 0xa1, 0x93, 0x99, 0x38, 0x99, 0x74, 0x37, 0xcf, 0xe0, 0x0b, 0x08, 0xbe, 0x90, 0x97, - 0x3e, 0x82, 0xd4, 0x17, 0x91, 0x99, 0xa4, 0x4d, 0x54, 0x10, 0xef, 0xe6, 0x7c, 0x73, 0xce, 0x3f, - 0xff, 0xf9, 0x19, 0x78, 0x50, 0x98, 0x6c, 0x27, 0x2c, 0xce, 0x0b, 0xa3, 0xad, 0xa6, 0xe3, 0x4c, - 0x59, 0x34, 0x4a, 0x48, 0x76, 0x05, 0x83, 0x45, 0x4a, 0x13, 0x38, 0xb9, 0xc9, 0x72, 0xbc, 0xae, - 0x84, 0xb2, 0x55, 0x1e, 0x91, 0x84, 0xcc, 0x26, 0xbc, 0x8f, 0x5c, 0xc7, 0xa5, 0x96, 0x55, 0xae, - 0x5e, 0x89, 0x35, 0xca, 0x68, 0xd0, 0x74, 0xf4, 0x10, 0xfb, 0x4c, 0x60, 0xf4, 0xd2, 0x88, 0x1c, - 0xff, 0x43, 0xed, 0x1c, 0xc6, 0x5c, 0xdf, 0xf6, 0xa5, 0x8e, 0x35, 0x7d, 0x04, 0xe1, 0x4a, 0xed, - 0xd0, 0x94, 0xb8, 0x54, 0x62, 0x2d, 0xf1, 0x7d, 0x34, 0x4c, 0xc8, 0x6c, 0xcc, 0xff, 0xa0, 0xf4, - 0x02, 0x26, 0x97, 0x62, 0xf3, 0x11, 0x6f, 0xea, 0x02, 0xa3, 0xc0, 0x8b, 0x74, 0x80, 0x31, 0x08, - 0x57, 0x79, 0xa1, 0x8d, 0xe5, 0x58, 0x16, 0x5a, 0x95, 0x48, 0xa7, 0x30, 0x5c, 0x1a, 0xd3, 0xba, - 0x71, 0x47, 0x76, 0x07, 0xd3, 0x54, 0xea, 0xcd, 0x76, 0x21, 0xac, 0xe0, 0xf8, 0xa9, 0xc2, 0xd2, - 0xd2, 0xd0, 0xe5, 0xd1, 0x36, 0xb9, 0x64, 0xce, 0xda, 0xa5, 0x5a, 0x9b, 0xed, 0x86, 0x67, 0x30, - 0xf2, 0x93, 0xde, 0x5a, 0xc0, 0x9b, 0xc2, 0xd1, 0xb7, 0x32, 0xdb, 0x34, 0x6e, 0x02, 0xde, 0x14, - 0x94, 0x42, 0xf0, 0x2e, 0xc3, 0xdb, 0x68, 0xe4, 0x05, 0xfc, 0x99, 0x5d, 0xc3, 0x69, 0xef, 0xe5, - 0xd6, 0xe0, 0x05, 0x4c, 0xd2, 0xcc, 0xe6, 0xa2, 0x58, 0x2d, 0xca, 0x88, 0x24, 0xc3, 0x59, 0xc0, - 0x3b, 0x40, 0x63, 0x80, 0x37, 0x46, 0x7f, 0xc8, 0x24, 0xba, 0xeb, 0x81, 0xbf, 0xee, 0x11, 0xf6, - 0x10, 0x46, 0x7e, 0xfb, 0x7f, 0xcb, 0xb0, 0xaf, 0x04, 0x4e, 0x5f, 0x8b, 0x3b, 0x6f, 0xad, 0x3c, - 0x3e, 0x7d, 0x05, 0x93, 0x23, 0xf4, 0x33, 0x27, 0x4f, 0x1e, 0xcf, 0x0f, 0x7f, 0x64, 0xfe, 0x57, - 0x7f, 0x47, 0x96, 0xca, 0x9a, 0x9a, 0x77, 0xc3, 0xe7, 0x2f, 0x20, 0xfc, 0xfd, 0xd2, 0xe5, 0xbe, - 0xc5, 0xfa, 0x90, 0xfb, 0x16, 0x6b, 0x97, 0xd3, 0x4e, 0xc8, 0xaa, 0xc9, 0x34, 0xe0, 0x4d, 0xf1, - 0x7c, 0xf0, 0x8c, 0xa4, 0xd3, 0x6f, 0xfb, 0x98, 0x7c, 0xdf, 0xc7, 0xe4, 0xc7, 0x3e, 0x26, 0x5f, - 0x7e, 0xc6, 0xf7, 0xd6, 0xf7, 0xfd, 0x87, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x99, 0x45, - 0xbe, 0x6d, 0xc1, 0x02, 0x00, 0x00, + // 603 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41, + 0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4, + 0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52, + 0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa, + 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xfa, 0xab, + 0xaf, 0xbe, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, + 0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58, + 0xbd, 0x08, 0x63, 0x3c, 0x1f, 0xf3, 0x44, 0x8c, 0x63, 0xd7, 0xe8, 0x19, 0x3b, 0x0e, 0x6b, 0x43, + 0xb2, 0xe2, 0x20, 0x8d, 0xc6, 0x71, 0x72, 0xc2, 0x07, 0x18, 0xb9, 0xa6, 0xae, 0x68, 0x41, 0xf4, + 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0xba, 0x58, 0xd3, + 0xd5, 0x31, 0x79, 0x0c, 0x9d, 0xe3, 0x64, 0x82, 0x79, 0x81, 0x87, 0x09, 0x1f, 0x44, 0x78, 0xa9, + 0xe8, 0x56, 0xd8, 0x1c, 0x4a, 0xb6, 0xc0, 0x39, 0xe0, 0xc3, 0xf7, 0x78, 0x31, 0xcd, 0xd0, 0xb5, + 0x14, 0x49, 0x03, 0xd4, 0xd9, 0x7e, 0xf8, 0x11, 0x5d, 0xbb, 0x67, 0xec, 0xac, 0xb1, 0x06, 0x98, + 0x9f, 0x68, 0xe9, 0xc6, 0x44, 0x94, 0x42, 0xe7, 0x38, 0xce, 0xd2, 0x5c, 0x30, 0x2c, 0xb2, 0x34, + 0x29, 0x90, 0x74, 0xc1, 0x3a, 0xcc, 0xf3, 0x52, 0xae, 0x3c, 0xd2, 0x6b, 0xe8, 0xfa, 0x51, 0x3a, + 0x1c, 0x05, 0x5c, 0x70, 0x86, 0x1f, 0xc6, 0x58, 0x08, 0xd2, 0x01, 0x33, 0xf0, 0xcb, 0x22, 0x33, + 0xf0, 0xc9, 0x7d, 0x58, 0x52, 0x63, 0x97, 0x9e, 0xe8, 0x40, 0xa2, 0xea, 0xa6, 0xd2, 0x6d, 0x33, + 0x1d, 0x48, 0xb4, 0x1f, 0x85, 0x43, 0xad, 0xd7, 0x66, 0x3a, 0x20, 0x04, 0xec, 0x37, 0x21, 0x5e, + 0x95, 0x22, 0xd5, 0x99, 0x9e, 0xc3, 0x46, 0xab, 0x73, 0x29, 0x70, 0x0b, 0x1c, 0x3f, 0x14, 0x31, + 0xcf, 0x8e, 0x83, 0xc2, 0x35, 0x7a, 0xd6, 0x8e, 0xcd, 0x1a, 0x80, 0x78, 0x00, 0xaf, 0xf2, 0xf4, + 0x6d, 0x18, 0xa1, 0x4c, 0x9b, 0x2a, 0xdd, 0x42, 0xe8, 0x23, 0x58, 0x52, 0xfe, 0xfc, 0x99, 0x86, + 0x7e, 0x31, 0x60, 0xe3, 0x94, 0x5f, 0x2b, 0x69, 0x45, 0xdd, 0xfa, 0x08, 0x9c, 0x1a, 0x54, 0x77, + 0x56, 0xf7, 0x9e, 0xec, 0x56, 0x9b, 0xb4, 0x7b, 0xa3, 0xbe, 0x41, 0x0e, 0x13, 0x91, 0x4f, 0x59, + 0x73, 0x79, 0xf3, 0x39, 0x74, 0x7e, 0x4f, 0x4a, 0xdf, 0x47, 0x38, 0xad, 0x7c, 0x1f, 0xe1, 0x54, + 0xfa, 0x34, 0xe1, 0xd1, 0x58, 0x7b, 0x6a, 0x33, 0x1d, 0xec, 0x9b, 0xcf, 0x0c, 0xba, 0x0f, 0xe4, + 0x20, 0x47, 0x2e, 0x50, 0x11, 0x9c, 0x62, 0x51, 0xf0, 0x77, 0xb8, 0xe8, 0x9b, 0x68, 0x9f, 0xcd, + 0x96, 0xcf, 0xf4, 0x01, 0xac, 0x07, 0x18, 0xa1, 0x40, 0xb9, 0xf5, 0x0b, 0x2f, 0xd2, 0x97, 0xb0, + 0xae, 0xe9, 0x6f, 0x2d, 0x21, 0x0f, 0xc1, 0x96, 0x1b, 0xae, 0xa8, 0x57, 0xf7, 0xba, 0x8d, 0x09, + 0xfa, 0x2d, 0x31, 0x95, 0xa5, 0xc3, 0x4a, 0x67, 0xf9, 0x24, 0x6e, 0xd5, 0xb9, 0x60, 0x77, 0xb6, + 0xcb, 0x0e, 0x96, 0xea, 0x70, 0xaf, 0xe9, 0x50, 0x3f, 0xaf, 0xb2, 0xc9, 0x3e, 0x10, 0x3d, 0xd0, + 0xff, 0x37, 0xa1, 0x41, 0x89, 0xca, 0xed, 0x3b, 0x93, 0x59, 0x7d, 0x41, 0x9d, 0x6b, 0x05, 0xe6, + 0xdf, 0x14, 0x7c, 0x32, 0x64, 0xb3, 0x85, 0x1c, 0xff, 0xe4, 0x93, 0xfc, 0x4f, 0x54, 0xdb, 0x50, + 0x3e, 0x95, 0x3a, 0x26, 0xdb, 0xb0, 0xac, 0xfa, 0x15, 0xae, 0xad, 0x16, 0x6e, 0x7d, 0x4e, 0x07, + 0x2b, 0xd3, 0xf4, 0x35, 0x38, 0x67, 0xe9, 0x25, 0xf6, 0x05, 0x17, 0x6a, 0x9e, 0xa3, 0xb4, 0x10, + 0x95, 0x16, 0x79, 0x56, 0xfb, 0x20, 0x93, 0x95, 0x05, 0xba, 0xd2, 0x03, 0x2b, 0xf0, 0x0b, 0xd7, + 0x52, 0xe4, 0x77, 0xdb, 0x02, 0x99, 0x4c, 0xf8, 0xdd, 0x6f, 0x33, 0xcf, 0xf8, 0x3e, 0xf3, 0x8c, + 0x1f, 0x33, 0xcf, 0xf8, 0xfc, 0xd3, 0xbb, 0x33, 0x58, 0x56, 0xbf, 0xd0, 0xa7, 0xbf, 0x02, 0x00, + 0x00, 0xff, 0xff, 0xc3, 0x9a, 0x29, 0xff, 0x53, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index acd9ec43c..40e83eb6b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -2,16 +2,17 @@ syntax = "proto3"; package internal; -message DB { +message DBMeta { string TimeQuantum = 1; string ColumnLabel = 2; } -message Frame { - string TimeQuantum = 1; - string RowLabel = 2; - bool InverseEnabled = 3; - string CacheType = 4; +message FrameMeta { + string RowLabel = 1; + bool InverseEnabled = 2; + string CacheType = 3; + uint32 CacheSize = 4; + string TimeQuantum = 5; } message ImportResponse { @@ -39,3 +40,45 @@ message MaxSlicesResponse { map MaxSlices = 1; } +message CreateSliceMessage { + string DB = 1; + uint64 Slice = 2; +} + +message DeleteDBMessage { + string DB = 1; +} + +message CreateDBMessage { + string DB = 1; + DBMeta Meta = 2; +} + +message CreateFrameMessage { + string DB = 1; + string Frame = 2; + FrameMeta Meta = 3; +} + +message DeleteFrameMessage { + string DB = 1; + string Frame = 2; +} + +message Frame { + string Name = 1; + FrameMeta Meta = 2; +} + +message DB { + string Name = 1; + DBMeta Meta = 2; + uint64 MaxSlice = 3; + repeated Frame Frames = 4; +} + +message NodeState { + string Host = 1; + string State = 2; + repeated DB DBs = 3; +} From 25e649add8c7d4e10608e0e55f4da00bcbd02617 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 11:41:41 -0500 Subject: [PATCH 42/50] remove `cacheSize` argument from `NewFragment()` --- fragment.go | 4 ++-- fragment_test.go | 6 +++--- view.go | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fragment.go b/fragment.go index 5704e774d..9bee4288e 100644 --- a/fragment.go +++ b/fragment.go @@ -94,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64, cacheSize uint32) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64) *Fragment { return &Fragment{ path: path, db: db, @@ -102,7 +102,7 @@ func NewFragment(path, db, frame, view string, slice uint64, cacheSize uint32) * view: view, slice: slice, cacheType: DefaultCacheType, - cacheSize: cacheSize, + cacheSize: DefaultCacheSize, LogOutput: ioutil.Discard, MaxOpN: DefaultFragmentMaxOpN, diff --git a/fragment_test.go b/fragment_test.go index e03d5111c..f54f5415b 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -593,7 +593,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultCacheSize) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -654,7 +654,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultCacheSize), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -685,7 +685,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultCacheSize) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/view.go b/view.go index 9ded867ce..c7ec92733 100644 --- a/view.go +++ b/view.go @@ -216,8 +216,9 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.db, v.frame, v.name, slice, v.cacheSize) + frag := NewFragment(path, v.db, v.frame, v.name, slice) frag.cacheType = v.cacheType + frag.cacheSize = v.cacheSize frag.LogOutput = v.LogOutput frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) return frag From dcaf7617de1ac5d1df4870123af9d7ab1f24e3f8 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 13:36:58 -0500 Subject: [PATCH 43/50] changed some json tag names to Lower Camel Case to be consistent with the rest --- cluster.go | 2 +- ctl/import.go | 2 +- handler.go | 6 +++--- handler_test.go | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cluster.go b/cluster.go index 6290f0050..f92b8c6e9 100644 --- a/cluster.go +++ b/cluster.go @@ -20,7 +20,7 @@ const ( // Node represents a node in the cluster. type Node struct { Host string `json:"host"` - InternalHost string `json:"internal_host"` + InternalHost string `json:"internalHost"` } // Nodes represents a list of nodes. diff --git a/ctl/import.go b/ctl/import.go index ad3281813..df8fb00eb 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -27,7 +27,7 @@ type ImportCommand struct { Paths []string `json:"paths"` // Size of buffer used to chunk import. - BufferSize int `json:"buffer-size"` + BufferSize int `json:"bufferSize"` // Reusable client. Client *pilosa.Client `json:"-"` diff --git a/handler.go b/handler.go index c9140867b..a6309e44a 100644 --- a/handler.go +++ b/handler.go @@ -214,7 +214,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { } type sliceMaxResponse struct { - MaxSlices map[string]uint64 `json:"MaxSlices"` + MaxSlices map[string]uint64 `json:"maxSlices"` } // handleGetDBs handles GET /db request. @@ -391,7 +391,7 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques } type patchDBTimeQuantumRequest struct { - TimeQuantum string `json:"time_quantum"` + TimeQuantum string `json:"timeQuantum"` } type patchDBTimeQuantumResponse struct{} @@ -611,7 +611,7 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req } type patchFrameTimeQuantumRequest struct { - TimeQuantum string `json:"time_quantum"` + TimeQuantum string `json:"timeQuantum"` } type patchFrameTimeQuantumResponse struct{} diff --git a/handler_test.go b/handler_test.go index 2611da50e..478720e23 100644 --- a/handler_test.go +++ b/handler_test.go @@ -83,7 +83,7 @@ func TestHandler_MaxSlices(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -123,7 +123,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -556,7 +556,7 @@ func TestHandler_SetDBTimeQuantum(t *testing.T) { h := NewHandler() h.Index = idx.Index w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"time_quantum":"ymdh"}`))) + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -579,7 +579,7 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { h := NewHandler() h.Index = idx.Index w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"time_quantum":"ymdh"}`))) + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -763,7 +763,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"host":"host1","internal_host":""},{"host":"host2","internal_host":""}]`+"\n" { + } else if w.Body.String() != `[{"host":"host1","internalHost":""},{"host":"host2","internalHost":""}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } From caf35bb4d9461b9e4ea20f26b7af039f6398d2b6 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 16:07:30 -0500 Subject: [PATCH 44/50] rename messenger_test to align with broadcast --- messenger_test.go => broadcast_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename messenger_test.go => broadcast_test.go (100%) diff --git a/messenger_test.go b/broadcast_test.go similarity index 100% rename from messenger_test.go rename to broadcast_test.go From 5a806850527c802494cd5db4b44970be44d19e5d Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 16:59:56 -0500 Subject: [PATCH 45/50] add support for CreateDB and CreateFrame without POST data (i.e. use defaults) --- handler.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/handler.go b/handler.go index a6309e44a..b94a9a0ed 100644 --- a/handler.go +++ b/handler.go @@ -309,6 +309,15 @@ func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { return } + // Send the delete message to all nodes. + err := h.Broadcaster.SendSync( + &internal.DeleteDBMessage{ + DB: dbName, + }) + if err != nil { + h.logger().Printf("problem sending DeleteDB message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) @@ -323,13 +332,17 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { // Decode request. var req postDBRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the database + // with default values. + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } // Create database. - _, err := h.Index.CreateDB(dbName, req.Options) + _, err = h.Index.CreateDB(dbName, req.Options) if err == ErrDatabaseExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -340,11 +353,15 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { // Send the delete message to all nodes. err = h.Broadcaster.SendSync( - &internal.DeleteDBMessage{ + &internal.CreateDBMessage{ DB: dbName, + Meta: &internal.DBMeta{ + ColumnLabel: req.Options.ColumnLabel, + TimeQuantum: string(req.Options.TimeQuantum), + }, }) if err != nil { - h.logger().Printf("problem sending DeleteDB message: %s", err) + h.logger().Printf("problem sending CreateDB message: %s", err) } // Encode response. @@ -460,7 +477,11 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { // Decode request. var req postFrameRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the frame + // with default values. + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -473,7 +494,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Create frame. - _, err := db.CreateFrame(frameName, req.Options) + _, err = db.CreateFrame(frameName, req.Options) if err == ErrFrameExists { http.Error(w, err.Error(), http.StatusConflict) return From d2e68b298050187329d3a9e06204f50c47dc64c8 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 17:08:10 -0500 Subject: [PATCH 46/50] adjust some handler comments. remove debugging line --- handler.go | 8 ++++---- server.go | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/handler.go b/handler.go index b94a9a0ed..d9c79eeba 100644 --- a/handler.go +++ b/handler.go @@ -309,7 +309,7 @@ func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { return } - // Send the delete message to all nodes. + // Send the delete database message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteDBMessage{ DB: dbName, @@ -351,7 +351,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { return } - // Send the delete message to all nodes. + // Send the create database message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateDBMessage{ DB: dbName, @@ -503,7 +503,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { return } - // Send the create message to all nodes. + // Send the create frame message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ DB: dbName, @@ -575,7 +575,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { return } - // Send the delete message to all nodes. + // Send the delete frame message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ DB: dbName, diff --git a/server.go b/server.go index 229214805..e93b8e111 100644 --- a/server.go +++ b/server.go @@ -265,7 +265,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.DeleteDBMessage: - fmt.Println("DELETE:", obj.DB) if err := s.Index.DeleteDB(obj.DB); err != nil { return err } From 95bd4913c840882af22499f0eaf21499b607e372 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 20 Apr 2017 10:41:17 -0500 Subject: [PATCH 47/50] adjusted DBOptions/FrameOptions logic and removed some unused tests --- db.go | 8 +++ frame.go | 23 +++++-- handler.go | 12 +--- handler_test.go | 54 --------------- index_test.go | 28 -------- internal/private.pb.go | 148 ++++++++++++++++++++--------------------- internal/private.proto | 4 +- 7 files changed, 105 insertions(+), 172 deletions(-) diff --git a/db.go b/db.go index d2ceaf13a..8bf33300e 100644 --- a/db.go +++ b/db.go @@ -536,6 +536,14 @@ type DBOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } +// Encode converts o into its internal representation. +func (o *DBOptions) Encode() *internal.DBMeta { + return &internal.DBMeta{ + ColumnLabel: o.ColumnLabel, + TimeQuantum: string(o.TimeQuantum), + } +} + // hasTime returns true if a contains a non-nil time. func hasTime(a []*time.Time) bool { for _, t := range a { diff --git a/frame.go b/frame.go index a497e6906..30fd41406 100644 --- a/frame.go +++ b/frame.go @@ -302,11 +302,11 @@ func (f *Frame) loadMeta() error { func (f *Frame) saveMeta() error { // Marshal metadata. buf, err := proto.Marshal(&internal.FrameMeta{ - TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, - CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, CacheSize: f.cacheSize, + TimeQuantum: string(f.timeQuantum), }) if err != nil { return err @@ -593,9 +593,11 @@ func encodeFrame(f *Frame) *internal.Frame { return &internal.Frame{ Name: f.name, Meta: &internal.FrameMeta{ - TimeQuantum: string(f.timeQuantum), - RowLabel: f.rowLabel, - CacheSize: f.cacheSize, + RowLabel: f.rowLabel, + InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, + CacheSize: f.cacheSize, + TimeQuantum: string(f.timeQuantum), }, } } @@ -627,6 +629,17 @@ type FrameOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } +// Encode converts o into its internal representation. +func (o *FrameOptions) Encode() *internal.FrameMeta { + return &internal.FrameMeta{ + RowLabel: o.RowLabel, + InverseEnabled: o.InverseEnabled, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + TimeQuantum: string(o.TimeQuantum), + } +} + // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { diff --git a/handler.go b/handler.go index d9c79eeba..8b79267bf 100644 --- a/handler.go +++ b/handler.go @@ -354,11 +354,8 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { // Send the create database message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateDBMessage{ - DB: dbName, - Meta: &internal.DBMeta{ - ColumnLabel: req.Options.ColumnLabel, - TimeQuantum: string(req.Options.TimeQuantum), - }, + DB: dbName, + Meta: req.Options.Encode(), }) if err != nil { h.logger().Printf("problem sending CreateDB message: %s", err) @@ -508,10 +505,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { &internal.CreateFrameMessage{ DB: dbName, Frame: frameName, - Meta: &internal.FrameMeta{ - RowLabel: req.Options.RowLabel, - TimeQuantum: string(req.Options.TimeQuantum), - }, + Meta: req.Options.Encode(), }) if err != nil { h.logger().Printf("problem sending CreateFrame message: %s", err) diff --git a/handler_test.go b/handler_test.go index 478720e23..049fdc036 100644 --- a/handler_test.go +++ b/handler_test.go @@ -874,57 +874,3 @@ func MustReadAll(r io.Reader) []byte { } return buf } - -/* -// TODO: move this test to messenger.go (with NewServer()) - -// Ensure that an HTTP message sent to the cluster reaches all nodes. -func TestHTTPNodeSet_Base(t *testing.T) { - - // servers - s1 := NewServer() - s1.Messenger = pilosa.NewMessenger() - n1 := NewHTTPMessageBroker() - n1.messenger = s1.Messenger - s1.Messenger.Broker = n1 - - s2 := NewServer() - s2.Messenger = pilosa.NewMessenger() - n2 := NewHTTPMessageBroker() - n2.messenger = s2.Messenger - s2.Messenger.Broker = n2 - - s3 := NewServer() - s3.Messenger = pilosa.NewMessenger() - n3 := NewHTTPMessageBroker() - n3.messenger = s3.Messenger - s3.Messenger.Broker = n3 - - nodes := []*pilosa.Node{ - {Host: s1.Host()}, - {Host: s2.Host()}, - {Host: s3.Host()}, - } - - // message - msg := &internal.CreateSliceMessage{ - DB: "d", - Slice: 8, - } - - // send message - if err := s1.Messenger.SendMessage(msg, ""); err != nil { - t.Fatalf("failure sending message: %s", err) - } - - if !reflect.DeepEqual(mb1.messageReceived, msg) { - t.Fatalf("unexpected message received by node1: %s", mb1.messageReceived) - } - if !reflect.DeepEqual(mb2.messageReceived, msg) { - t.Fatalf("unexpected message received by node2: %s", mb2.messageReceived) - } - if !reflect.DeepEqual(mb3.messageReceived, msg) { - t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived) - } -} -*/ diff --git a/index_test.go b/index_test.go index c09fe69f5..4a1be4cf2 100644 --- a/index_test.go +++ b/index_test.go @@ -162,34 +162,6 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } } -/* TODO: move this to messenger.go -// Ensure index can handle Messenger messages. -func TestIndex_HandleMessage(t *testing.T) { - // Create a local index. - idx0 := MustOpenIndex() - defer idx0.Close() - - idx0.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - - msg0 := &internal.CreateSliceMessage{ - DB: "d", - Slice: 8, - } - idx0.HandleMessage(msg0) - if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{"d": 8}) { - t.Fatalf("unexpected max slice: %s", ms) - } - - msg1 := &internal.DeleteDBMessage{ - DB: "d", - } - idx0.HandleMessage(msg1) - if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{}) { - t.Fatalf("unexpected delete db: %s", ms) - } -} -*/ - // Index is a test wrapper for pilosa.Index. type Index struct { *pilosa.Index diff --git a/internal/private.pb.go b/internal/private.pb.go index 5ceb800c2..0423cfa08 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -45,8 +45,8 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type DBMeta struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` + ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` + TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *DBMeta) Reset() { *m = DBMeta{} } @@ -281,18 +281,18 @@ func (m *DBMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } if len(m.ColumnLabel) > 0 { - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.ColumnLabel))) i += copy(dAtA[i:], m.ColumnLabel) } + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } return i, nil } @@ -852,11 +852,11 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *DBMeta) Size() (n int) { var l int _ = l - l = len(m.TimeQuantum) + l = len(m.ColumnLabel) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.ColumnLabel) + l = len(m.TimeQuantum) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -1137,35 +1137,6 @@ func (m *DBMeta) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType) } @@ -1194,6 +1165,35 @@ func (m *DBMeta) Unmarshal(dAtA []byte) error { } m.ColumnLabel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3146,43 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 603 bytes of a gzipped FileDescriptorProto + // 600 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41, 0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4, 0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52, 0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa, - 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xfa, 0xab, - 0xaf, 0xbe, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, + 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xea, 0xab, + 0xaf, 0xbf, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, 0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58, - 0xbd, 0x08, 0x63, 0x3c, 0x1f, 0xf3, 0x44, 0x8c, 0x63, 0xd7, 0xe8, 0x19, 0x3b, 0x0e, 0x6b, 0x43, - 0xb2, 0xe2, 0x20, 0x8d, 0xc6, 0x71, 0x72, 0xc2, 0x07, 0x18, 0xb9, 0xa6, 0xae, 0x68, 0x41, 0xf4, - 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0xba, 0x58, 0xd3, - 0xd5, 0x31, 0x79, 0x0c, 0x9d, 0xe3, 0x64, 0x82, 0x79, 0x81, 0x87, 0x09, 0x1f, 0x44, 0x78, 0xa9, - 0xe8, 0x56, 0xd8, 0x1c, 0x4a, 0xb6, 0xc0, 0x39, 0xe0, 0xc3, 0xf7, 0x78, 0x31, 0xcd, 0xd0, 0xb5, - 0x14, 0x49, 0x03, 0xd4, 0xd9, 0x7e, 0xf8, 0x11, 0x5d, 0xbb, 0x67, 0xec, 0xac, 0xb1, 0x06, 0x98, - 0x9f, 0x68, 0xe9, 0xc6, 0x44, 0x94, 0x42, 0xe7, 0x38, 0xce, 0xd2, 0x5c, 0x30, 0x2c, 0xb2, 0x34, - 0x29, 0x90, 0x74, 0xc1, 0x3a, 0xcc, 0xf3, 0x52, 0xae, 0x3c, 0xd2, 0x6b, 0xe8, 0xfa, 0x51, 0x3a, - 0x1c, 0x05, 0x5c, 0x70, 0x86, 0x1f, 0xc6, 0x58, 0x08, 0xd2, 0x01, 0x33, 0xf0, 0xcb, 0x22, 0x33, - 0xf0, 0xc9, 0x7d, 0x58, 0x52, 0x63, 0x97, 0x9e, 0xe8, 0x40, 0xa2, 0xea, 0xa6, 0xd2, 0x6d, 0x33, - 0x1d, 0x48, 0xb4, 0x1f, 0x85, 0x43, 0xad, 0xd7, 0x66, 0x3a, 0x20, 0x04, 0xec, 0x37, 0x21, 0x5e, - 0x95, 0x22, 0xd5, 0x99, 0x9e, 0xc3, 0x46, 0xab, 0x73, 0x29, 0x70, 0x0b, 0x1c, 0x3f, 0x14, 0x31, - 0xcf, 0x8e, 0x83, 0xc2, 0x35, 0x7a, 0xd6, 0x8e, 0xcd, 0x1a, 0x80, 0x78, 0x00, 0xaf, 0xf2, 0xf4, - 0x6d, 0x18, 0xa1, 0x4c, 0x9b, 0x2a, 0xdd, 0x42, 0xe8, 0x23, 0x58, 0x52, 0xfe, 0xfc, 0x99, 0x86, - 0x7e, 0x31, 0x60, 0xe3, 0x94, 0x5f, 0x2b, 0x69, 0x45, 0xdd, 0xfa, 0x08, 0x9c, 0x1a, 0x54, 0x77, - 0x56, 0xf7, 0x9e, 0xec, 0x56, 0x9b, 0xb4, 0x7b, 0xa3, 0xbe, 0x41, 0x0e, 0x13, 0x91, 0x4f, 0x59, - 0x73, 0x79, 0xf3, 0x39, 0x74, 0x7e, 0x4f, 0x4a, 0xdf, 0x47, 0x38, 0xad, 0x7c, 0x1f, 0xe1, 0x54, - 0xfa, 0x34, 0xe1, 0xd1, 0x58, 0x7b, 0x6a, 0x33, 0x1d, 0xec, 0x9b, 0xcf, 0x0c, 0xba, 0x0f, 0xe4, - 0x20, 0x47, 0x2e, 0x50, 0x11, 0x9c, 0x62, 0x51, 0xf0, 0x77, 0xb8, 0xe8, 0x9b, 0x68, 0x9f, 0xcd, - 0x96, 0xcf, 0xf4, 0x01, 0xac, 0x07, 0x18, 0xa1, 0x40, 0xb9, 0xf5, 0x0b, 0x2f, 0xd2, 0x97, 0xb0, - 0xae, 0xe9, 0x6f, 0x2d, 0x21, 0x0f, 0xc1, 0x96, 0x1b, 0xae, 0xa8, 0x57, 0xf7, 0xba, 0x8d, 0x09, - 0xfa, 0x2d, 0x31, 0x95, 0xa5, 0xc3, 0x4a, 0x67, 0xf9, 0x24, 0x6e, 0xd5, 0xb9, 0x60, 0x77, 0xb6, - 0xcb, 0x0e, 0x96, 0xea, 0x70, 0xaf, 0xe9, 0x50, 0x3f, 0xaf, 0xb2, 0xc9, 0x3e, 0x10, 0x3d, 0xd0, - 0xff, 0x37, 0xa1, 0x41, 0x89, 0xca, 0xed, 0x3b, 0x93, 0x59, 0x7d, 0x41, 0x9d, 0x6b, 0x05, 0xe6, - 0xdf, 0x14, 0x7c, 0x32, 0x64, 0xb3, 0x85, 0x1c, 0xff, 0xe4, 0x93, 0xfc, 0x4f, 0x54, 0xdb, 0x50, - 0x3e, 0x95, 0x3a, 0x26, 0xdb, 0xb0, 0xac, 0xfa, 0x15, 0xae, 0xad, 0x16, 0x6e, 0x7d, 0x4e, 0x07, - 0x2b, 0xd3, 0xf4, 0x35, 0x38, 0x67, 0xe9, 0x25, 0xf6, 0x05, 0x17, 0x6a, 0x9e, 0xa3, 0xb4, 0x10, - 0x95, 0x16, 0x79, 0x56, 0xfb, 0x20, 0x93, 0x95, 0x05, 0xba, 0xd2, 0x03, 0x2b, 0xf0, 0x0b, 0xd7, - 0x52, 0xe4, 0x77, 0xdb, 0x02, 0x99, 0x4c, 0xf8, 0xdd, 0x6f, 0x33, 0xcf, 0xf8, 0x3e, 0xf3, 0x8c, - 0x1f, 0x33, 0xcf, 0xf8, 0xfc, 0xd3, 0xbb, 0x33, 0x58, 0x56, 0xbf, 0xd0, 0xa7, 0xbf, 0x02, 0x00, - 0x00, 0xff, 0xff, 0xc3, 0x9a, 0x29, 0xff, 0x53, 0x05, 0x00, 0x00, + 0x3d, 0x48, 0xa3, 0x71, 0x9c, 0x9c, 0xf0, 0x01, 0x46, 0xae, 0xd1, 0x33, 0x76, 0x1c, 0xd6, 0x86, + 0x64, 0xc5, 0x45, 0x18, 0xe3, 0xf9, 0x98, 0x27, 0x62, 0x1c, 0xbb, 0xa6, 0xae, 0x68, 0x41, 0xf4, + 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0x6d, 0xba, 0x3a, + 0x26, 0x8f, 0xa1, 0x73, 0x9c, 0x4c, 0x30, 0x2f, 0xf0, 0x30, 0xe1, 0x83, 0x08, 0x2f, 0x15, 0xdd, + 0x0a, 0x9b, 0x43, 0xc9, 0x16, 0x38, 0x07, 0x7c, 0xf8, 0x1e, 0x2f, 0xa6, 0x19, 0xba, 0x96, 0x22, + 0x69, 0x80, 0x3a, 0xdb, 0x0f, 0x3f, 0xa2, 0x6b, 0xf7, 0x8c, 0x9d, 0x35, 0xd6, 0x00, 0xf3, 0x7a, + 0x97, 0x6e, 0xea, 0xa5, 0xd0, 0x39, 0x8e, 0xb3, 0x34, 0x17, 0x0c, 0x8b, 0x2c, 0x4d, 0x0a, 0x24, + 0x5d, 0xb0, 0x0e, 0xf3, 0xbc, 0x94, 0x2b, 0x8f, 0xf4, 0x1a, 0xba, 0x7e, 0x94, 0x0e, 0x47, 0x01, + 0x17, 0x9c, 0xe1, 0x87, 0x31, 0x16, 0x82, 0x74, 0xc0, 0x0c, 0xfc, 0xb2, 0xc8, 0x0c, 0x7c, 0x72, + 0x1f, 0x96, 0xd4, 0xb5, 0x4b, 0x4f, 0x74, 0x20, 0x51, 0xd5, 0xa9, 0x74, 0xdb, 0x4c, 0x07, 0x12, + 0xed, 0x47, 0xe1, 0x50, 0xeb, 0xb5, 0x99, 0x0e, 0x08, 0x01, 0xfb, 0x4d, 0x88, 0x57, 0xa5, 0x48, + 0x75, 0xa6, 0xe7, 0xb0, 0xd1, 0x9a, 0x5c, 0x0a, 0xdc, 0x02, 0xc7, 0x0f, 0x45, 0xcc, 0xb3, 0xe3, + 0xa0, 0x70, 0x8d, 0x9e, 0xb5, 0x63, 0xb3, 0x06, 0x20, 0x1e, 0xc0, 0xab, 0x3c, 0x7d, 0x1b, 0x46, + 0x28, 0xd3, 0xa6, 0x4a, 0xb7, 0x10, 0xfa, 0x08, 0x96, 0x94, 0x3f, 0x7f, 0xa6, 0xa1, 0x5f, 0x0c, + 0xd8, 0x38, 0xe5, 0xd7, 0x4a, 0x5a, 0x51, 0x8f, 0x3e, 0x02, 0xa7, 0x06, 0x55, 0xcf, 0xea, 0xde, + 0x93, 0xdd, 0x6a, 0x93, 0x76, 0x6f, 0xd4, 0x37, 0xc8, 0x61, 0x22, 0xf2, 0x29, 0x6b, 0x9a, 0x37, + 0x9f, 0x43, 0xe7, 0xf7, 0xa4, 0xf4, 0x7d, 0x84, 0xd3, 0xca, 0xf7, 0x11, 0x4e, 0xa5, 0x4f, 0x13, + 0x1e, 0x8d, 0xb5, 0xa7, 0x36, 0xd3, 0xc1, 0xbe, 0xf9, 0xcc, 0xa0, 0xfb, 0x40, 0x0e, 0x72, 0xe4, + 0x02, 0x15, 0xc1, 0x29, 0x16, 0x05, 0x7f, 0x87, 0x8b, 0xbe, 0x89, 0xf6, 0xd9, 0x6c, 0xf9, 0x4c, + 0x1f, 0xc0, 0x7a, 0x80, 0x11, 0x0a, 0x94, 0x5b, 0xbf, 0xb0, 0x91, 0xbe, 0x84, 0x75, 0x4d, 0x7f, + 0x6b, 0x09, 0x79, 0x08, 0xb6, 0xdc, 0x70, 0x45, 0xbd, 0xba, 0xd7, 0x6d, 0x4c, 0xd0, 0x6f, 0x89, + 0xa9, 0x2c, 0x1d, 0x56, 0x3a, 0xcb, 0x27, 0x71, 0xab, 0xce, 0x05, 0xbb, 0xb3, 0x5d, 0x4e, 0xb0, + 0xd4, 0x84, 0x7b, 0xcd, 0x84, 0xfa, 0x79, 0x95, 0x43, 0xf6, 0x81, 0xe8, 0x0b, 0xfd, 0xff, 0x10, + 0x1a, 0x94, 0xa8, 0xdc, 0xbe, 0x33, 0x99, 0xd5, 0x0d, 0xea, 0x5c, 0x2b, 0x30, 0xff, 0xa6, 0xe0, + 0x93, 0x21, 0x87, 0x2d, 0xe4, 0xf8, 0x27, 0x9f, 0xe4, 0x7f, 0xa2, 0xda, 0x86, 0xf2, 0xa9, 0xd4, + 0x31, 0xd9, 0x86, 0x65, 0x35, 0xaf, 0x70, 0x6d, 0xb5, 0x70, 0xeb, 0x73, 0x3a, 0x58, 0x99, 0xa6, + 0xaf, 0xc1, 0x39, 0x4b, 0x2f, 0xb1, 0x2f, 0xb8, 0x50, 0xf7, 0x39, 0x4a, 0x0b, 0x51, 0x69, 0x91, + 0x67, 0xb5, 0x0f, 0x32, 0x59, 0x59, 0xa0, 0x2b, 0x3d, 0xb0, 0x02, 0xbf, 0x70, 0x2d, 0x45, 0x7e, + 0xb7, 0x2d, 0x90, 0xc9, 0x84, 0xdf, 0xfd, 0x36, 0xf3, 0x8c, 0xef, 0x33, 0xcf, 0xf8, 0x31, 0xf3, + 0x8c, 0xcf, 0x3f, 0xbd, 0x3b, 0x83, 0x65, 0xf5, 0x0b, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, + 0x3a, 0x23, 0x0f, 0xb4, 0x53, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 40e83eb6b..ec060e355 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,8 +3,8 @@ syntax = "proto3"; package internal; message DBMeta { - string TimeQuantum = 1; - string ColumnLabel = 2; + string ColumnLabel = 1; + string TimeQuantum = 2; } message FrameMeta { From a4921ac55a3e6c48efdb0d4fa42235cc814c61bc Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 20 Apr 2017 15:14:11 -0500 Subject: [PATCH 48/50] remove leftover Receive() method from gossip implementation --- gossip/gossip.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 050435441..772892926 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -154,13 +154,6 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error { return nil } -func (g *GossipNodeSet) Receive(pb proto.Message) error { - if err := g.handler.ReceiveMessage(pb); err != nil { - return err - } - return nil -} - // implementation of the memberlist.Delegate interface func (g *GossipNodeSet) NodeMeta(limit int) []byte { return []byte{} @@ -172,7 +165,7 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) { g.logger().Printf("unmarshal message error: %s", err) return } - if err := g.Receive(m); err != nil { + if err := g.handler.ReceiveMessage(m); err != nil { g.logger().Printf("receive message error: %s", err) return } From 3487bc373f136bd3bb89cd1e472ebff8693d666a Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 20 Apr 2017 15:45:00 -0500 Subject: [PATCH 49/50] group Server interface implementation function together --- server.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/server.go b/server.go index e93b8e111..567461ccc 100644 --- a/server.go +++ b/server.go @@ -236,20 +236,7 @@ func (s *Server) monitorMaxSlices() { } } -// LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) LocalState() (proto.Message, error) { - if s.Index == nil { - return nil, errors.New("Server.Index is nil.") - } - return &internal.NodeState{ - Host: s.Host, - State: "OK", // TODO: make this work, pull from s.Cluster.Node - DBs: encodeDBs(s.Index.DBs()), - }, nil -} - +// ReceiveMessage represents an implementation of BroadcastHandler. func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { case *internal.CreateSliceMessage: @@ -284,6 +271,21 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return nil } +// Server implements gossip.StateHandler. +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +func (s *Server) LocalState() (proto.Message, error) { + if s.Index == nil { + return nil, errors.New("Server.Index is nil.") + } + return &internal.NodeState{ + Host: s.Host, + State: "OK", // TODO: make this work, pull from s.Cluster.Node + DBs: encodeDBs(s.Index.DBs()), + }, nil +} + // HandleRemoteState receives incoming NodeState from remote nodes. func (s *Server) HandleRemoteState(pb proto.Message) error { return s.mergeRemoteState(pb.(*internal.NodeState)) From 93f36d0e9ab18f51f30aae4f74d52c128ee43029 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 21 Apr 2017 13:54:39 -0500 Subject: [PATCH 50/50] adds a basic BroadcastReceiver test --- broadcast_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/broadcast_test.go b/broadcast_test.go index ee2d720a9..a2842a919 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -35,3 +35,57 @@ func testMessageMarshal(t *testing.T, m proto.Message) { t.Fatalf("unexpected message marshalling: %s", unmarshalled) } } + +// Ensure that BroadcastReceiver can register a BroadcastHandler. +func TestBroadcast_BroadcastReceiver(t *testing.T) { + + s := pilosa.NewServer() + + sbr := NewSimpleBroadcastReceiver() + sbh := NewSimpleBroadcastHandler() + + s.BroadcastReceiver = sbr + s.BroadcastReceiver.Start(sbh) + + msg := &internal.DeleteDBMessage{ + DB: "d", + } + + s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg) + + // Make sure the message received is what was sentd + if !reflect.DeepEqual(sbh.receivedMessage, msg) { + t.Fatalf("unexpected message: %s", sbh.receivedMessage) + } +} + +type SimpleBroadcastReceiver struct { + broadcastHandler pilosa.BroadcastHandler +} + +func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver { + return &SimpleBroadcastReceiver{} +} + +func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error { + r.broadcastHandler = h + return nil +} + +func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error { + r.broadcastHandler.ReceiveMessage(pb) + return nil +} + +type SimpleBroadcastHandler struct { + receivedMessage proto.Message +} + +func NewSimpleBroadcastHandler() *SimpleBroadcastHandler { + return &SimpleBroadcastHandler{} +} + +func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error { + h.receivedMessage = pb.(proto.Message) + return nil +}