From 40f1457eef45c00eaf129a4ab636a2f98950c0bb Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:39:57 -0500 Subject: [PATCH 01/48] Diagnostics package --- diagnostics/diagnostics.go | 282 ++++++++++++++++++++++++++++++++ diagnostics/diagnostics_test.go | 143 ++++++++++++++++ 2 files changed, 425 insertions(+) create mode 100644 diagnostics/diagnostics.go create mode 100644 diagnostics/diagnostics_test.go diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go new file mode 100644 index 000000000..fc78b94cb --- /dev/null +++ b/diagnostics/diagnostics.go @@ -0,0 +1,282 @@ +package diagnostics + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/pilosa/pilosa" +) + +// TODO: white list of statsd metrics to use +// TODO: unique Cluster ID +// TODO: how should this be disabled, config + +// Default interval to sync diagnostics metrics. +const ( + DefaultDiagnosticsInterval = 10 * time.Second + DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" +) + +type versionResponse struct { + Version string `json:"version"` + Message string `json:"message"` +} + +// Diagnostics represents a client to the Pilosa cluster. +type Diagnostics struct { + mu sync.Mutex + wg sync.WaitGroup + closing chan struct{} + host string + VersionURL string + version string + startTime int64 + start time.Time + + counts map[string]int64 + metrics map[string]string + + client *http.Client + interval time.Duration + + logOutput io.Writer +} + +// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". +func New(host string) *Diagnostics { + return &Diagnostics{ + closing: make(chan struct{}), + host: host, + VersionURL: DefaultVersionCheckURL, + startTime: time.Now().Unix(), + start: time.Now(), + client: http.DefaultClient, + counts: make(map[string]int64), + metrics: make(map[string]string), + interval: DefaultDiagnosticsInterval, + logOutput: ioutil.Discard, + } +} + +// SetVersion of locally running Pilosa Cluster to check against master. +func (d *Diagnostics) SetVersion(v string) { + d.version = v + d.Set("Version", v, 1.0) +} + +// schedule start the diagnostics service ticker. +func (d *Diagnostics) schedule() { + ticker := time.NewTicker(d.interval) + defer ticker.Stop() + + for { + select { + case <-d.closing: + return + case <-ticker.C: + d.CheckVersion() + d.Flush() + } + } +} + +// Flush sends the current metrics. +func (d *Diagnostics) Flush() error { + d.mu.Lock() + d.metrics["uptime"] = strconv.FormatInt((time.Now().Unix() - d.startTime), 10) + + buf, _ := d.MarshalJSON() + d.Reset() + d.mu.Unlock() + + // d.logger().Println(string(buf)) + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // TODO verify response + // Read response into buffer. + // body, err := ioutil.ReadAll(resp.Body) + // if err != nil { + // return err + // } + + // TODO circuit breaker + return nil +} + +// Reset clears the incremented metrics. +func (d *Diagnostics) Reset() { + d.counts = make(map[string]int64) +} + +// Open starts the diagnostics metric go routine. +func (d *Diagnostics) Open() { + d.wg.Add(1) + go func() { defer d.wg.Done(); d.schedule() }() +} + +// Close notify goroutine to stop. +func (d *Diagnostics) Close() error { + close(d.closing) + d.wg.Wait() + return nil +} + +// CheckVersion of the local build against Pilosa master. +func (d *Diagnostics) CheckVersion() error { + var rsp versionResponse + req, err := http.NewRequest("GET", d.VersionURL, nil) + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return fmt.Errorf("json decode: %s", err) + } + + if err := d.CompareVersion(rsp.Version); err != nil { + d.logger().Printf("%s\n", err.Error()) + } + + return nil +} + +// CompareVersion check version strings. +func (d *Diagnostics) CompareVersion(value string) error { + currentVersion := VersionSegments(value) + localVersion := VersionSegments(d.version) + + if localVersion[0] < currentVersion[0] { //Major + return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Major release is %s", d.version, value) + } else if localVersion[1] < currentVersion[1] { // Minor + return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Minor release is %s", d.version, value) + } else if localVersion[2] < currentVersion[2] { // Patch + return fmt.Errorf("There is a new patch relese of Pilosa availbale: %s", value) + } + + return nil +} + +// MarshalJSON custom marshall string and int maps together. +func (d *Diagnostics) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString("{") + length := len(d.counts) + count := 0 + + for key, value := range d.counts { + jsonValue, err := json.Marshal(value) + if err != nil { + return nil, err + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) + count++ + if count < length { + buffer.WriteString(",") + } + } + if length > 0 { + buffer.WriteString(",") + } + length = len(d.metrics) + count = 0 + for key, value := range d.metrics { + jsonValue, err := json.Marshal(value) + if err != nil { + return nil, err + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) + count++ + if count < length { + buffer.WriteString(",") + } + } + + buffer.WriteString("}") + return buffer.Bytes(), nil +} + +// Stats interface implementation. + +// Tags no-op. +func (d *Diagnostics) Tags() []string { + return nil +} + +// WithTags no-op. +func (d *Diagnostics) WithTags(tags ...string) pilosa.StatsClient { + return d +} + +// Count tracks the number of times something occurs per diagnostic period. +func (d *Diagnostics) Count(name string, value int64, rate float64) { + d.mu.Lock() + defer d.mu.Unlock() + d.counts[name] += value +} + +// CountWithCustomTags Tracks the number of times something occurs per diagnostic period. +func (d *Diagnostics) CountWithCustomTags(name string, value int64, rate float64, tags []string) { + d.mu.Lock() + defer d.mu.Unlock() + d.counts[name] += value +} + +// Gauge records the value of a metric. +func (d *Diagnostics) Gauge(name string, value float64, rate float64) { + d.Set(name, strconv.FormatFloat(value, 'f', -1, 64), rate) +} + +// Histogram is a no-op. +func (d *Diagnostics) Histogram(name string, value float64, rate float64) { +} + +// Set adds a key value metric. +func (d *Diagnostics) Set(name string, value string, rate float64) { + d.mu.Lock() + defer d.mu.Unlock() + d.metrics[name] = value +} + +// Timing no-op. +func (d *Diagnostics) Timing(name string, value time.Duration, rate float64) { +} + +// SetLogger Set the logger output type. +func (d *Diagnostics) SetLogger(logger io.Writer) { + d.logOutput = logger +} + +// logger returns a logger that writes to LogOutput. +func (d *Diagnostics) logger() *log.Logger { + return log.New(d.logOutput, "", log.LstdFlags) +} + +// VersionSegments returns the numeric segments of the version as a slice of ints. +func VersionSegments(segments string) []int { + segments = strings.Trim(segments, "v") + segments = strings.Split(segments, "-")[0] + s := strings.Split(segments, ".") + segmentSlice := make([]int, len(s)) + for i, v := range s { + segmentSlice[i], _ = strconv.Atoi(v) + } + return segmentSlice +} diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go new file mode 100644 index 000000000..85cdc56d6 --- /dev/null +++ b/diagnostics/diagnostics_test.go @@ -0,0 +1,143 @@ +package diagnostics_test + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/pilosa/pilosa/diagnostics" +) + +func TestDiagnosticsClient(t *testing.T) { + // Mock server. + server := httptest.NewServer(nil) + defer server.Close() + + // Create a new client. + d := diagnostics.New(server.URL) + d.SetLogger(ioutil.Discard) + defer d.Close() + + dur, _ := time.ParseDuration("123us") + d.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) + d.Count("cc", 1, 1.0) + d.Gauge("gg", 10, 1.0) + d.Histogram("hh", 1, 1.0) + d.Timing("tt", dur, 1.0) + d.Set("ss", "ss", 1.0) + + d1 := d.WithTags("test") + if !reflect.DeepEqual(d, d1) { + t.Fatalf("Diagnostics is a singleton") + } + + if s := d.Tags(); s != nil { + t.Fatalf("No Diagnostics Tags") + } + + data, err := d.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + // Test the recorded metrics, note that some types are skipped. + var eq bool + output1 := []byte(`{"ct":1,"cc":1,"gg":"10","ss":"ss"}`) + if eq, err = compareJSON(data, output1); err != nil { + t.Fatal(err) + } + if !eq { + t.Fatalf("unexpected diagnostics: %+v", string(data)) + } + + // Test the metrics after a flush. + d.Flush() + data, err = d.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + output2 := []byte(`{"gg":"10","ss":"ss","uptime":"0"}`) + if eq, err = compareJSON(data, output2); err != nil { + t.Fatal(err) + } + if !eq { + t.Fatalf("unexpected diagnostics after flush: %+v", string(data)) + } +} + +func TestDiagnosticsVersion_Parse(t *testing.T) { + version := "0.1.1" + vs := diagnostics.VersionSegments(version) + + output := []int{0, 1, 1} + if !reflect.DeepEqual(vs, output) { + t.Fatalf("unexpected version: %+v", vs) + } +} + +func TestDiagnosticsVersion_Compare(t *testing.T) { + d := diagnostics.New("localhost:10101") + defer d.Close() + + version := "0.1.1" + d.SetVersion(version) + + err := d.CompareVersion("1.7.0") + if !strings.Contains(err.Error(), "The latest Major release is") { + t.Fatalf("Expected Major Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.7.0") + if !strings.Contains(err.Error(), "The latest Minor release is") { + t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.1.2") + if !strings.Contains(err.Error(), "There is a new patch relese of Pilosa") { + t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.1.1") + if err != nil { + t.Fatalf("Versions should match") + } +} + +func TestDiagnosticsVersion_Check(t *testing.T) { + // Mock server. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(versionResponse{ + Version: "1.1.1", + }) + })) + defer server.Close() + + // Create a new client. + d := diagnostics.New("localhost:10101") + defer d.Close() + + version := "0.1.1" + d.SetVersion(version) + d.VersionURL = server.URL + + d.CheckVersion() +} + +type versionResponse struct { + Version string `json:"version"` +} + +func compareJSON(a, b []byte) (bool, error) { + var j1, j2 interface{} + if err := json.Unmarshal(a, &j1); err != nil { + return false, err + } + if err := json.Unmarshal(b, &j2); err != nil { + return false, err + } + return reflect.DeepEqual(j1, j2), nil +} From 68ce07b45d281fd3d65ff7a737daafda45c4756a Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:40:55 -0500 Subject: [PATCH 02/48] add Open/Close interface to Stats packages --- stats.go | 36 ++++++++++++++++++++++++++++++++++-- stats_test.go | 2 ++ statsd/statsd.go | 3 +++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/stats.go b/stats.go index 71f0d56b1..998b84511 100644 --- a/stats.go +++ b/stats.go @@ -58,6 +58,12 @@ type StatsClient interface { // SetLogger Set the logger output type SetLogger(logger io.Writer) + + // Starts the service + Open() + + // Closes the client + Close() error } // NopStatsClient represents a client that doesn't do anything. @@ -74,6 +80,8 @@ func (c *nopStatsClient) Histogram(name string, value float64, rate float64) func (c *nopStatsClient) Set(name string, value string, rate float64) {} func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} func (c *nopStatsClient) SetLogger(logger io.Writer) {} +func (c *nopStatsClient) Open() {} +func (c *nopStatsClient) Close() error { return nil } // ExpvarStatsClient writes stats out to expvars. type ExpvarStatsClient struct { @@ -145,10 +153,16 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 c.mu.Unlock() } -// SetLogger has no logger +// SetLogger has no logger. func (c *ExpvarStatsClient) SetLogger(logger io.Writer) { } +// Open no-op. +func (c *ExpvarStatsClient) Open() {} + +// Close no-op. +func (c *ExpvarStatsClient) Close() error { return nil } + // MultiStatsClient joins multiple stats clients together. type MultiStatsClient []StatsClient @@ -211,13 +225,31 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) } } -// SetLogger Sets the StatsD logger output type +// SetLogger Sets the StatsD logger output type. func (a MultiStatsClient) SetLogger(logger io.Writer) { for _, c := range a { c.SetLogger(logger) } } +// Open starts the stat service. +func (a MultiStatsClient) Open() { + for _, c := range a { + c.Open() + } +} + +// Close shuts down the stats clients. +func (a MultiStatsClient) Close() error { + for _, c := range a { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + // UnionStringSlice returns a sorted set of tags which combine a & b. func UnionStringSlice(a, b []string) []string { // Sort both sets first. diff --git a/stats_test.go b/stats_test.go index 7ed679f21..07254a702 100644 --- a/stats_test.go +++ b/stats_test.go @@ -344,3 +344,5 @@ func (c *MockStats) Histogram(name string, value float64, rate float64) {} func (c *MockStats) Set(name string, value string, rate float64) {} func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} func (c *MockStats) SetLogger(logger io.Writer) {} +func (c *MockStats) Open() {} +func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go index e55b190ad..d46dd637f 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -58,6 +58,9 @@ func NewStatsClient(host string) (*StatsClient, error) { }, nil } +// Open no-op +func (c *StatsClient) Open() {} + // Close closes the connection to the agent. func (c *StatsClient) Close() error { return c.client.Close() From 566f4116f17e5c223ef02ec90d4cc2077646f184 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:21 -0500 Subject: [PATCH 03/48] Using MultiStatsClient add Diagnostics --- server/server.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/server.go b/server/server.go index 157fc69f3..ba60e8923 100644 --- a/server/server.go +++ b/server/server.go @@ -31,6 +31,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -41,7 +42,8 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" + DefaultDataDir = "~/.pilosa" + DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" ) // Command represents the state of the pilosa server command. @@ -222,12 +224,20 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { + ms := make(pilosa.MultiStatsClient, 1) + d := diagnostics.New(DefaultDiagnosticServer) + d.SetVersion(pilosa.Version) + ms[0] = d + switch name { case "expvar": - return pilosa.NewExpvarStatsClient(), nil + ms = append(ms, pilosa.NewExpvarStatsClient()) case "statsd": - return statsd.NewStatsClient(host) - default: - return pilosa.NopStatsClient, nil + r, err := statsd.NewStatsClient(host) + if err != nil { + return nil, err + } + ms = append(ms, r) } + return ms, nil } From e87e7fda68b995148eef060e9419bcc3ebfb3bf9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:35 -0500 Subject: [PATCH 04/48] Add some new Diagnostics metrics --- holder.go | 3 +++ server.go | 26 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/holder.go b/holder.go index 0cf5f6901..e358739f5 100644 --- a/holder.go +++ b/holder.go @@ -123,11 +123,14 @@ func (h *Holder) Open() error { h.wg.Add(1) go func() { defer h.wg.Done(); h.monitorCacheFlush() }() + h.Stats.Open() return nil } // Close closes all open fragments. func (h *Holder) Close() error { + h.Stats.Close() + // Notify goroutines of closing and wait for completion. close(h.closing) h.wg.Wait() diff --git a/server.go b/server.go index 8194278ba..e4d772349 100644 --- a/server.go +++ b/server.go @@ -202,10 +202,10 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } +// Logger returns a logger that writes to LogOutput func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { - t := time.Now() ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() @@ -219,7 +219,7 @@ func (s *Server) monitorAntiEntropy() { case <-ticker.C: s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } - + t := time.Now() s.Logger().Printf("holder sync beginning") // Initialize syncer with local holder and remote client. @@ -237,9 +237,9 @@ func (s *Server) monitorAntiEntropy() { // Record successful sync in log. s.Logger().Printf("holder sync complete") + dif := time.Since(t) + s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } - dif := time.Since(t) - s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. @@ -486,7 +486,13 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) { // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - // Disable metrics when poll interval is zero + s.Holder.Stats.Set("Host", s.Host, 1.0) + s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) + s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) + s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) + // TODO should we force this to run for diagnostics? + + // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return } @@ -506,18 +512,18 @@ func (s *Server) monitorRuntime() { case <-s.closing: return case <-gcn.AfterGC(): - // GC just ran + // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: } - // Record the number of go routines + // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - // Open File handles + // Open File handles. s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) - // Runtime memory metrics + // Runtime memory metrics. runtime.ReadMemStats(&m) s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) @@ -527,7 +533,7 @@ func (s *Server) monitorRuntime() { } } -// CountOpenFiles on opperating systems that support lsof +// CountOpenFiles on operating systems that support lsof. func CountOpenFiles() int { count := 0 From e91817701862b7c9a3f92f2529c4003a4ba50ed3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Sep 2017 17:58:13 -0500 Subject: [PATCH 05/48] Add basic diagnostics benchmark --- diagnostics/diagnostics_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 85cdc56d6..1a811df9c 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "runtime" "strings" "testing" "time" @@ -141,3 +142,26 @@ func compareJSON(a, b []byte) (bool, error) { } return reflect.DeepEqual(j1, j2), nil } + +func BenchmarkDiagnostics(b *testing.B) { + // Mock server. + server := httptest.NewServer(nil) + defer server.Close() + + // Create a new client. + d := diagnostics.New(server.URL) + d.SetLogger(ioutil.Discard) + defer d.Close() + + prev := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(prev) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + d.Count("cc", 1, 1.0) + d.Gauge("gg", 10, 1.0) + } + }) +} From 0cb0a79becf92dbf839b6fe7f0b9d15adcdc6de2 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 10 Oct 2017 15:40:19 -0500 Subject: [PATCH 06/48] Remove row and column labels --- webui/assets/main.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index b3f48ebe3..6a4415a5c 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -344,12 +344,11 @@ function render_status(status) { tbody = document.createElement("tbody") table.appendChild(tbody) var caption = document.createElement("caption") - caption.innerHTML = indexes[n]["Name"] + " (Column Label: " + indexes[n]["Meta"]["ColumnLabel"] + ")" + caption.innerHTML = indexes[n]["Name"] table.appendChild(caption) var header = document.createElement('tr') markup = `Name - Row Label Cache Type Cache Size` header.innerHTML = markup @@ -360,7 +359,6 @@ function render_status(status) { for(var m=0; m${frames[m]["Name"]} - ${frames[m]["Meta"]["RowLabel"]} ${frames[m]["Meta"]["CacheType"]} ${frames[m]["Meta"]["CacheSize"]}` tbody.appendChild(row) From 9876b51fc49230c923578ad1b542825309737b40 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 16 Oct 2017 12:08:48 -0500 Subject: [PATCH 07/48] simplifying the diagnostics client. Using circuit breaker to manage the diagnostics http connection. --- diagnostics/diagnostics.go | 142 +++++++------------------------- diagnostics/diagnostics_test.go | 31 ++----- server.go | 70 +++++++++++++--- server/server.go | 20 ++--- 4 files changed, 105 insertions(+), 158 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index fc78b94cb..761e7adca 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -13,16 +13,14 @@ import ( "sync" "time" - "github.com/pilosa/pilosa" + "github.com/sony/gobreaker" ) -// TODO: white list of statsd metrics to use // TODO: unique Cluster ID -// TODO: how should this be disabled, config // Default interval to sync diagnostics metrics. const ( - DefaultDiagnosticsInterval = 10 * time.Second + DefaultDiagnosticsInterval = 1 * time.Hour DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" ) @@ -42,17 +40,20 @@ type Diagnostics struct { startTime int64 start time.Time - counts map[string]int64 - metrics map[string]string + metrics map[string]interface{} client *http.Client interval time.Duration + cb *gobreaker.CircuitBreaker logOutput io.Writer } // New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". func New(host string) *Diagnostics { + var st gobreaker.Settings + st.Timeout = DefaultDiagnosticsInterval * 2 + return &Diagnostics{ closing: make(chan struct{}), host: host, @@ -60,17 +61,17 @@ func New(host string) *Diagnostics { startTime: time.Now().Unix(), start: time.Now(), client: http.DefaultClient, - counts: make(map[string]int64), - metrics: make(map[string]string), + metrics: make(map[string]interface{}), interval: DefaultDiagnosticsInterval, logOutput: ioutil.Discard, + cb: gobreaker.NewCircuitBreaker(st), } } // SetVersion of locally running Pilosa Cluster to check against master. func (d *Diagnostics) SetVersion(v string) { d.version = v - d.Set("Version", v, 1.0) + d.Set("Version", v) } // schedule start the diagnostics service ticker. @@ -92,35 +93,28 @@ func (d *Diagnostics) schedule() { // Flush sends the current metrics. func (d *Diagnostics) Flush() error { d.mu.Lock() - d.metrics["uptime"] = strconv.FormatInt((time.Now().Unix() - d.startTime), 10) - - buf, _ := d.MarshalJSON() - d.Reset() + d.metrics["uptime"] = (time.Now().Unix() - d.startTime) + buf, _ := d.Encode() d.mu.Unlock() - // d.logger().Println(string(buf)) - req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - resp, err := d.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() + _, err := d.cb.Execute(func() (interface{}, error) { + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() - // TODO verify response - // Read response into buffer. - // body, err := ioutil.ReadAll(resp.Body) - // if err != nil { - // return err - // } + // TODO verify response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, nil + }) - // TODO circuit breaker - return nil -} - -// Reset clears the incremented metrics. -func (d *Diagnostics) Reset() { - d.counts = make(map[string]int64) + return err } // Open starts the diagnostics metric go routine. @@ -175,90 +169,18 @@ func (d *Diagnostics) CompareVersion(value string) error { return nil } -// MarshalJSON custom marshall string and int maps together. -func (d *Diagnostics) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString("{") - length := len(d.counts) - count := 0 - - for key, value := range d.counts { - jsonValue, err := json.Marshal(value) - if err != nil { - return nil, err - } - buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) - count++ - if count < length { - buffer.WriteString(",") - } - } - if length > 0 { - buffer.WriteString(",") - } - length = len(d.metrics) - count = 0 - for key, value := range d.metrics { - jsonValue, err := json.Marshal(value) - if err != nil { - return nil, err - } - buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) - count++ - if count < length { - buffer.WriteString(",") - } - } - - buffer.WriteString("}") - return buffer.Bytes(), nil -} - -// Stats interface implementation. - -// Tags no-op. -func (d *Diagnostics) Tags() []string { - return nil -} - -// WithTags no-op. -func (d *Diagnostics) WithTags(tags ...string) pilosa.StatsClient { - return d -} - -// Count tracks the number of times something occurs per diagnostic period. -func (d *Diagnostics) Count(name string, value int64, rate float64) { - d.mu.Lock() - defer d.mu.Unlock() - d.counts[name] += value -} - -// CountWithCustomTags Tracks the number of times something occurs per diagnostic period. -func (d *Diagnostics) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - d.mu.Lock() - defer d.mu.Unlock() - d.counts[name] += value -} - -// Gauge records the value of a metric. -func (d *Diagnostics) Gauge(name string, value float64, rate float64) { - d.Set(name, strconv.FormatFloat(value, 'f', -1, 64), rate) -} - -// Histogram is a no-op. -func (d *Diagnostics) Histogram(name string, value float64, rate float64) { +// Encode metrics maps into the json message format +func (d *Diagnostics) Encode() ([]byte, error) { + return json.Marshal(d.metrics) } // Set adds a key value metric. -func (d *Diagnostics) Set(name string, value string, rate float64) { +func (d *Diagnostics) Set(name string, value interface{}) { d.mu.Lock() defer d.mu.Unlock() d.metrics[name] = value } -// Timing no-op. -func (d *Diagnostics) Timing(name string, value time.Duration, rate float64) { -} - // SetLogger Set the logger output type. func (d *Diagnostics) SetLogger(logger io.Writer) { d.logOutput = logger diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 1a811df9c..d780a276f 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -9,7 +9,6 @@ import ( "runtime" "strings" "testing" - "time" "github.com/pilosa/pilosa/diagnostics" ) @@ -24,31 +23,17 @@ func TestDiagnosticsClient(t *testing.T) { d.SetLogger(ioutil.Discard) defer d.Close() - dur, _ := time.ParseDuration("123us") - d.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - d.Count("cc", 1, 1.0) - d.Gauge("gg", 10, 1.0) - d.Histogram("hh", 1, 1.0) - d.Timing("tt", dur, 1.0) - d.Set("ss", "ss", 1.0) + d.Set("gg", 10) + d.Set("ss", "ss") - d1 := d.WithTags("test") - if !reflect.DeepEqual(d, d1) { - t.Fatalf("Diagnostics is a singleton") - } - - if s := d.Tags(); s != nil { - t.Fatalf("No Diagnostics Tags") - } - - data, err := d.MarshalJSON() + data, err := d.Encode() if err != nil { t.Fatal(err) } // Test the recorded metrics, note that some types are skipped. var eq bool - output1 := []byte(`{"ct":1,"cc":1,"gg":"10","ss":"ss"}`) + output1 := []byte(`{"gg":10,"ss":"ss"}`) if eq, err = compareJSON(data, output1); err != nil { t.Fatal(err) } @@ -58,12 +43,12 @@ func TestDiagnosticsClient(t *testing.T) { // Test the metrics after a flush. d.Flush() - data, err = d.MarshalJSON() + data, err = d.Encode() if err != nil { t.Fatal(err) } - output2 := []byte(`{"gg":"10","ss":"ss","uptime":"0"}`) + output2 := []byte(`{"gg":10,"ss":"ss","uptime":0}`) if eq, err = compareJSON(data, output2); err != nil { t.Fatal(err) } @@ -160,8 +145,8 @@ func BenchmarkDiagnostics(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - d.Count("cc", 1, 1.0) - d.Gauge("gg", 10, 1.0) + d.Set("cc", 1) + d.Set("gg", "test") } }) } diff --git a/server.go b/server.go index e4d772349..3813a6822 100644 --- a/server.go +++ b/server.go @@ -33,6 +33,7 @@ import ( "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" ) @@ -40,6 +41,7 @@ import ( const ( DefaultAntiEntropyInterval = 10 * time.Minute DefaultPollingInterval = 60 * time.Second + DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Server represents a holder wrapped by a running HTTP server. @@ -58,14 +60,16 @@ type Server struct { // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". - Network string - Host string - Cluster *Cluster + Network string + Host string + Cluster *Cluster + diagnostics *diagnostics.Diagnostics // Background monitoring intervals. AntiEntropyInterval time.Duration PollingInterval time.Duration MetricInterval time.Duration + DiagnosticInterval time.Duration // Misc options. MaxWritesPerRequest int @@ -82,12 +86,14 @@ func NewServer() *Server { Handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, + diagnostics: diagnostics.New(DefaultDiagnosticServer), Network: "tcp", AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, + DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval LogOutput: os.Stderr, } @@ -170,10 +176,11 @@ func (s *Server) Open() error { }() // Start background monitoring. - s.wg.Add(3) + s.wg.Add(4) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() go func() { defer s.wg.Done(); s.monitorMaxSlices() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() + go func() { defer s.wg.Done(); s.monitorDiagnostics() }() return nil } @@ -484,14 +491,57 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) { return pb.MaxSlices, nil } +// monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info. +func (s *Server) monitorDiagnostics() { + if s.DiagnosticInterval <= 0 { + return + } + + s.diagnostics.SetLogger(s.LogOutput) + s.diagnostics.SetVersion(Version) + s.diagnostics.Set("Host", s.Host) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) + s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) + s.diagnostics.Set("NumCPU", runtime.NumCPU()) + // TODO: unique cluster ID + + ticker := time.NewTicker(s.DiagnosticInterval) + defer ticker.Stop() + + for { + // Wait for tick or a close. + select { + case <-s.closing: + return + case <-ticker.C: + numFrames := 0 + numSlices := uint64(0) + for _, index := range s.Holder.Indexes() { + numSlices += index.MaxSlice() + 1 + for _, f := range index.Frames() { + numFrames++ + if f.rangeEnabled { + s.diagnostics.Set("BSIEnabled", true) + } + if f.timeQuantum != "" { + s.diagnostics.Set("TimeQuantumEnabled", true) + } + } + } + + s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) + s.diagnostics.Set("NumFrames", numFrames) + s.diagnostics.Set("NumSlices", numSlices) + s.diagnostics.Set("OpenFiles", CountOpenFiles()) + s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) + s.diagnostics.CheckVersion() + s.diagnostics.Flush() + } + } +} + // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - s.Holder.Stats.Set("Host", s.Host, 1.0) - s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) - s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) - s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) - // TODO should we force this to run for diagnostics? - // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return diff --git a/server/server.go b/server/server.go index ba60e8923..157fc69f3 100644 --- a/server/server.go +++ b/server/server.go @@ -31,7 +31,6 @@ import ( "time" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -42,8 +41,7 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" + DefaultDataDir = "~/.pilosa" ) // Command represents the state of the pilosa server command. @@ -224,20 +222,12 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { - ms := make(pilosa.MultiStatsClient, 1) - d := diagnostics.New(DefaultDiagnosticServer) - d.SetVersion(pilosa.Version) - ms[0] = d - switch name { case "expvar": - ms = append(ms, pilosa.NewExpvarStatsClient()) + return pilosa.NewExpvarStatsClient(), nil case "statsd": - r, err := statsd.NewStatsClient(host) - if err != nil { - return nil, err - } - ms = append(ms, r) + return statsd.NewStatsClient(host) + default: + return pilosa.NopStatsClient, nil } - return ms, nil } From 32cae0a2002a85709963b78c7018108fcfc593a8 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:39:57 -0500 Subject: [PATCH 08/48] Diagnostics package --- diagnostics/diagnostics.go | 282 ++++++++++++++++++++++++++++++++ diagnostics/diagnostics_test.go | 143 ++++++++++++++++ 2 files changed, 425 insertions(+) create mode 100644 diagnostics/diagnostics.go create mode 100644 diagnostics/diagnostics_test.go diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go new file mode 100644 index 000000000..fc78b94cb --- /dev/null +++ b/diagnostics/diagnostics.go @@ -0,0 +1,282 @@ +package diagnostics + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/pilosa/pilosa" +) + +// TODO: white list of statsd metrics to use +// TODO: unique Cluster ID +// TODO: how should this be disabled, config + +// Default interval to sync diagnostics metrics. +const ( + DefaultDiagnosticsInterval = 10 * time.Second + DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" +) + +type versionResponse struct { + Version string `json:"version"` + Message string `json:"message"` +} + +// Diagnostics represents a client to the Pilosa cluster. +type Diagnostics struct { + mu sync.Mutex + wg sync.WaitGroup + closing chan struct{} + host string + VersionURL string + version string + startTime int64 + start time.Time + + counts map[string]int64 + metrics map[string]string + + client *http.Client + interval time.Duration + + logOutput io.Writer +} + +// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". +func New(host string) *Diagnostics { + return &Diagnostics{ + closing: make(chan struct{}), + host: host, + VersionURL: DefaultVersionCheckURL, + startTime: time.Now().Unix(), + start: time.Now(), + client: http.DefaultClient, + counts: make(map[string]int64), + metrics: make(map[string]string), + interval: DefaultDiagnosticsInterval, + logOutput: ioutil.Discard, + } +} + +// SetVersion of locally running Pilosa Cluster to check against master. +func (d *Diagnostics) SetVersion(v string) { + d.version = v + d.Set("Version", v, 1.0) +} + +// schedule start the diagnostics service ticker. +func (d *Diagnostics) schedule() { + ticker := time.NewTicker(d.interval) + defer ticker.Stop() + + for { + select { + case <-d.closing: + return + case <-ticker.C: + d.CheckVersion() + d.Flush() + } + } +} + +// Flush sends the current metrics. +func (d *Diagnostics) Flush() error { + d.mu.Lock() + d.metrics["uptime"] = strconv.FormatInt((time.Now().Unix() - d.startTime), 10) + + buf, _ := d.MarshalJSON() + d.Reset() + d.mu.Unlock() + + // d.logger().Println(string(buf)) + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // TODO verify response + // Read response into buffer. + // body, err := ioutil.ReadAll(resp.Body) + // if err != nil { + // return err + // } + + // TODO circuit breaker + return nil +} + +// Reset clears the incremented metrics. +func (d *Diagnostics) Reset() { + d.counts = make(map[string]int64) +} + +// Open starts the diagnostics metric go routine. +func (d *Diagnostics) Open() { + d.wg.Add(1) + go func() { defer d.wg.Done(); d.schedule() }() +} + +// Close notify goroutine to stop. +func (d *Diagnostics) Close() error { + close(d.closing) + d.wg.Wait() + return nil +} + +// CheckVersion of the local build against Pilosa master. +func (d *Diagnostics) CheckVersion() error { + var rsp versionResponse + req, err := http.NewRequest("GET", d.VersionURL, nil) + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return fmt.Errorf("json decode: %s", err) + } + + if err := d.CompareVersion(rsp.Version); err != nil { + d.logger().Printf("%s\n", err.Error()) + } + + return nil +} + +// CompareVersion check version strings. +func (d *Diagnostics) CompareVersion(value string) error { + currentVersion := VersionSegments(value) + localVersion := VersionSegments(d.version) + + if localVersion[0] < currentVersion[0] { //Major + return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Major release is %s", d.version, value) + } else if localVersion[1] < currentVersion[1] { // Minor + return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Minor release is %s", d.version, value) + } else if localVersion[2] < currentVersion[2] { // Patch + return fmt.Errorf("There is a new patch relese of Pilosa availbale: %s", value) + } + + return nil +} + +// MarshalJSON custom marshall string and int maps together. +func (d *Diagnostics) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString("{") + length := len(d.counts) + count := 0 + + for key, value := range d.counts { + jsonValue, err := json.Marshal(value) + if err != nil { + return nil, err + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) + count++ + if count < length { + buffer.WriteString(",") + } + } + if length > 0 { + buffer.WriteString(",") + } + length = len(d.metrics) + count = 0 + for key, value := range d.metrics { + jsonValue, err := json.Marshal(value) + if err != nil { + return nil, err + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) + count++ + if count < length { + buffer.WriteString(",") + } + } + + buffer.WriteString("}") + return buffer.Bytes(), nil +} + +// Stats interface implementation. + +// Tags no-op. +func (d *Diagnostics) Tags() []string { + return nil +} + +// WithTags no-op. +func (d *Diagnostics) WithTags(tags ...string) pilosa.StatsClient { + return d +} + +// Count tracks the number of times something occurs per diagnostic period. +func (d *Diagnostics) Count(name string, value int64, rate float64) { + d.mu.Lock() + defer d.mu.Unlock() + d.counts[name] += value +} + +// CountWithCustomTags Tracks the number of times something occurs per diagnostic period. +func (d *Diagnostics) CountWithCustomTags(name string, value int64, rate float64, tags []string) { + d.mu.Lock() + defer d.mu.Unlock() + d.counts[name] += value +} + +// Gauge records the value of a metric. +func (d *Diagnostics) Gauge(name string, value float64, rate float64) { + d.Set(name, strconv.FormatFloat(value, 'f', -1, 64), rate) +} + +// Histogram is a no-op. +func (d *Diagnostics) Histogram(name string, value float64, rate float64) { +} + +// Set adds a key value metric. +func (d *Diagnostics) Set(name string, value string, rate float64) { + d.mu.Lock() + defer d.mu.Unlock() + d.metrics[name] = value +} + +// Timing no-op. +func (d *Diagnostics) Timing(name string, value time.Duration, rate float64) { +} + +// SetLogger Set the logger output type. +func (d *Diagnostics) SetLogger(logger io.Writer) { + d.logOutput = logger +} + +// logger returns a logger that writes to LogOutput. +func (d *Diagnostics) logger() *log.Logger { + return log.New(d.logOutput, "", log.LstdFlags) +} + +// VersionSegments returns the numeric segments of the version as a slice of ints. +func VersionSegments(segments string) []int { + segments = strings.Trim(segments, "v") + segments = strings.Split(segments, "-")[0] + s := strings.Split(segments, ".") + segmentSlice := make([]int, len(s)) + for i, v := range s { + segmentSlice[i], _ = strconv.Atoi(v) + } + return segmentSlice +} diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go new file mode 100644 index 000000000..85cdc56d6 --- /dev/null +++ b/diagnostics/diagnostics_test.go @@ -0,0 +1,143 @@ +package diagnostics_test + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/pilosa/pilosa/diagnostics" +) + +func TestDiagnosticsClient(t *testing.T) { + // Mock server. + server := httptest.NewServer(nil) + defer server.Close() + + // Create a new client. + d := diagnostics.New(server.URL) + d.SetLogger(ioutil.Discard) + defer d.Close() + + dur, _ := time.ParseDuration("123us") + d.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) + d.Count("cc", 1, 1.0) + d.Gauge("gg", 10, 1.0) + d.Histogram("hh", 1, 1.0) + d.Timing("tt", dur, 1.0) + d.Set("ss", "ss", 1.0) + + d1 := d.WithTags("test") + if !reflect.DeepEqual(d, d1) { + t.Fatalf("Diagnostics is a singleton") + } + + if s := d.Tags(); s != nil { + t.Fatalf("No Diagnostics Tags") + } + + data, err := d.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + // Test the recorded metrics, note that some types are skipped. + var eq bool + output1 := []byte(`{"ct":1,"cc":1,"gg":"10","ss":"ss"}`) + if eq, err = compareJSON(data, output1); err != nil { + t.Fatal(err) + } + if !eq { + t.Fatalf("unexpected diagnostics: %+v", string(data)) + } + + // Test the metrics after a flush. + d.Flush() + data, err = d.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + output2 := []byte(`{"gg":"10","ss":"ss","uptime":"0"}`) + if eq, err = compareJSON(data, output2); err != nil { + t.Fatal(err) + } + if !eq { + t.Fatalf("unexpected diagnostics after flush: %+v", string(data)) + } +} + +func TestDiagnosticsVersion_Parse(t *testing.T) { + version := "0.1.1" + vs := diagnostics.VersionSegments(version) + + output := []int{0, 1, 1} + if !reflect.DeepEqual(vs, output) { + t.Fatalf("unexpected version: %+v", vs) + } +} + +func TestDiagnosticsVersion_Compare(t *testing.T) { + d := diagnostics.New("localhost:10101") + defer d.Close() + + version := "0.1.1" + d.SetVersion(version) + + err := d.CompareVersion("1.7.0") + if !strings.Contains(err.Error(), "The latest Major release is") { + t.Fatalf("Expected Major Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.7.0") + if !strings.Contains(err.Error(), "The latest Minor release is") { + t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.1.2") + if !strings.Contains(err.Error(), "There is a new patch relese of Pilosa") { + t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.1.1") + if err != nil { + t.Fatalf("Versions should match") + } +} + +func TestDiagnosticsVersion_Check(t *testing.T) { + // Mock server. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(versionResponse{ + Version: "1.1.1", + }) + })) + defer server.Close() + + // Create a new client. + d := diagnostics.New("localhost:10101") + defer d.Close() + + version := "0.1.1" + d.SetVersion(version) + d.VersionURL = server.URL + + d.CheckVersion() +} + +type versionResponse struct { + Version string `json:"version"` +} + +func compareJSON(a, b []byte) (bool, error) { + var j1, j2 interface{} + if err := json.Unmarshal(a, &j1); err != nil { + return false, err + } + if err := json.Unmarshal(b, &j2); err != nil { + return false, err + } + return reflect.DeepEqual(j1, j2), nil +} From c80b959d0dc6e9886b02bcc0eb6c28597909d0c8 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:40:55 -0500 Subject: [PATCH 09/48] add Open/Close interface to Stats packages --- stats.go | 36 ++++++++++++++++++++++++++++++++++-- stats_test.go | 2 ++ statsd/statsd.go | 3 +++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/stats.go b/stats.go index 71f0d56b1..998b84511 100644 --- a/stats.go +++ b/stats.go @@ -58,6 +58,12 @@ type StatsClient interface { // SetLogger Set the logger output type SetLogger(logger io.Writer) + + // Starts the service + Open() + + // Closes the client + Close() error } // NopStatsClient represents a client that doesn't do anything. @@ -74,6 +80,8 @@ func (c *nopStatsClient) Histogram(name string, value float64, rate float64) func (c *nopStatsClient) Set(name string, value string, rate float64) {} func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} func (c *nopStatsClient) SetLogger(logger io.Writer) {} +func (c *nopStatsClient) Open() {} +func (c *nopStatsClient) Close() error { return nil } // ExpvarStatsClient writes stats out to expvars. type ExpvarStatsClient struct { @@ -145,10 +153,16 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 c.mu.Unlock() } -// SetLogger has no logger +// SetLogger has no logger. func (c *ExpvarStatsClient) SetLogger(logger io.Writer) { } +// Open no-op. +func (c *ExpvarStatsClient) Open() {} + +// Close no-op. +func (c *ExpvarStatsClient) Close() error { return nil } + // MultiStatsClient joins multiple stats clients together. type MultiStatsClient []StatsClient @@ -211,13 +225,31 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) } } -// SetLogger Sets the StatsD logger output type +// SetLogger Sets the StatsD logger output type. func (a MultiStatsClient) SetLogger(logger io.Writer) { for _, c := range a { c.SetLogger(logger) } } +// Open starts the stat service. +func (a MultiStatsClient) Open() { + for _, c := range a { + c.Open() + } +} + +// Close shuts down the stats clients. +func (a MultiStatsClient) Close() error { + for _, c := range a { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + // UnionStringSlice returns a sorted set of tags which combine a & b. func UnionStringSlice(a, b []string) []string { // Sort both sets first. diff --git a/stats_test.go b/stats_test.go index 7ed679f21..07254a702 100644 --- a/stats_test.go +++ b/stats_test.go @@ -344,3 +344,5 @@ func (c *MockStats) Histogram(name string, value float64, rate float64) {} func (c *MockStats) Set(name string, value string, rate float64) {} func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} func (c *MockStats) SetLogger(logger io.Writer) {} +func (c *MockStats) Open() {} +func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go index e55b190ad..d46dd637f 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -58,6 +58,9 @@ func NewStatsClient(host string) (*StatsClient, error) { }, nil } +// Open no-op +func (c *StatsClient) Open() {} + // Close closes the connection to the agent. func (c *StatsClient) Close() error { return c.client.Close() From 539c4bc14eb7c67cfbc3a88c6380b566216ebc4b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:21 -0500 Subject: [PATCH 10/48] Using MultiStatsClient add Diagnostics --- server/server.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/server.go b/server/server.go index d1063f306..0ce1680c2 100644 --- a/server/server.go +++ b/server/server.go @@ -31,6 +31,7 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -41,7 +42,8 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" + DefaultDataDir = "~/.pilosa" + DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" ) // Command represents the state of the pilosa server command. @@ -245,12 +247,20 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { + ms := make(pilosa.MultiStatsClient, 1) + d := diagnostics.New(DefaultDiagnosticServer) + d.SetVersion(pilosa.Version) + ms[0] = d + switch name { case "expvar": - return pilosa.NewExpvarStatsClient(), nil + ms = append(ms, pilosa.NewExpvarStatsClient()) case "statsd": - return statsd.NewStatsClient(host) - default: - return pilosa.NopStatsClient, nil + r, err := statsd.NewStatsClient(host) + if err != nil { + return nil, err + } + ms = append(ms, r) } + return ms, nil } From ec4d2a75c9b263d1e7a2b510efde11e6cd378054 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:35 -0500 Subject: [PATCH 11/48] Add some new Diagnostics metrics --- holder.go | 3 +++ server.go | 26 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/holder.go b/holder.go index f3713faa2..d85f16814 100644 --- a/holder.go +++ b/holder.go @@ -123,11 +123,14 @@ func (h *Holder) Open() error { h.wg.Add(1) go func() { defer h.wg.Done(); h.monitorCacheFlush() }() + h.Stats.Open() return nil } // Close closes all open fragments. func (h *Holder) Close() error { + h.Stats.Close() + // Notify goroutines of closing and wait for completion. close(h.closing) h.wg.Wait() diff --git a/server.go b/server.go index 773cf9025..596f3be86 100644 --- a/server.go +++ b/server.go @@ -223,10 +223,10 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } +// Logger returns a logger that writes to LogOutput func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { - t := time.Now() ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() @@ -240,7 +240,7 @@ func (s *Server) monitorAntiEntropy() { case <-ticker.C: s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } - + t := time.Now() s.Logger().Printf("holder sync beginning") // Initialize syncer with local holder and remote client. @@ -259,9 +259,9 @@ func (s *Server) monitorAntiEntropy() { // Record successful sync in log. s.Logger().Printf("holder sync complete") + dif := time.Since(t) + s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } - dif := time.Since(t) - s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. @@ -509,7 +509,13 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - // Disable metrics when poll interval is zero + s.Holder.Stats.Set("Host", s.Host, 1.0) + s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) + s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) + s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) + // TODO should we force this to run for diagnostics? + + // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return } @@ -529,18 +535,18 @@ func (s *Server) monitorRuntime() { case <-s.closing: return case <-gcn.AfterGC(): - // GC just ran + // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: } - // Record the number of go routines + // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - // Open File handles + // Open File handles. s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) - // Runtime memory metrics + // Runtime memory metrics. runtime.ReadMemStats(&m) s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) @@ -558,7 +564,7 @@ func (s *Server) createDefaultClient() { s.defaultClient = &http.Client{Transport: transport} } -// CountOpenFiles on opperating systems that support lsof +// CountOpenFiles on operating systems that support lsof. func CountOpenFiles() int { count := 0 From d91865465cbc970694de8d6df14e95d18edf5f98 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Sep 2017 17:58:13 -0500 Subject: [PATCH 12/48] Add basic diagnostics benchmark --- diagnostics/diagnostics_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 85cdc56d6..1a811df9c 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "runtime" "strings" "testing" "time" @@ -141,3 +142,26 @@ func compareJSON(a, b []byte) (bool, error) { } return reflect.DeepEqual(j1, j2), nil } + +func BenchmarkDiagnostics(b *testing.B) { + // Mock server. + server := httptest.NewServer(nil) + defer server.Close() + + // Create a new client. + d := diagnostics.New(server.URL) + d.SetLogger(ioutil.Discard) + defer d.Close() + + prev := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(prev) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + d.Count("cc", 1, 1.0) + d.Gauge("gg", 10, 1.0) + } + }) +} From 64fd7d34299898b4ba22865510820e1249e6c49a Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 16 Oct 2017 12:08:48 -0500 Subject: [PATCH 13/48] simplifying the diagnostics client. Using circuit breaker to manage the diagnostics http connection. --- diagnostics/diagnostics.go | 142 +++++++------------------------- diagnostics/diagnostics_test.go | 31 ++----- server.go | 70 +++++++++++++--- server/server.go | 20 ++--- 4 files changed, 105 insertions(+), 158 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index fc78b94cb..761e7adca 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -13,16 +13,14 @@ import ( "sync" "time" - "github.com/pilosa/pilosa" + "github.com/sony/gobreaker" ) -// TODO: white list of statsd metrics to use // TODO: unique Cluster ID -// TODO: how should this be disabled, config // Default interval to sync diagnostics metrics. const ( - DefaultDiagnosticsInterval = 10 * time.Second + DefaultDiagnosticsInterval = 1 * time.Hour DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" ) @@ -42,17 +40,20 @@ type Diagnostics struct { startTime int64 start time.Time - counts map[string]int64 - metrics map[string]string + metrics map[string]interface{} client *http.Client interval time.Duration + cb *gobreaker.CircuitBreaker logOutput io.Writer } // New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". func New(host string) *Diagnostics { + var st gobreaker.Settings + st.Timeout = DefaultDiagnosticsInterval * 2 + return &Diagnostics{ closing: make(chan struct{}), host: host, @@ -60,17 +61,17 @@ func New(host string) *Diagnostics { startTime: time.Now().Unix(), start: time.Now(), client: http.DefaultClient, - counts: make(map[string]int64), - metrics: make(map[string]string), + metrics: make(map[string]interface{}), interval: DefaultDiagnosticsInterval, logOutput: ioutil.Discard, + cb: gobreaker.NewCircuitBreaker(st), } } // SetVersion of locally running Pilosa Cluster to check against master. func (d *Diagnostics) SetVersion(v string) { d.version = v - d.Set("Version", v, 1.0) + d.Set("Version", v) } // schedule start the diagnostics service ticker. @@ -92,35 +93,28 @@ func (d *Diagnostics) schedule() { // Flush sends the current metrics. func (d *Diagnostics) Flush() error { d.mu.Lock() - d.metrics["uptime"] = strconv.FormatInt((time.Now().Unix() - d.startTime), 10) - - buf, _ := d.MarshalJSON() - d.Reset() + d.metrics["uptime"] = (time.Now().Unix() - d.startTime) + buf, _ := d.Encode() d.mu.Unlock() - // d.logger().Println(string(buf)) - req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - resp, err := d.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() + _, err := d.cb.Execute(func() (interface{}, error) { + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() - // TODO verify response - // Read response into buffer. - // body, err := ioutil.ReadAll(resp.Body) - // if err != nil { - // return err - // } + // TODO verify response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, nil + }) - // TODO circuit breaker - return nil -} - -// Reset clears the incremented metrics. -func (d *Diagnostics) Reset() { - d.counts = make(map[string]int64) + return err } // Open starts the diagnostics metric go routine. @@ -175,90 +169,18 @@ func (d *Diagnostics) CompareVersion(value string) error { return nil } -// MarshalJSON custom marshall string and int maps together. -func (d *Diagnostics) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString("{") - length := len(d.counts) - count := 0 - - for key, value := range d.counts { - jsonValue, err := json.Marshal(value) - if err != nil { - return nil, err - } - buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) - count++ - if count < length { - buffer.WriteString(",") - } - } - if length > 0 { - buffer.WriteString(",") - } - length = len(d.metrics) - count = 0 - for key, value := range d.metrics { - jsonValue, err := json.Marshal(value) - if err != nil { - return nil, err - } - buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) - count++ - if count < length { - buffer.WriteString(",") - } - } - - buffer.WriteString("}") - return buffer.Bytes(), nil -} - -// Stats interface implementation. - -// Tags no-op. -func (d *Diagnostics) Tags() []string { - return nil -} - -// WithTags no-op. -func (d *Diagnostics) WithTags(tags ...string) pilosa.StatsClient { - return d -} - -// Count tracks the number of times something occurs per diagnostic period. -func (d *Diagnostics) Count(name string, value int64, rate float64) { - d.mu.Lock() - defer d.mu.Unlock() - d.counts[name] += value -} - -// CountWithCustomTags Tracks the number of times something occurs per diagnostic period. -func (d *Diagnostics) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - d.mu.Lock() - defer d.mu.Unlock() - d.counts[name] += value -} - -// Gauge records the value of a metric. -func (d *Diagnostics) Gauge(name string, value float64, rate float64) { - d.Set(name, strconv.FormatFloat(value, 'f', -1, 64), rate) -} - -// Histogram is a no-op. -func (d *Diagnostics) Histogram(name string, value float64, rate float64) { +// Encode metrics maps into the json message format +func (d *Diagnostics) Encode() ([]byte, error) { + return json.Marshal(d.metrics) } // Set adds a key value metric. -func (d *Diagnostics) Set(name string, value string, rate float64) { +func (d *Diagnostics) Set(name string, value interface{}) { d.mu.Lock() defer d.mu.Unlock() d.metrics[name] = value } -// Timing no-op. -func (d *Diagnostics) Timing(name string, value time.Duration, rate float64) { -} - // SetLogger Set the logger output type. func (d *Diagnostics) SetLogger(logger io.Writer) { d.logOutput = logger diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 1a811df9c..d780a276f 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -9,7 +9,6 @@ import ( "runtime" "strings" "testing" - "time" "github.com/pilosa/pilosa/diagnostics" ) @@ -24,31 +23,17 @@ func TestDiagnosticsClient(t *testing.T) { d.SetLogger(ioutil.Discard) defer d.Close() - dur, _ := time.ParseDuration("123us") - d.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - d.Count("cc", 1, 1.0) - d.Gauge("gg", 10, 1.0) - d.Histogram("hh", 1, 1.0) - d.Timing("tt", dur, 1.0) - d.Set("ss", "ss", 1.0) + d.Set("gg", 10) + d.Set("ss", "ss") - d1 := d.WithTags("test") - if !reflect.DeepEqual(d, d1) { - t.Fatalf("Diagnostics is a singleton") - } - - if s := d.Tags(); s != nil { - t.Fatalf("No Diagnostics Tags") - } - - data, err := d.MarshalJSON() + data, err := d.Encode() if err != nil { t.Fatal(err) } // Test the recorded metrics, note that some types are skipped. var eq bool - output1 := []byte(`{"ct":1,"cc":1,"gg":"10","ss":"ss"}`) + output1 := []byte(`{"gg":10,"ss":"ss"}`) if eq, err = compareJSON(data, output1); err != nil { t.Fatal(err) } @@ -58,12 +43,12 @@ func TestDiagnosticsClient(t *testing.T) { // Test the metrics after a flush. d.Flush() - data, err = d.MarshalJSON() + data, err = d.Encode() if err != nil { t.Fatal(err) } - output2 := []byte(`{"gg":"10","ss":"ss","uptime":"0"}`) + output2 := []byte(`{"gg":10,"ss":"ss","uptime":0}`) if eq, err = compareJSON(data, output2); err != nil { t.Fatal(err) } @@ -160,8 +145,8 @@ func BenchmarkDiagnostics(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - d.Count("cc", 1, 1.0) - d.Gauge("gg", 10, 1.0) + d.Set("cc", 1) + d.Set("gg", "test") } }) } diff --git a/server.go b/server.go index 596f3be86..cab93b4b5 100644 --- a/server.go +++ b/server.go @@ -34,6 +34,7 @@ import ( "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" ) @@ -41,6 +42,7 @@ import ( const ( DefaultAntiEntropyInterval = 10 * time.Minute DefaultPollingInterval = 60 * time.Second + DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Server represents a holder wrapped by a running HTTP server. @@ -59,14 +61,16 @@ type Server struct { // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". - Network string - URI *URI - Cluster *Cluster + Network string + URI *URI + Cluster *Cluster + diagnostics *diagnostics.Diagnostics // Background monitoring intervals. AntiEntropyInterval time.Duration PollingInterval time.Duration MetricInterval time.Duration + DiagnosticInterval time.Duration // TLS configuration TLS *tls.Config @@ -88,12 +92,14 @@ func NewServer() *Server { Handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, + diagnostics: diagnostics.New(DefaultDiagnosticServer), Network: "tcp", AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, + DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval LogOutput: os.Stderr, } @@ -191,10 +197,11 @@ func (s *Server) Open() error { }() // Start background monitoring. - s.wg.Add(3) + s.wg.Add(4) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() go func() { defer s.wg.Done(); s.monitorMaxSlices() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() + go func() { defer s.wg.Done(); s.monitorDiagnostics() }() return nil } @@ -507,14 +514,57 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint return pb.MaxSlices, nil } +// monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info. +func (s *Server) monitorDiagnostics() { + if s.DiagnosticInterval <= 0 { + return + } + + s.diagnostics.SetLogger(s.LogOutput) + s.diagnostics.SetVersion(Version) + s.diagnostics.Set("Host", s.Host) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) + s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) + s.diagnostics.Set("NumCPU", runtime.NumCPU()) + // TODO: unique cluster ID + + ticker := time.NewTicker(s.DiagnosticInterval) + defer ticker.Stop() + + for { + // Wait for tick or a close. + select { + case <-s.closing: + return + case <-ticker.C: + numFrames := 0 + numSlices := uint64(0) + for _, index := range s.Holder.Indexes() { + numSlices += index.MaxSlice() + 1 + for _, f := range index.Frames() { + numFrames++ + if f.rangeEnabled { + s.diagnostics.Set("BSIEnabled", true) + } + if f.timeQuantum != "" { + s.diagnostics.Set("TimeQuantumEnabled", true) + } + } + } + + s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) + s.diagnostics.Set("NumFrames", numFrames) + s.diagnostics.Set("NumSlices", numSlices) + s.diagnostics.Set("OpenFiles", CountOpenFiles()) + s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) + s.diagnostics.CheckVersion() + s.diagnostics.Flush() + } + } +} + // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - s.Holder.Stats.Set("Host", s.Host, 1.0) - s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) - s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) - s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) - // TODO should we force this to run for diagnostics? - // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return diff --git a/server/server.go b/server/server.go index 0ce1680c2..d1063f306 100644 --- a/server/server.go +++ b/server/server.go @@ -31,7 +31,6 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -42,8 +41,7 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" + DefaultDataDir = "~/.pilosa" ) // Command represents the state of the pilosa server command. @@ -247,20 +245,12 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { - ms := make(pilosa.MultiStatsClient, 1) - d := diagnostics.New(DefaultDiagnosticServer) - d.SetVersion(pilosa.Version) - ms[0] = d - switch name { case "expvar": - ms = append(ms, pilosa.NewExpvarStatsClient()) + return pilosa.NewExpvarStatsClient(), nil case "statsd": - r, err := statsd.NewStatsClient(host) - if err != nil { - return nil, err - } - ms = append(ms, r) + return statsd.NewStatsClient(host) + default: + return pilosa.NopStatsClient, nil } - return ms, nil } From fb534d74933e622743c10b3acf0a4e5733216c22 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 16 Oct 2017 14:51:15 -0500 Subject: [PATCH 14/48] Flush diagnostics at startup, and then on each interval --- server.go | 57 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/server.go b/server.go index cab93b4b5..430bce7dc 100644 --- a/server.go +++ b/server.go @@ -62,7 +62,7 @@ type Server struct { // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". Network string - URI *URI + URI *URI Cluster *Cluster diagnostics *diagnostics.Diagnostics @@ -99,7 +99,7 @@ func NewServer() *Server { AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, - DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval + DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval, LogOutput: os.Stderr, } @@ -522,43 +522,48 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.SetLogger(s.LogOutput) s.diagnostics.SetVersion(Version) - s.diagnostics.Set("Host", s.Host) + s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) // TODO: unique cluster ID + // Flush the diagnostics metrics at startup, then on each tick interval + flush := func() { + numFrames := 0 + numSlices := uint64(0) + for _, index := range s.Holder.Indexes() { + numSlices += index.MaxSlice() + 1 + for _, f := range index.Frames() { + numFrames++ + if f.rangeEnabled { + s.diagnostics.Set("BSIEnabled", true) + } + if f.timeQuantum != "" { + s.diagnostics.Set("TimeQuantumEnabled", true) + } + } + } + + s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) + s.diagnostics.Set("NumFrames", numFrames) + s.diagnostics.Set("NumSlices", numSlices) + s.diagnostics.Set("OpenFiles", CountOpenFiles()) + s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) + s.diagnostics.CheckVersion() + s.diagnostics.Flush() + } + ticker := time.NewTicker(s.DiagnosticInterval) defer ticker.Stop() - + flush() for { // Wait for tick or a close. select { case <-s.closing: return case <-ticker.C: - numFrames := 0 - numSlices := uint64(0) - for _, index := range s.Holder.Indexes() { - numSlices += index.MaxSlice() + 1 - for _, f := range index.Frames() { - numFrames++ - if f.rangeEnabled { - s.diagnostics.Set("BSIEnabled", true) - } - if f.timeQuantum != "" { - s.diagnostics.Set("TimeQuantumEnabled", true) - } - } - } - - s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) - s.diagnostics.Set("NumFrames", numFrames) - s.diagnostics.Set("NumSlices", numSlices) - s.diagnostics.Set("OpenFiles", CountOpenFiles()) - s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) - s.diagnostics.CheckVersion() - s.diagnostics.Flush() + flush() } } } From 48c0dbaee75bf5c9e13216add47c3e6d1cdc85b1 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 17 Oct 2017 11:35:01 +0300 Subject: [PATCH 15/48] Initial client refactoring --- client.go | 75 +++++++++++++++++++++++++----------------------- client_test.go | 35 ++++++++++++++-------- ctl/bench.go | 9 ++++-- executor.go | 60 ++++++++------------------------------ fragment.go | 6 +++- holder_test.go | 5 +++- server.go | 5 +++- test/executor.go | 3 +- uri.go | 5 ++++ uri_test.go | 11 +++++++ 10 files changed, 111 insertions(+), 103 deletions(-) diff --git a/client.go b/client.go index fbd0d3717..3babe2919 100644 --- a/client.go +++ b/client.go @@ -43,8 +43,8 @@ type ClientOptions struct { // Client represents a client to the Pilosa cluster. type Client struct { - host *URI - options *ClientOptions + defaultURI *URI + options *ClientOptions // The client to use for HTTP communication. HTTPClient *http.Client @@ -64,10 +64,7 @@ func NewClient(host string, options *ClientOptions) (*Client, error) { return NewClientFromURI(uri, options) } -func NewClientFromURI(uri *URI, options *ClientOptions) (*Client, error) { - if uri == nil { - return nil, ErrHostRequired - } +func NewClientFromURI(defaultURI *URI, options *ClientOptions) (*Client, error) { if options == nil { options = &ClientOptions{} } @@ -77,13 +74,13 @@ func NewClientFromURI(uri *URI, options *ClientOptions) (*Client, error) { } client := &http.Client{Transport: transport} return &Client{ - host: uri, + defaultURI: defaultURI, HTTPClient: client, }, nil } // Host returns the host the client was initialized with. -func (c *Client) Host() *URI { return c.host } +func (c *Client) Host() *URI { return c.defaultURI } // MaxSliceByIndex returns the number of slices on a server by index. func (c *Client) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { @@ -98,7 +95,7 @@ func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, // maxSliceByIndex returns the number of slices on a server by index. func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.host, "/slices/max") + u := uriPathToURL(c.defaultURI, "/slices/max") u.RawQuery = (&url.Values{ "inverse": {strconv.FormatBool(inverse)}, }).Encode() @@ -131,7 +128,7 @@ func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string] // Schema returns all index and frame schema information. func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request against the host. - u := uriPathToURL(c.host, "/schema") + u := uriPathToURL(c.defaultURI, "/schema") // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -168,7 +165,7 @@ func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions } // Create URL & HTTP request. - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s", index)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { return err @@ -205,7 +202,7 @@ func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions // FragmentNodes returns a list of nodes that own a slice. func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { // Execute request against the host. - u := uriPathToURL(c.host, "/fragment/nodes") + u := uriPathToURL(c.defaultURI, "/fragment/nodes") u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() // Build request. @@ -234,28 +231,30 @@ func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) } // ExecuteQuery executes query against index on the server. -func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRedirect bool) (result interface{}, err error) { +func (c *Client) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { if index == "" { return nil, ErrIndexRequired - } else if query == "" { + } else if queryRequest.Query == "" { return nil, ErrQueryRequired } - // Encode query request. - buf, err := proto.Marshal(&internal.QueryRequest{ - Query: query, - Remote: !allowRedirect, - }) - if err != nil { - return nil, fmt.Errorf("marshal: %s", err) - } - - // Create URL & HTTP request. - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/query", index)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + // Encode request object. + buf, err := proto.Marshal(queryRequest) if err != nil { return nil, err } + + // Create HTTP request. + clientURI := c.defaultURI + if contextURI, ok := ctx.Value("uri").(*URI); ok { + clientURI = contextURI + } + u := clientURI.Path(fmt.Sprintf("/index/%s/query", index)) + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") @@ -276,8 +275,8 @@ func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRed return nil, errors.New(string(body)) } - var qresp internal.QueryResponse - if err := proto.Unmarshal(body, &qresp); err != nil { + qresp := &internal.QueryResponse{} + if err := proto.Unmarshal(body, qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } else if s := qresp.Err; s != "" { return nil, errors.New(s) @@ -288,7 +287,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRed // ExecutePQL executes query string against index on the server. func (c *Client) ExecutePQL(ctx context.Context, index, query string) (interface{}, error) { - u := uriPathToURL(c.host, "/query") + u := uriPathToURL(c.defaultURI, "/query") u.RawQuery = url.Values{"index": {index}}.Encode() req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query))) @@ -811,7 +810,7 @@ func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt Frame } // Create URL & HTTP request. - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s", index, frame)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { return err @@ -847,7 +846,7 @@ func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt Frame // RestoreFrame restores an entire frame from a host in another cluster. func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) error { - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s/restore", index, frame)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/restore", index, frame)) u.RawQuery = url.Values{ "host": {host}, }.Encode() @@ -878,7 +877,7 @@ func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) er // FrameViews returns a list of view names for a frame. func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, error) { // Create URL & HTTP request. - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s/views", index, frame)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame)) req, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, err @@ -914,7 +913,7 @@ func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) { - u := uriPathToURL(c.host, "/fragment/blocks") + u := uriPathToURL(c.defaultURI, "/fragment/blocks") u.RawQuery = url.Values{ "index": {index}, "frame": {frame}, @@ -967,7 +966,7 @@ func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice return nil, nil, err } - u := uriPathToURL(c.host, "/fragment/block/data") + u := uriPathToURL(c.defaultURI, "/fragment/block/data") req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) if err != nil { return nil, nil, err @@ -1004,7 +1003,7 @@ func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice // ColumnAttrDiff returns data from differing blocks on a remote host. func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/attr/diff", index)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index)) // Encode request. buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) @@ -1044,7 +1043,7 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBl // RowAttrDiff returns data from differing blocks on a remote host. func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.host, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame)) + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame)) // Encode request. buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) @@ -1229,3 +1228,7 @@ func nodePathToURL(node *Node, path string) url.URL { Path: path, } } + +type InternalClient interface { + ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) +} diff --git a/client_test.go b/client_test.go index 70ea6c669..509039618 100644 --- a/client_test.go +++ b/client_test.go @@ -54,7 +54,10 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e, err := pilosa.NewExecutor(nil) + if err != nil { + t.Fatal(err) + } e.Holder = hldr[0].Holder e.Scheme = cluster.Nodes[0].Scheme e.Host = cluster.Nodes[0].Host @@ -62,7 +65,10 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e, err := pilosa.NewExecutor(nil) + if err != nil { + t.Fatal(err) + } e.Holder = hldr[1].Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host @@ -70,7 +76,10 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e, err := pilosa.NewExecutor(nil) + if err != nil { + t.Fatal(err) + } e.Holder = hldr[2].Holder e.Scheme = cluster.Nodes[2].Scheme e.Host = cluster.Nodes[2].Host @@ -140,15 +149,17 @@ func TestClient_MultiNode(t *testing.T) { client[2] = test.MustNewClient(s[2].Host()) topN := 4 - q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN) - - result, err := client[0].ExecuteQuery(context.Background(), "i", q, true) + queryRequest := &internal.QueryRequest{ + Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN), + Remote: false, + } + result, err := client[0].ExecuteQuery(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } // Check the results before every node has the correct max slice value. - pairs := result.(internal.QueryResponse).Results[0].Pairs + pairs := result.Results[0].Pairs for _, pair := range pairs { if pair.Key == 22 && pair.Count != 3 { t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) @@ -160,13 +171,13 @@ func TestClient_MultiNode(t *testing.T) { hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) - result, err = client[0].ExecuteQuery(context.Background(), "i", q, true) + result, err = client[0].ExecuteQuery(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } // Test must return exactly N results. - if len(result.(internal.QueryResponse).Results[0].Pairs) != topN { + if len(result.Results[0].Pairs) != topN { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } p := []*internal.Pair{ @@ -176,15 +187,15 @@ func TestClient_MultiNode(t *testing.T) { {Key: 99, Count: 7}} // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.(internal.QueryResponse).Results[0].Pairs, p) { + if !reflect.DeepEqual(result.Results[0].Pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } - result1, err := client[1].ExecuteQuery(context.Background(), "i", q, true) + result1, err := client[1].ExecuteQuery(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } - result2, err := client[2].ExecuteQuery(context.Background(), "i", q, true) + result2, err := client[2].ExecuteQuery(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } diff --git a/ctl/bench.go b/ctl/bench.go index 24e6e7658..2f1e7a52f 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" ) // BenchCommand represents a command for benchmarking index operations. @@ -89,9 +90,11 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e rowID := rand.Intn(maxRowID) columnID := rand.Intn(maxColumnID) - q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID) - - if _, err := client.ExecuteQuery(ctx, cmd.Index, q, true); err != nil { + queryRequest := &internal.QueryRequest{ + Query: fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID), + Remote: false, + } + if _, err := client.ExecuteQuery(ctx, cmd.Index, queryRequest); err != nil { return err } } diff --git a/executor.go b/executor.go index 64a068149..8d70cb2f1 100644 --- a/executor.go +++ b/executor.go @@ -15,16 +15,12 @@ package pilosa import ( - "bytes" "context" "errors" "fmt" - "io/ioutil" - "net/http" "sort" "time" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -47,26 +43,25 @@ type Executor struct { Host string Cluster *Cluster - // Client used for remote HTTP requests. - HTTPClient *http.Client + // Client used for remote requests. + client InternalClient // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int } // NewExecutor returns a new instance of Executor. -func NewExecutor(clientOptions *ClientOptions) *Executor { +func NewExecutor(clientOptions *ClientOptions) (*Executor, error) { if clientOptions == nil { clientOptions = &ClientOptions{} } - transport := &http.Transport{} - if clientOptions.TLS != nil { - transport.TLSClientConfig = clientOptions.TLS + client, err := NewClientFromURI(nil, clientOptions) + if err != nil { + return nil, err } - client := &http.Client{Transport: transport} return &Executor{ - HTTPClient: client, - } + client: client, + }, nil } // Execute executes a PQL query. @@ -1377,48 +1372,17 @@ func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Qu Slices: slices, Remote: true, } - buf, err := proto.Marshal(pbreq) + uri, err := NewURIFromAddress(node.Host) if err != nil { return nil, err } - - // Create HTTP request. - u := nodePathToURL(node, fmt.Sprintf("/index/%s/query", index)) - u.Scheme = e.Scheme - req, err := http.NewRequest("POST", (&u).String(), bytes.NewReader(buf)) + uri.SetScheme(node.Scheme) + ctx = context.WithValue(ctx, "uri", uri) + pb, err := e.client.ExecuteQuery(ctx, index, pbreq) if err != nil { return nil, err } - // Require protobuf encoding. - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Send request to remote node. - resp, err := e.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // Read response into buffer. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - // Check status code. - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("invalid status Executor.exec: code=%d, err=%s, req: %v", resp.StatusCode, body, req) - } - - // Decode response object. - var pb internal.QueryResponse - if err := proto.Unmarshal(body, &pb); err != nil { - return nil, err - } - // Return an error, if specified on response. if err := decodeError(pb.Err); err != nil { return nil, err diff --git a/fragment.go b/fragment.go index 67f9ff000..cdc8c00ff 100644 --- a/fragment.go +++ b/fragment.go @@ -1848,7 +1848,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Execute query. - _, err := clients[i].ExecuteQuery(context.Background(), f.Index(), buf.String(), false) + queryRequest := &internal.QueryRequest{ + Query: buf.String(), + Remote: true, + } + _, err := clients[i].ExecuteQuery(context.Background(), f.Index(), queryRequest) if err != nil { return err } diff --git a/holder_test.go b/holder_test.go index 104bd1bb0..cb33045e6 100644 --- a/holder_test.go +++ b/holder_test.go @@ -314,7 +314,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer s.Close() s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(nil) + e, err := pilosa.NewExecutor(nil) + if err != nil { + t.Fatal(err) + } e.Holder = hldr1.Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host diff --git a/server.go b/server.go index 773cf9025..e73354cf2 100644 --- a/server.go +++ b/server.go @@ -164,7 +164,10 @@ func (s *Server) Open() error { s.createDefaultClient() // Create executor for executing queries. - e := NewExecutor(&ClientOptions{TLS: s.TLS}) + e, err := NewExecutor(&ClientOptions{TLS: s.TLS}) + if err != nil { + return err + } e.Holder = s.Holder e.Scheme = s.URI.Scheme() e.Host = s.URI.HostPort() diff --git a/test/executor.go b/test/executor.go index 73445a1cd..4ef99d351 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,7 +15,8 @@ type Executor struct { // NewExecutor returns a new instance of Executor. // The executor always matches the hostname of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - e := &Executor{Executor: pilosa.NewExecutor(nil)} + executor, _ := pilosa.NewExecutor(nil) + e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster e.Scheme = cluster.Nodes[0].Scheme diff --git a/uri.go b/uri.go index de6e4bae6..ee5d23d75 100644 --- a/uri.go +++ b/uri.go @@ -144,6 +144,11 @@ func (u URI) Equals(other *URI) bool { u.port == other.port } +// Path returns URI with path +func (u *URI) Path(path string) string { + return fmt.Sprintf("%s%s", u.Normalize(), path) +} + // The following methods are required to implement pflag Value interface. // Set sets the time quantum value. diff --git a/uri_test.go b/uri_test.go index 33ccd35f9..70ea3f273 100644 --- a/uri_test.go +++ b/uri_test.go @@ -83,6 +83,17 @@ func TestNormalizedAddress(t *testing.T) { } } +func TestURIPath(t *testing.T) { + uri, err := NewURIFromAddress("http+protobuf://big-data.pilosa.com:6888") + if err != nil { + t.Fatal(err) + } + target := "http://big-data.pilosa.com:6888/index/foo" + if uri.Path("/index/foo") != target { + t.Fatalf("%s != %s", uri.Path("/index/foo"), target) + } +} + func TestEquals(t *testing.T) { uri1 := DefaultURI() if uri1.Equals(nil) { From 0f8f59637623abe648367958506b289fb36073ae Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 17 Oct 2017 12:07:45 +0300 Subject: [PATCH 16/48] Trivial NewClientFromURI simplification --- client.go | 7 ++++--- executor.go | 6 +----- handler.go | 6 +----- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index 3babe2919..70cb18047 100644 --- a/client.go +++ b/client.go @@ -61,10 +61,11 @@ func NewClient(host string, options *ClientOptions) (*Client, error) { return nil, err } - return NewClientFromURI(uri, options) + client := NewClientFromURI(uri, options) + return client, nil } -func NewClientFromURI(defaultURI *URI, options *ClientOptions) (*Client, error) { +func NewClientFromURI(defaultURI *URI, options *ClientOptions) *Client { if options == nil { options = &ClientOptions{} } @@ -76,7 +77,7 @@ func NewClientFromURI(defaultURI *URI, options *ClientOptions) (*Client, error) return &Client{ defaultURI: defaultURI, HTTPClient: client, - }, nil + } } // Host returns the host the client was initialized with. diff --git a/executor.go b/executor.go index 8d70cb2f1..7aee4e472 100644 --- a/executor.go +++ b/executor.go @@ -55,12 +55,8 @@ func NewExecutor(clientOptions *ClientOptions) (*Executor, error) { if clientOptions == nil { clientOptions = &ClientOptions{} } - client, err := NewClientFromURI(nil, clientOptions) - if err != nil { - return nil, err - } return &Executor{ - client: client, + client: NewClientFromURI(nil, clientOptions), }, nil } diff --git a/handler.go b/handler.go index 016e3a348..811ab14f1 100644 --- a/handler.go +++ b/handler.go @@ -1506,11 +1506,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Create a client for the remote cluster. - client, err := NewClientFromURI(host, h.ClientOptions) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + client := NewClientFromURI(host, h.ClientOptions) // Determine the maximum number of slices. maxSlices, err := client.MaxSliceByIndex(r.Context()) From c9f526f5bf42c755e6767bc823fad8761ee40e55 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 17 Oct 2017 14:56:43 +0300 Subject: [PATCH 17/48] Removed unused Client.ExecutePQL function --- client.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/client.go b/client.go index 70cb18047..af28b068c 100644 --- a/client.go +++ b/client.go @@ -286,34 +286,6 @@ func (c *Client) ExecuteQuery(ctx context.Context, index string, queryRequest *i return qresp, nil } -// ExecutePQL executes query string against index on the server. -func (c *Client) ExecutePQL(ctx context.Context, index, query string) (interface{}, error) { - u := uriPathToURL(c.defaultURI, "/query") - u.RawQuery = url.Values{"index": {index}}.Encode() - - req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(query))) - if err != nil { - return nil, err - } - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } else if resp.StatusCode != http.StatusOK { - return nil, errors.New(string(body)) - } - return string(body), nil - -} - // Import bulk imports bits for a single slice to a host. func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { if index == "" { From e201afe241668c147fe4529e8457ec17dc421f45 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 17 Oct 2017 15:53:42 +0300 Subject: [PATCH 18/48] Replaced all http.Clients with InternalClient; updated InternalClient interface. --- client.go | 52 +++++++++++++++++++++++++++++++++++++-------------- ctl/bench.go | 2 +- ctl/import.go | 2 +- fragment.go | 2 +- server.go | 32 +++++++------------------------ 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/client.go b/client.go index af28b068c..74e051758 100644 --- a/client.go +++ b/client.go @@ -96,7 +96,7 @@ func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, // maxSliceByIndex returns the number of slices on a server by index. func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") + u := uriPathToURL(c.clientURI(ctx), "/slices/max") u.RawQuery = (&url.Values{ "inverse": {strconv.FormatBool(inverse)}, }).Encode() @@ -129,10 +129,10 @@ func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string] // Schema returns all index and frame schema information. func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/schema") + u := c.defaultURI.Path("/schema") // Build request. - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } @@ -246,11 +246,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, index string, queryRequest *i } // Create HTTP request. - clientURI := c.defaultURI - if contextURI, ok := ctx.Value("uri").(*URI); ok { - clientURI = contextURI - } - u := clientURI.Path(fmt.Sprintf("/index/%s/query", index)) + u := c.clientURI(ctx).Path(fmt.Sprintf("/index/%s/query", index)) req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) if err != nil { return nil, err @@ -294,7 +290,7 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, return ErrFrameRequired } - buf, err := MarshalImportPayload(index, frame, slice, bits) + buf, err := marshalImportPayload(index, frame, slice, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -331,8 +327,8 @@ func (c *Client) EnsureFrame(ctx context.Context, indexName string, frameName st return err } -// MarshalImportPayload marshalls the import parameters into a protobuf byte slice. -func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { +// marshalImportPayload marshalls the import parameters into a protobuf byte slice. +func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() @@ -399,7 +395,7 @@ func (c *Client) ImportValue(ctx context.Context, index, frame, field string, sl return ErrFrameRequired } - buf, err := MarshalImportValuePayload(index, frame, field, slice, vals) + buf, err := marshalImportValuePayload(index, frame, field, slice, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -420,8 +416,8 @@ func (c *Client) ImportValue(ctx context.Context, index, frame, field string, sl return nil } -// MarshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func MarshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) { +// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. +func marshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() @@ -1056,6 +1052,14 @@ func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []At return rsp.Attrs, nil } +func (c *Client) clientURI(ctx context.Context) *URI { + clientURI := c.defaultURI + if contextURI, ok := ctx.Value("uri").(*URI); ok { + clientURI = contextURI + } + return clientURI +} + // Bit represents the location of a single bit. type Bit struct { RowID uint64 @@ -1203,5 +1207,25 @@ func nodePathToURL(node *Node, path string) url.URL { } type InternalClient interface { + MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) + MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) + Schema(ctx context.Context) ([]*IndexInfo, error) + CreateIndex(ctx context.Context, index string, opt IndexOptions) error + FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error + EnsureIndex(ctx context.Context, name string, options IndexOptions) error + EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error + ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error + ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error + BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error + BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) + RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error + CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error + RestoreFrame(ctx context.Context, host, index, frame string) error + FrameViews(ctx context.Context, index, frame string) ([]string, error) + FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) + ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) } diff --git a/ctl/bench.go b/ctl/bench.go index 2f1e7a52f..01e07cc14 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -71,7 +71,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { } // runSetBit executes a benchmark of random SetBit() operations. -func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error { +func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error { if cmd.N == 0 { return errors.New("operation count required") } else if cmd.Index == "" { diff --git a/ctl/import.go b/ctl/import.go index e3eda260e..d2f888956 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -58,7 +58,7 @@ type ImportCommand struct { Sort bool `json:"sort"` // Reusable client. - Client *pilosa.Client `json:"-"` + Client pilosa.InternalClient `json:"-"` // Standard input/output *pilosa.CmdIO diff --git a/fragment.go b/fragment.go index cdc8c00ff..148fd6a48 100644 --- a/fragment.go +++ b/fragment.go @@ -1782,7 +1782,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet - var clients []*Client + var clients []InternalClient for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { if s.Host == node.Host { continue diff --git a/server.go b/server.go index e73354cf2..3ce2883c3 100644 --- a/server.go +++ b/server.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -35,6 +34,7 @@ import ( "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "golang.org/x/net/context" ) // Default server settings. @@ -76,7 +76,7 @@ type Server struct { LogOutput io.Writer - defaultClient *http.Client + defaultClient InternalClient } // NewServer returns a new instance of Server. @@ -483,31 +483,13 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("User-Agent", "pilosa/"+Version) - resp, err := s.defaultClient.Do(req) + nodeURI, err := NewURIFromAddress(hostPort) if err != nil { return nil, err } - defer resp.Body.Close() - - // Read response into buffer. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - // Check status code. - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("invalid status checkMaxSlices: code=%d, err=%s, req=%v", resp.StatusCode, body, req) - } - - // Decode response object. - pb := internal.MaxSlicesResponse{} - - if err = proto.Unmarshal(body, &pb); err != nil { - return nil, err - } - - return pb.MaxSlices, nil + nodeURI.SetScheme(scheme) + ctx := context.WithValue(context.Background(), "uri", nodeURI) + return s.defaultClient.MaxSliceByIndex(ctx) } // monitorRuntime periodically polls the Go runtime metrics. @@ -558,7 +540,7 @@ func (s *Server) createDefaultClient() { if s.TLS != nil { transport.TLSClientConfig = s.TLS } - s.defaultClient = &http.Client{Transport: transport} + s.defaultClient = NewClientFromURI(nil, &ClientOptions{TLS: s.TLS}) } // CountOpenFiles on opperating systems that support lsof From 838d56011cb411ea98fb7919a385b07cc446a8ab Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 17 Oct 2017 16:04:42 +0300 Subject: [PATCH 19/48] Renamed Client to InternalHTTPClient --- client.go | 74 +++++++++++++++++++++---------------------- ctl/common.go | 4 +-- server/server_test.go | 2 +- test/client.go | 4 +-- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/client.go b/client.go index 74e051758..8528e52dd 100644 --- a/client.go +++ b/client.go @@ -36,13 +36,13 @@ import ( "github.com/pilosa/pilosa/internal" ) -// ClientOptions represents the configuration for a Client +// ClientOptions represents the configuration for a InternalHTTPClient type ClientOptions struct { TLS *tls.Config } -// Client represents a client to the Pilosa cluster. -type Client struct { +// InternalHTTPClient represents a client to the Pilosa cluster. +type InternalHTTPClient struct { defaultURI *URI options *ClientOptions @@ -50,8 +50,8 @@ type Client struct { HTTPClient *http.Client } -// NewClient returns a new instance of Client to connect to host. -func NewClient(host string, options *ClientOptions) (*Client, error) { +// NewClient returns a new instance of InternalHTTPClient to connect to host. +func NewClient(host string, options *ClientOptions) (*InternalHTTPClient, error) { if host == "" { return nil, ErrHostRequired } @@ -65,7 +65,7 @@ func NewClient(host string, options *ClientOptions) (*Client, error) { return client, nil } -func NewClientFromURI(defaultURI *URI, options *ClientOptions) *Client { +func NewClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient { if options == nil { options = &ClientOptions{} } @@ -74,27 +74,27 @@ func NewClientFromURI(defaultURI *URI, options *ClientOptions) *Client { transport.TLSClientConfig = options.TLS } client := &http.Client{Transport: transport} - return &Client{ + return &InternalHTTPClient{ defaultURI: defaultURI, HTTPClient: client, } } // Host returns the host the client was initialized with. -func (c *Client) Host() *URI { return c.defaultURI } +func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } // MaxSliceByIndex returns the number of slices on a server by index. -func (c *Client) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { return c.maxSliceByIndex(ctx, false) } // MaxInverseSliceByIndex returns the number of inverse slices on a server by index. -func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) { +func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) { return c.maxSliceByIndex(ctx, true) } // maxSliceByIndex returns the number of slices on a server by index. -func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { +func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. u := uriPathToURL(c.clientURI(ctx), "/slices/max") u.RawQuery = (&url.Values{ @@ -127,7 +127,7 @@ func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string] } // Schema returns all index and frame schema information. -func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) { +func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request against the host. u := c.defaultURI.Path("/schema") @@ -156,7 +156,7 @@ func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) { } // CreateIndex creates a new index on the server. -func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { +func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { // Encode query request. buf, err := json.Marshal(&postIndexRequest{ Options: opt, @@ -201,7 +201,7 @@ func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions } // FragmentNodes returns a list of nodes that own a slice. -func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { +func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/fragment/nodes") u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() @@ -232,7 +232,7 @@ func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) } // ExecuteQuery executes query against index on the server. -func (c *Client) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { if index == "" { return nil, ErrIndexRequired } else if queryRequest.Query == "" { @@ -283,7 +283,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, index string, queryRequest *i } // Import bulk imports bits for a single slice to a host. -func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { +func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { if index == "" { return ErrIndexRequired } else if frame == "" { @@ -311,7 +311,7 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, return nil } -func (c *Client) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { +func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { err := c.CreateIndex(ctx, name, options) if err == nil || err == ErrIndexExists { return nil @@ -319,7 +319,7 @@ func (c *Client) EnsureIndex(ctx context.Context, name string, options IndexOpti return err } -func (c *Client) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error { +func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error { err := c.CreateFrame(ctx, indexName, frameName, options) if err == nil || err == ErrFrameExists { return nil @@ -350,7 +350,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte } // importNode sends a pre-marshaled import request to a node. -func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { +func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []byte) error { // Create URL & HTTP request. u := nodePathToURL(node, "/import") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) @@ -388,7 +388,7 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { } // ImportValue bulk imports field values for a single slice to a host. -func (c *Client) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error { +func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error { if index == "" { return ErrIndexRequired } else if frame == "" { @@ -438,7 +438,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals [] } // importValueNode sends a pre-marshaled import request to a node. -func (c *Client) importValueNode(ctx context.Context, node *Node, buf []byte) error { +func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, buf []byte) error { // Create URL & HTTP request. u := nodePathToURL(node, "/import-value") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) @@ -476,7 +476,7 @@ func (c *Client) importValueNode(ctx context.Context, node *Node, buf []byte) er } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error { +func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error { if index == "" { return ErrIndexRequired } else if frame == "" { @@ -508,7 +508,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice } // exportNode copies a CSV export from a node to w. -func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error { +func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error { // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ @@ -547,7 +547,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, vi } // BackupTo backs up an entire frame from a cluster to w. -func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error { +func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error { if index == "" { return ErrIndexRequired } else if frame == "" { @@ -588,7 +588,7 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view s } // backupSliceTo backs up a single slice to tw. -func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error { +func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error { // Return error if unable to backup from any slice. r, err := c.BackupSlice(ctx, index, frame, view, slice) if err != nil { @@ -626,7 +626,7 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame // BackupSlice retrieves a streaming backup from a single slice. // This function tries slice owners until one succeeds. -func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) { +func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) { // Retrieve a list of nodes that own the slice. nodes, err := c.FragmentNodes(ctx, index, slice) if err != nil { @@ -649,7 +649,7 @@ func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, sli return nil, fmt.Errorf("unable to connect to any owner") } -func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { +func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, @@ -685,7 +685,7 @@ func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, } // RestoreFrom restores a frame from a backup file to an entire cluster. -func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error { +func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error { if index == "" { return ErrIndexRequired } else if frame == "" { @@ -724,7 +724,7 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, vie } // restoreSliceFrom restores a single slice to all owning nodes. -func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error { +func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error { // Retrieve a list of nodes that own the slice. nodes, err := c.FragmentNodes(ctx, index, slice) if err != nil { @@ -765,7 +765,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, } // CreateFrame creates a new frame on the server. -func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { +func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { if index == "" { return ErrIndexRequired } @@ -814,7 +814,7 @@ func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt Frame } // RestoreFrame restores an entire frame from a host in another cluster. -func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) error { +func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, frame string) error { u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/restore", index, frame)) u.RawQuery = url.Values{ "host": {host}, @@ -844,7 +844,7 @@ func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) er } // FrameViews returns a list of view names for a frame. -func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, error) { +func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string) ([]string, error) { // Create URL & HTTP request. u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame)) req, err := http.NewRequest("GET", u.String(), nil) @@ -881,7 +881,7 @@ func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) { +func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) { u := uriPathToURL(c.defaultURI, "/fragment/blocks") u.RawQuery = url.Values{ "index": {index}, @@ -923,7 +923,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, } // BlockData returns row/column id pairs for a block. -func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Frame: frame, @@ -971,7 +971,7 @@ func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice } // ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index)) // Encode request. @@ -1011,7 +1011,7 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBl } // RowAttrDiff returns data from differing blocks on a remote host. -func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame)) // Encode request. @@ -1052,7 +1052,7 @@ func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []At return rsp.Attrs, nil } -func (c *Client) clientURI(ctx context.Context) *URI { +func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI { clientURI := c.defaultURI if contextURI, ok := ctx.Value("uri").(*URI); ok { clientURI = contextURI diff --git a/ctl/common.go b/ctl/common.go index 975f19898..52b4e6e7e 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -19,8 +19,8 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)") } -// CommandClient returns a pilosa.Client for the command -func CommandClient(cmd CommandWithTLSSupport) (*pilosa.Client, error) { +// CommandClient returns a pilosa.InternalHTTPClient for the command +func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { tlsConfig := cmd.TLSConfiguration() var clientOptions *pilosa.ClientOptions if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { diff --git a/server/server_test.go b/server/server_test.go index a5de5d68c..f03e3331b 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -696,7 +696,7 @@ func (m *Main) Reopen() error { func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.Client { +func (m *Main) Client() *pilosa.InternalHTTPClient { client, err := pilosa.NewClient(m.Server.URI.HostPort(), nil) if err != nil { panic(err) diff --git a/test/client.go b/test/client.go index 10ef6368e..d9044df66 100644 --- a/test/client.go +++ b/test/client.go @@ -6,7 +6,7 @@ import ( // Client represents a test wrapper for pilosa.Client. type Client struct { - *pilosa.Client + *pilosa.InternalHTTPClient } // MustNewClient returns a new instance of Client. Panic on error. @@ -15,5 +15,5 @@ func MustNewClient(host string) *Client { if err != nil { panic(err) } - return &Client{Client: c} + return &Client{InternalHTTPClient: c} } From e48ade4a9ec25605b80ef6cb3f9b3e4be453de04 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 17 Oct 2017 13:49:35 -0500 Subject: [PATCH 20/48] Set the diagnostics interval and circuit breaker timeout at Open() --- diagnostics/diagnostics.go | 25 ++++++++++++++----------- diagnostics/diagnostics_test.go | 2 ++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index 761e7adca..35d37f6cc 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -18,10 +18,9 @@ import ( // TODO: unique Cluster ID -// Default interval to sync diagnostics metrics. +// Default version check URL. const ( - DefaultDiagnosticsInterval = 1 * time.Hour - DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" + DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" ) type versionResponse struct { @@ -51,8 +50,6 @@ type Diagnostics struct { // New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". func New(host string) *Diagnostics { - var st gobreaker.Settings - st.Timeout = DefaultDiagnosticsInterval * 2 return &Diagnostics{ closing: make(chan struct{}), @@ -62,9 +59,7 @@ func New(host string) *Diagnostics { start: time.Now(), client: http.DefaultClient, metrics: make(map[string]interface{}), - interval: DefaultDiagnosticsInterval, logOutput: ioutil.Discard, - cb: gobreaker.NewCircuitBreaker(st), } } @@ -74,6 +69,11 @@ func (d *Diagnostics) SetVersion(v string) { d.Set("Version", v) } +// SetInterval of the diagnostic go routine and match with the circuit breaker timeout. +func (d *Diagnostics) SetInterval(i time.Duration) { + d.interval = i +} + // schedule start the diagnostics service ticker. func (d *Diagnostics) schedule() { ticker := time.NewTicker(d.interval) @@ -117,10 +117,13 @@ func (d *Diagnostics) Flush() error { return err } -// Open starts the diagnostics metric go routine. +// Open configures the circuit breaker used by the HTTP client. func (d *Diagnostics) Open() { - d.wg.Add(1) - go func() { defer d.wg.Done(); d.schedule() }() + var st gobreaker.Settings + if d.interval > 0 { + st.Timeout = d.interval * 2 + } + d.cb = gobreaker.NewCircuitBreaker(st) } // Close notify goroutine to stop. @@ -169,7 +172,7 @@ func (d *Diagnostics) CompareVersion(value string) error { return nil } -// Encode metrics maps into the json message format +// Encode metrics maps into the json message format. func (d *Diagnostics) Encode() ([]byte, error) { return json.Marshal(d.metrics) } diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index d780a276f..81d6b7cf6 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -21,6 +21,7 @@ func TestDiagnosticsClient(t *testing.T) { // Create a new client. d := diagnostics.New(server.URL) d.SetLogger(ioutil.Discard) + d.Open() defer d.Close() d.Set("gg", 10) @@ -69,6 +70,7 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { func TestDiagnosticsVersion_Compare(t *testing.T) { d := diagnostics.New("localhost:10101") + d.Open() defer d.Close() version := "0.1.1" From 722e73d30534999fd4950dae0b963acc9e96e121 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 17 Oct 2017 13:50:28 -0500 Subject: [PATCH 21/48] Config option for diagnostics interval. Default to 1 hour --- config.go | 11 ++++++++--- ctl/server.go | 1 + server.go | 4 +++- server/server.go | 2 ++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index fe93be19e..68a351523 100644 --- a/config.go +++ b/config.go @@ -43,6 +43,9 @@ const ( // DefaultMaxWritesPerRequest is the default number of writes per request. DefaultMaxWritesPerRequest = 5000 + + // DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. + DefaultDiagnosticsInterval = 1 * time.Hour ) // ClusterTypes set of cluster types. @@ -88,9 +91,10 @@ type Config struct { LogPath string `toml:"log-path"` Metric struct { - Service string `toml:"service"` - Host string `toml:"host"` - PollInterval Duration `toml:"poll-interval"` + Service string `toml:"service"` + Host string `toml:"host"` + PollInterval Duration `toml:"poll-interval"` + DiagnosticInterval Duration `toml:"diagnostics"` } `toml:"metric"` TLS TLSConfig @@ -108,6 +112,7 @@ func NewConfig() *Config { c.Cluster.Hosts = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) c.Metric.Service = DefaultMetrics + c.Metric.DiagnosticInterval = Duration(DefaultDiagnosticsInterval) c.TLS = TLSConfig{} return c } diff --git a/ctl/server.go b/ctl/server.go index a0c080729..a39c86e3e 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -41,6 +41,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") + flags.DurationVarP((*time.Duration)(&srv.Config.Metric.DiagnosticInterval), "metric.diagnostics", "", time.Hour*1, "Diagnostic reporting interval back to Pilosa.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) } diff --git a/server.go b/server.go index 430bce7dc..d9b4209fc 100644 --- a/server.go +++ b/server.go @@ -99,7 +99,7 @@ func NewServer() *Server { AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, - DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval, + DiagnosticInterval: 0, LogOutput: os.Stderr, } @@ -522,6 +522,8 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.SetLogger(s.LogOutput) s.diagnostics.SetVersion(Version) + s.diagnostics.SetInterval(s.DiagnosticInterval) + s.diagnostics.Open() s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) diff --git a/server/server.go b/server/server.go index d1063f306..fb18137ad 100644 --- a/server/server.go +++ b/server/server.go @@ -30,6 +30,7 @@ import ( "time" "crypto/tls" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" @@ -143,6 +144,7 @@ func (m *Command) SetupServer() error { m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + m.Server.DiagnosticInterval = time.Duration(m.Config.Metric.DiagnosticInterval) if err != nil { return err } From 78b599f1f11ead9037799dae9b21887caede9c8e Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 18 Oct 2017 09:22:55 -0500 Subject: [PATCH 22/48] diagnostics docs --- docs/administration.md | 21 +++++++++++++++++++++ docs/configuration.md | 13 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/administration.md b/docs/administration.md index ed4da61ee..861d7f6a1 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -112,6 +112,27 @@ Note: This will only work when the replication factor is >= 2 - Restart the cluster - Wait for the 1st sync (10 minutes) to validate Index connections +#### Diagnostics + +Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. + +Version: Version string of the build. +Host: Host URI. +Cluster: List of nodes in the Cluster. +NumNodes: Number of nodes in the Cluster. +NumCPU: Number of Cores per Node +BSIEnabled: Bit Slice Index Frames in use. +TimeQuantumEnabled: Time Quantum Frames in use. +InverseEnabled: Inverse Frames in use. +NumIndexes: Number of Indexes in the Cluster. +NumFrames: Number of Frames in the Cluster. +NumSlices: Number of Slices in the Cluster. +NumViews: Number of Views in the Cluster. +OpenFiles: Open file handle count. +GoRoutines: Go routine count. + +You can opt-out of the Pilosa diagnostics reporting by setting the `diagnostics` configuration option under `metric` to `0m0s`. + #### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. diff --git a/docs/configuration.md b/docs/configuration.md index 6741bc74b..d786d0513 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,6 +206,19 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "0m15s" ``` +##### Metric Diagnostics Interval + +* Description: Diagnostic reporting interval. To disable diagnostics set to zero. +* Flag: `metric.diagnostics=”60m0s”` +* Env: `PILOSA_METRIC_DIAGNOSTICS=60m0s` +* Config: + + ```toml + [metric] + diagnostics = "60m0s" + ``` + + ##### TLS Certificate * Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions. From 66650ec1f727fd48b425060dc13f488aacc0ab15 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 18 Oct 2017 17:38:20 +0300 Subject: [PATCH 23/48] NewExecutor doesn't return an error; renamed NewClient to NewInternalHTTPClient --- client.go | 4 ++-- client_test.go | 15 +++------------ ctl/common.go | 2 +- executor.go | 4 ++-- fragment.go | 4 ++-- holder.go | 4 ++-- holder_test.go | 5 +---- server.go | 5 +---- server/server_test.go | 6 +++--- test/client.go | 2 +- test/executor.go | 2 +- 11 files changed, 19 insertions(+), 34 deletions(-) diff --git a/client.go b/client.go index 8528e52dd..3c9ff0709 100644 --- a/client.go +++ b/client.go @@ -50,8 +50,8 @@ type InternalHTTPClient struct { HTTPClient *http.Client } -// NewClient returns a new instance of InternalHTTPClient to connect to host. -func NewClient(host string, options *ClientOptions) (*InternalHTTPClient, error) { +// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. +func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPClient, error) { if host == "" { return nil, ErrHostRequired } diff --git a/client_test.go b/client_test.go index 509039618..20f601fe7 100644 --- a/client_test.go +++ b/client_test.go @@ -54,10 +54,7 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e, err := pilosa.NewExecutor(nil) - if err != nil { - t.Fatal(err) - } + e := pilosa.NewExecutor(nil) e.Holder = hldr[0].Holder e.Scheme = cluster.Nodes[0].Scheme e.Host = cluster.Nodes[0].Host @@ -65,10 +62,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e, err := pilosa.NewExecutor(nil) - if err != nil { - t.Fatal(err) - } + e := pilosa.NewExecutor(nil) e.Holder = hldr[1].Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host @@ -76,10 +70,7 @@ func TestClient_MultiNode(t *testing.T) { return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e, err := pilosa.NewExecutor(nil) - if err != nil { - t.Fatal(err) - } + e := pilosa.NewExecutor(nil) e.Holder = hldr[2].Holder e.Scheme = cluster.Nodes[2].Scheme e.Host = cluster.Nodes[2].Host diff --git a/ctl/common.go b/ctl/common.go index 52b4e6e7e..dc64c0bee 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -34,7 +34,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error } clientOptions = &pilosa.ClientOptions{TLS: TLSConfig} } - client, err := pilosa.NewClient(cmd.TLSHost(), clientOptions) + client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), clientOptions) if err != nil { return nil, err } diff --git a/executor.go b/executor.go index 7aee4e472..6cb4aad0d 100644 --- a/executor.go +++ b/executor.go @@ -51,13 +51,13 @@ type Executor struct { } // NewExecutor returns a new instance of Executor. -func NewExecutor(clientOptions *ClientOptions) (*Executor, error) { +func NewExecutor(clientOptions *ClientOptions) *Executor { if clientOptions == nil { clientOptions = &ClientOptions{} } return &Executor{ client: NewClientFromURI(nil, clientOptions), - }, nil + } } // Execute executes a PQL query. diff --git a/fragment.go b/fragment.go index 148fd6a48..e5fbb7988 100644 --- a/fragment.go +++ b/fragment.go @@ -1714,7 +1714,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client, err := NewClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) if err != nil { return err } @@ -1793,7 +1793,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client, err := NewClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) if err != nil { return err } diff --git a/holder.go b/holder.go index f3713faa2..f3b4247d0 100644 --- a/holder.go +++ b/holder.go @@ -515,7 +515,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) if err != nil { return err } @@ -560,7 +560,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewClient(node.Host, s.ClientOptions) + client, err := NewInternalHTTPClient(node.Host, s.ClientOptions) if err != nil { return err } diff --git a/holder_test.go b/holder_test.go index cb33045e6..104bd1bb0 100644 --- a/holder_test.go +++ b/holder_test.go @@ -314,10 +314,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer s.Close() s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e, err := pilosa.NewExecutor(nil) - if err != nil { - t.Fatal(err) - } + e := pilosa.NewExecutor(nil) e.Holder = hldr1.Holder e.Scheme = cluster.Nodes[1].Scheme e.Host = cluster.Nodes[1].Host diff --git a/server.go b/server.go index 3ce2883c3..7c255079e 100644 --- a/server.go +++ b/server.go @@ -164,10 +164,7 @@ func (s *Server) Open() error { s.createDefaultClient() // Create executor for executing queries. - e, err := NewExecutor(&ClientOptions{TLS: s.TLS}) - if err != nil { - return err - } + e := NewExecutor(&ClientOptions{TLS: s.TLS}) e.Holder = s.Holder e.Scheme = s.URI.Scheme() e.Host = s.URI.HostPort() diff --git a/server/server_test.go b/server/server_test.go index f03e3331b..484753a4b 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -53,7 +53,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewClient(m.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil) if err != nil { t.Fatal(err) } @@ -326,7 +326,7 @@ func TestMain_FrameRestore(t *testing.T) { defer m2.Close() // Import from first cluster. - client, err := pilosa.NewClient(m2.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), nil) if err != nil { t.Fatal(err) } else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { @@ -697,7 +697,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewClient(m.Server.URI.HostPort(), nil) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), nil) if err != nil { panic(err) } diff --git a/test/client.go b/test/client.go index d9044df66..4ea2fbbad 100644 --- a/test/client.go +++ b/test/client.go @@ -11,7 +11,7 @@ type Client struct { // MustNewClient returns a new instance of Client. Panic on error. func MustNewClient(host string) *Client { - c, err := pilosa.NewClient(host, nil) + c, err := pilosa.NewInternalHTTPClient(host, nil) if err != nil { panic(err) } diff --git a/test/executor.go b/test/executor.go index 4ef99d351..be370908d 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,7 +15,7 @@ type Executor struct { // NewExecutor returns a new instance of Executor. // The executor always matches the hostname of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor, _ := pilosa.NewExecutor(nil) + executor := pilosa.NewExecutor(nil) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster From 55c37b51eb10ab5963ebcf90ca1170834db18d26 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:39:57 -0500 Subject: [PATCH 24/48] Diagnostics package --- diagnostics/diagnostics.go | 282 ++++++++++++++++++++++++++++++++ diagnostics/diagnostics_test.go | 143 ++++++++++++++++ 2 files changed, 425 insertions(+) create mode 100644 diagnostics/diagnostics.go create mode 100644 diagnostics/diagnostics_test.go diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go new file mode 100644 index 000000000..fc78b94cb --- /dev/null +++ b/diagnostics/diagnostics.go @@ -0,0 +1,282 @@ +package diagnostics + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/pilosa/pilosa" +) + +// TODO: white list of statsd metrics to use +// TODO: unique Cluster ID +// TODO: how should this be disabled, config + +// Default interval to sync diagnostics metrics. +const ( + DefaultDiagnosticsInterval = 10 * time.Second + DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" +) + +type versionResponse struct { + Version string `json:"version"` + Message string `json:"message"` +} + +// Diagnostics represents a client to the Pilosa cluster. +type Diagnostics struct { + mu sync.Mutex + wg sync.WaitGroup + closing chan struct{} + host string + VersionURL string + version string + startTime int64 + start time.Time + + counts map[string]int64 + metrics map[string]string + + client *http.Client + interval time.Duration + + logOutput io.Writer +} + +// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". +func New(host string) *Diagnostics { + return &Diagnostics{ + closing: make(chan struct{}), + host: host, + VersionURL: DefaultVersionCheckURL, + startTime: time.Now().Unix(), + start: time.Now(), + client: http.DefaultClient, + counts: make(map[string]int64), + metrics: make(map[string]string), + interval: DefaultDiagnosticsInterval, + logOutput: ioutil.Discard, + } +} + +// SetVersion of locally running Pilosa Cluster to check against master. +func (d *Diagnostics) SetVersion(v string) { + d.version = v + d.Set("Version", v, 1.0) +} + +// schedule start the diagnostics service ticker. +func (d *Diagnostics) schedule() { + ticker := time.NewTicker(d.interval) + defer ticker.Stop() + + for { + select { + case <-d.closing: + return + case <-ticker.C: + d.CheckVersion() + d.Flush() + } + } +} + +// Flush sends the current metrics. +func (d *Diagnostics) Flush() error { + d.mu.Lock() + d.metrics["uptime"] = strconv.FormatInt((time.Now().Unix() - d.startTime), 10) + + buf, _ := d.MarshalJSON() + d.Reset() + d.mu.Unlock() + + // d.logger().Println(string(buf)) + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // TODO verify response + // Read response into buffer. + // body, err := ioutil.ReadAll(resp.Body) + // if err != nil { + // return err + // } + + // TODO circuit breaker + return nil +} + +// Reset clears the incremented metrics. +func (d *Diagnostics) Reset() { + d.counts = make(map[string]int64) +} + +// Open starts the diagnostics metric go routine. +func (d *Diagnostics) Open() { + d.wg.Add(1) + go func() { defer d.wg.Done(); d.schedule() }() +} + +// Close notify goroutine to stop. +func (d *Diagnostics) Close() error { + close(d.closing) + d.wg.Wait() + return nil +} + +// CheckVersion of the local build against Pilosa master. +func (d *Diagnostics) CheckVersion() error { + var rsp versionResponse + req, err := http.NewRequest("GET", d.VersionURL, nil) + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return fmt.Errorf("json decode: %s", err) + } + + if err := d.CompareVersion(rsp.Version); err != nil { + d.logger().Printf("%s\n", err.Error()) + } + + return nil +} + +// CompareVersion check version strings. +func (d *Diagnostics) CompareVersion(value string) error { + currentVersion := VersionSegments(value) + localVersion := VersionSegments(d.version) + + if localVersion[0] < currentVersion[0] { //Major + return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Major release is %s", d.version, value) + } else if localVersion[1] < currentVersion[1] { // Minor + return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Minor release is %s", d.version, value) + } else if localVersion[2] < currentVersion[2] { // Patch + return fmt.Errorf("There is a new patch relese of Pilosa availbale: %s", value) + } + + return nil +} + +// MarshalJSON custom marshall string and int maps together. +func (d *Diagnostics) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString("{") + length := len(d.counts) + count := 0 + + for key, value := range d.counts { + jsonValue, err := json.Marshal(value) + if err != nil { + return nil, err + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) + count++ + if count < length { + buffer.WriteString(",") + } + } + if length > 0 { + buffer.WriteString(",") + } + length = len(d.metrics) + count = 0 + for key, value := range d.metrics { + jsonValue, err := json.Marshal(value) + if err != nil { + return nil, err + } + buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) + count++ + if count < length { + buffer.WriteString(",") + } + } + + buffer.WriteString("}") + return buffer.Bytes(), nil +} + +// Stats interface implementation. + +// Tags no-op. +func (d *Diagnostics) Tags() []string { + return nil +} + +// WithTags no-op. +func (d *Diagnostics) WithTags(tags ...string) pilosa.StatsClient { + return d +} + +// Count tracks the number of times something occurs per diagnostic period. +func (d *Diagnostics) Count(name string, value int64, rate float64) { + d.mu.Lock() + defer d.mu.Unlock() + d.counts[name] += value +} + +// CountWithCustomTags Tracks the number of times something occurs per diagnostic period. +func (d *Diagnostics) CountWithCustomTags(name string, value int64, rate float64, tags []string) { + d.mu.Lock() + defer d.mu.Unlock() + d.counts[name] += value +} + +// Gauge records the value of a metric. +func (d *Diagnostics) Gauge(name string, value float64, rate float64) { + d.Set(name, strconv.FormatFloat(value, 'f', -1, 64), rate) +} + +// Histogram is a no-op. +func (d *Diagnostics) Histogram(name string, value float64, rate float64) { +} + +// Set adds a key value metric. +func (d *Diagnostics) Set(name string, value string, rate float64) { + d.mu.Lock() + defer d.mu.Unlock() + d.metrics[name] = value +} + +// Timing no-op. +func (d *Diagnostics) Timing(name string, value time.Duration, rate float64) { +} + +// SetLogger Set the logger output type. +func (d *Diagnostics) SetLogger(logger io.Writer) { + d.logOutput = logger +} + +// logger returns a logger that writes to LogOutput. +func (d *Diagnostics) logger() *log.Logger { + return log.New(d.logOutput, "", log.LstdFlags) +} + +// VersionSegments returns the numeric segments of the version as a slice of ints. +func VersionSegments(segments string) []int { + segments = strings.Trim(segments, "v") + segments = strings.Split(segments, "-")[0] + s := strings.Split(segments, ".") + segmentSlice := make([]int, len(s)) + for i, v := range s { + segmentSlice[i], _ = strconv.Atoi(v) + } + return segmentSlice +} diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go new file mode 100644 index 000000000..85cdc56d6 --- /dev/null +++ b/diagnostics/diagnostics_test.go @@ -0,0 +1,143 @@ +package diagnostics_test + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/pilosa/pilosa/diagnostics" +) + +func TestDiagnosticsClient(t *testing.T) { + // Mock server. + server := httptest.NewServer(nil) + defer server.Close() + + // Create a new client. + d := diagnostics.New(server.URL) + d.SetLogger(ioutil.Discard) + defer d.Close() + + dur, _ := time.ParseDuration("123us") + d.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) + d.Count("cc", 1, 1.0) + d.Gauge("gg", 10, 1.0) + d.Histogram("hh", 1, 1.0) + d.Timing("tt", dur, 1.0) + d.Set("ss", "ss", 1.0) + + d1 := d.WithTags("test") + if !reflect.DeepEqual(d, d1) { + t.Fatalf("Diagnostics is a singleton") + } + + if s := d.Tags(); s != nil { + t.Fatalf("No Diagnostics Tags") + } + + data, err := d.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + // Test the recorded metrics, note that some types are skipped. + var eq bool + output1 := []byte(`{"ct":1,"cc":1,"gg":"10","ss":"ss"}`) + if eq, err = compareJSON(data, output1); err != nil { + t.Fatal(err) + } + if !eq { + t.Fatalf("unexpected diagnostics: %+v", string(data)) + } + + // Test the metrics after a flush. + d.Flush() + data, err = d.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + output2 := []byte(`{"gg":"10","ss":"ss","uptime":"0"}`) + if eq, err = compareJSON(data, output2); err != nil { + t.Fatal(err) + } + if !eq { + t.Fatalf("unexpected diagnostics after flush: %+v", string(data)) + } +} + +func TestDiagnosticsVersion_Parse(t *testing.T) { + version := "0.1.1" + vs := diagnostics.VersionSegments(version) + + output := []int{0, 1, 1} + if !reflect.DeepEqual(vs, output) { + t.Fatalf("unexpected version: %+v", vs) + } +} + +func TestDiagnosticsVersion_Compare(t *testing.T) { + d := diagnostics.New("localhost:10101") + defer d.Close() + + version := "0.1.1" + d.SetVersion(version) + + err := d.CompareVersion("1.7.0") + if !strings.Contains(err.Error(), "The latest Major release is") { + t.Fatalf("Expected Major Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.7.0") + if !strings.Contains(err.Error(), "The latest Minor release is") { + t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.1.2") + if !strings.Contains(err.Error(), "There is a new patch relese of Pilosa") { + t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) + } + err = d.CompareVersion("0.1.1") + if err != nil { + t.Fatalf("Versions should match") + } +} + +func TestDiagnosticsVersion_Check(t *testing.T) { + // Mock server. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(versionResponse{ + Version: "1.1.1", + }) + })) + defer server.Close() + + // Create a new client. + d := diagnostics.New("localhost:10101") + defer d.Close() + + version := "0.1.1" + d.SetVersion(version) + d.VersionURL = server.URL + + d.CheckVersion() +} + +type versionResponse struct { + Version string `json:"version"` +} + +func compareJSON(a, b []byte) (bool, error) { + var j1, j2 interface{} + if err := json.Unmarshal(a, &j1); err != nil { + return false, err + } + if err := json.Unmarshal(b, &j2); err != nil { + return false, err + } + return reflect.DeepEqual(j1, j2), nil +} From dbeec1d678c34f06927b11d7e08c26c5d34f65f5 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:40:55 -0500 Subject: [PATCH 25/48] add Open/Close interface to Stats packages --- stats.go | 36 ++++++++++++++++++++++++++++++++++-- stats_test.go | 2 ++ statsd/statsd.go | 3 +++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/stats.go b/stats.go index 71f0d56b1..998b84511 100644 --- a/stats.go +++ b/stats.go @@ -58,6 +58,12 @@ type StatsClient interface { // SetLogger Set the logger output type SetLogger(logger io.Writer) + + // Starts the service + Open() + + // Closes the client + Close() error } // NopStatsClient represents a client that doesn't do anything. @@ -74,6 +80,8 @@ func (c *nopStatsClient) Histogram(name string, value float64, rate float64) func (c *nopStatsClient) Set(name string, value string, rate float64) {} func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} func (c *nopStatsClient) SetLogger(logger io.Writer) {} +func (c *nopStatsClient) Open() {} +func (c *nopStatsClient) Close() error { return nil } // ExpvarStatsClient writes stats out to expvars. type ExpvarStatsClient struct { @@ -145,10 +153,16 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 c.mu.Unlock() } -// SetLogger has no logger +// SetLogger has no logger. func (c *ExpvarStatsClient) SetLogger(logger io.Writer) { } +// Open no-op. +func (c *ExpvarStatsClient) Open() {} + +// Close no-op. +func (c *ExpvarStatsClient) Close() error { return nil } + // MultiStatsClient joins multiple stats clients together. type MultiStatsClient []StatsClient @@ -211,13 +225,31 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) } } -// SetLogger Sets the StatsD logger output type +// SetLogger Sets the StatsD logger output type. func (a MultiStatsClient) SetLogger(logger io.Writer) { for _, c := range a { c.SetLogger(logger) } } +// Open starts the stat service. +func (a MultiStatsClient) Open() { + for _, c := range a { + c.Open() + } +} + +// Close shuts down the stats clients. +func (a MultiStatsClient) Close() error { + for _, c := range a { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + // UnionStringSlice returns a sorted set of tags which combine a & b. func UnionStringSlice(a, b []string) []string { // Sort both sets first. diff --git a/stats_test.go b/stats_test.go index 7ed679f21..07254a702 100644 --- a/stats_test.go +++ b/stats_test.go @@ -344,3 +344,5 @@ func (c *MockStats) Histogram(name string, value float64, rate float64) {} func (c *MockStats) Set(name string, value string, rate float64) {} func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} func (c *MockStats) SetLogger(logger io.Writer) {} +func (c *MockStats) Open() {} +func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go index e55b190ad..d46dd637f 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -58,6 +58,9 @@ func NewStatsClient(host string) (*StatsClient, error) { }, nil } +// Open no-op +func (c *StatsClient) Open() {} + // Close closes the connection to the agent. func (c *StatsClient) Close() error { return c.client.Close() From bf2827b2738b9a780f961b244422bac5db260c8c Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:21 -0500 Subject: [PATCH 26/48] Using MultiStatsClient add Diagnostics --- server/server.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/server.go b/server/server.go index d1063f306..0ce1680c2 100644 --- a/server/server.go +++ b/server/server.go @@ -31,6 +31,7 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -41,7 +42,8 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" + DefaultDataDir = "~/.pilosa" + DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" ) // Command represents the state of the pilosa server command. @@ -245,12 +247,20 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { + ms := make(pilosa.MultiStatsClient, 1) + d := diagnostics.New(DefaultDiagnosticServer) + d.SetVersion(pilosa.Version) + ms[0] = d + switch name { case "expvar": - return pilosa.NewExpvarStatsClient(), nil + ms = append(ms, pilosa.NewExpvarStatsClient()) case "statsd": - return statsd.NewStatsClient(host) - default: - return pilosa.NopStatsClient, nil + r, err := statsd.NewStatsClient(host) + if err != nil { + return nil, err + } + ms = append(ms, r) } + return ms, nil } From cf543c9641caa850846df44e7baf261bb5d8dfc1 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:35 -0500 Subject: [PATCH 27/48] Add some new Diagnostics metrics --- holder.go | 3 +++ server.go | 24 +++++++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/holder.go b/holder.go index f3713faa2..d85f16814 100644 --- a/holder.go +++ b/holder.go @@ -123,11 +123,14 @@ func (h *Holder) Open() error { h.wg.Add(1) go func() { defer h.wg.Done(); h.monitorCacheFlush() }() + h.Stats.Open() return nil } // Close closes all open fragments. func (h *Holder) Close() error { + h.Stats.Close() + // Notify goroutines of closing and wait for completion. close(h.closing) h.wg.Wait() diff --git a/server.go b/server.go index 773cf9025..d74b7ad3d 100644 --- a/server.go +++ b/server.go @@ -223,10 +223,10 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } +// Logger returns a logger that writes to LogOutput func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { - t := time.Now() ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() @@ -240,7 +240,7 @@ func (s *Server) monitorAntiEntropy() { case <-ticker.C: s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } - + t := time.Now() s.Logger().Printf("holder sync beginning") // Initialize syncer with local holder and remote client. @@ -259,9 +259,9 @@ func (s *Server) monitorAntiEntropy() { // Record successful sync in log. s.Logger().Printf("holder sync complete") + dif := time.Since(t) + s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } - dif := time.Since(t) - s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. @@ -509,7 +509,13 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - // Disable metrics when poll interval is zero + s.Holder.Stats.Set("Host", s.Host, 1.0) + s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) + s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) + s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) + // TODO should we force this to run for diagnostics? + + // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return } @@ -529,18 +535,18 @@ func (s *Server) monitorRuntime() { case <-s.closing: return case <-gcn.AfterGC(): - // GC just ran + // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: } - // Record the number of go routines + // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - // Open File handles + // Open File handles. s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0) - // Runtime memory metrics + // Runtime memory metrics. runtime.ReadMemStats(&m) s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) From 3288b015fc182255a8512df844f4354c95362627 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Sep 2017 17:58:13 -0500 Subject: [PATCH 28/48] Add basic diagnostics benchmark --- diagnostics/diagnostics_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 85cdc56d6..1a811df9c 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "runtime" "strings" "testing" "time" @@ -141,3 +142,26 @@ func compareJSON(a, b []byte) (bool, error) { } return reflect.DeepEqual(j1, j2), nil } + +func BenchmarkDiagnostics(b *testing.B) { + // Mock server. + server := httptest.NewServer(nil) + defer server.Close() + + // Create a new client. + d := diagnostics.New(server.URL) + d.SetLogger(ioutil.Discard) + defer d.Close() + + prev := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(prev) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + d.Count("cc", 1, 1.0) + d.Gauge("gg", 10, 1.0) + } + }) +} From 8b87886a6860814affd8e3010ac24ed7e3622de8 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 16 Oct 2017 12:08:48 -0500 Subject: [PATCH 29/48] simplifying the diagnostics client. Using circuit breaker to manage the diagnostics http connection. --- diagnostics/diagnostics.go | 142 +++++++------------------------- diagnostics/diagnostics_test.go | 31 ++----- server.go | 70 +++++++++++++--- server/server.go | 20 ++--- 4 files changed, 105 insertions(+), 158 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index fc78b94cb..761e7adca 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -13,16 +13,14 @@ import ( "sync" "time" - "github.com/pilosa/pilosa" + "github.com/sony/gobreaker" ) -// TODO: white list of statsd metrics to use // TODO: unique Cluster ID -// TODO: how should this be disabled, config // Default interval to sync diagnostics metrics. const ( - DefaultDiagnosticsInterval = 10 * time.Second + DefaultDiagnosticsInterval = 1 * time.Hour DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" ) @@ -42,17 +40,20 @@ type Diagnostics struct { startTime int64 start time.Time - counts map[string]int64 - metrics map[string]string + metrics map[string]interface{} client *http.Client interval time.Duration + cb *gobreaker.CircuitBreaker logOutput io.Writer } // New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". func New(host string) *Diagnostics { + var st gobreaker.Settings + st.Timeout = DefaultDiagnosticsInterval * 2 + return &Diagnostics{ closing: make(chan struct{}), host: host, @@ -60,17 +61,17 @@ func New(host string) *Diagnostics { startTime: time.Now().Unix(), start: time.Now(), client: http.DefaultClient, - counts: make(map[string]int64), - metrics: make(map[string]string), + metrics: make(map[string]interface{}), interval: DefaultDiagnosticsInterval, logOutput: ioutil.Discard, + cb: gobreaker.NewCircuitBreaker(st), } } // SetVersion of locally running Pilosa Cluster to check against master. func (d *Diagnostics) SetVersion(v string) { d.version = v - d.Set("Version", v, 1.0) + d.Set("Version", v) } // schedule start the diagnostics service ticker. @@ -92,35 +93,28 @@ func (d *Diagnostics) schedule() { // Flush sends the current metrics. func (d *Diagnostics) Flush() error { d.mu.Lock() - d.metrics["uptime"] = strconv.FormatInt((time.Now().Unix() - d.startTime), 10) - - buf, _ := d.MarshalJSON() - d.Reset() + d.metrics["uptime"] = (time.Now().Unix() - d.startTime) + buf, _ := d.Encode() d.mu.Unlock() - // d.logger().Println(string(buf)) - req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - resp, err := d.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() + _, err := d.cb.Execute(func() (interface{}, error) { + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() - // TODO verify response - // Read response into buffer. - // body, err := ioutil.ReadAll(resp.Body) - // if err != nil { - // return err - // } + // TODO verify response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, nil + }) - // TODO circuit breaker - return nil -} - -// Reset clears the incremented metrics. -func (d *Diagnostics) Reset() { - d.counts = make(map[string]int64) + return err } // Open starts the diagnostics metric go routine. @@ -175,90 +169,18 @@ func (d *Diagnostics) CompareVersion(value string) error { return nil } -// MarshalJSON custom marshall string and int maps together. -func (d *Diagnostics) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString("{") - length := len(d.counts) - count := 0 - - for key, value := range d.counts { - jsonValue, err := json.Marshal(value) - if err != nil { - return nil, err - } - buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) - count++ - if count < length { - buffer.WriteString(",") - } - } - if length > 0 { - buffer.WriteString(",") - } - length = len(d.metrics) - count = 0 - for key, value := range d.metrics { - jsonValue, err := json.Marshal(value) - if err != nil { - return nil, err - } - buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, string(jsonValue))) - count++ - if count < length { - buffer.WriteString(",") - } - } - - buffer.WriteString("}") - return buffer.Bytes(), nil -} - -// Stats interface implementation. - -// Tags no-op. -func (d *Diagnostics) Tags() []string { - return nil -} - -// WithTags no-op. -func (d *Diagnostics) WithTags(tags ...string) pilosa.StatsClient { - return d -} - -// Count tracks the number of times something occurs per diagnostic period. -func (d *Diagnostics) Count(name string, value int64, rate float64) { - d.mu.Lock() - defer d.mu.Unlock() - d.counts[name] += value -} - -// CountWithCustomTags Tracks the number of times something occurs per diagnostic period. -func (d *Diagnostics) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - d.mu.Lock() - defer d.mu.Unlock() - d.counts[name] += value -} - -// Gauge records the value of a metric. -func (d *Diagnostics) Gauge(name string, value float64, rate float64) { - d.Set(name, strconv.FormatFloat(value, 'f', -1, 64), rate) -} - -// Histogram is a no-op. -func (d *Diagnostics) Histogram(name string, value float64, rate float64) { +// Encode metrics maps into the json message format +func (d *Diagnostics) Encode() ([]byte, error) { + return json.Marshal(d.metrics) } // Set adds a key value metric. -func (d *Diagnostics) Set(name string, value string, rate float64) { +func (d *Diagnostics) Set(name string, value interface{}) { d.mu.Lock() defer d.mu.Unlock() d.metrics[name] = value } -// Timing no-op. -func (d *Diagnostics) Timing(name string, value time.Duration, rate float64) { -} - // SetLogger Set the logger output type. func (d *Diagnostics) SetLogger(logger io.Writer) { d.logOutput = logger diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 1a811df9c..d780a276f 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -9,7 +9,6 @@ import ( "runtime" "strings" "testing" - "time" "github.com/pilosa/pilosa/diagnostics" ) @@ -24,31 +23,17 @@ func TestDiagnosticsClient(t *testing.T) { d.SetLogger(ioutil.Discard) defer d.Close() - dur, _ := time.ParseDuration("123us") - d.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - d.Count("cc", 1, 1.0) - d.Gauge("gg", 10, 1.0) - d.Histogram("hh", 1, 1.0) - d.Timing("tt", dur, 1.0) - d.Set("ss", "ss", 1.0) + d.Set("gg", 10) + d.Set("ss", "ss") - d1 := d.WithTags("test") - if !reflect.DeepEqual(d, d1) { - t.Fatalf("Diagnostics is a singleton") - } - - if s := d.Tags(); s != nil { - t.Fatalf("No Diagnostics Tags") - } - - data, err := d.MarshalJSON() + data, err := d.Encode() if err != nil { t.Fatal(err) } // Test the recorded metrics, note that some types are skipped. var eq bool - output1 := []byte(`{"ct":1,"cc":1,"gg":"10","ss":"ss"}`) + output1 := []byte(`{"gg":10,"ss":"ss"}`) if eq, err = compareJSON(data, output1); err != nil { t.Fatal(err) } @@ -58,12 +43,12 @@ func TestDiagnosticsClient(t *testing.T) { // Test the metrics after a flush. d.Flush() - data, err = d.MarshalJSON() + data, err = d.Encode() if err != nil { t.Fatal(err) } - output2 := []byte(`{"gg":"10","ss":"ss","uptime":"0"}`) + output2 := []byte(`{"gg":10,"ss":"ss","uptime":0}`) if eq, err = compareJSON(data, output2); err != nil { t.Fatal(err) } @@ -160,8 +145,8 @@ func BenchmarkDiagnostics(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - d.Count("cc", 1, 1.0) - d.Gauge("gg", 10, 1.0) + d.Set("cc", 1) + d.Set("gg", "test") } }) } diff --git a/server.go b/server.go index d74b7ad3d..e6bbfe1c0 100644 --- a/server.go +++ b/server.go @@ -34,6 +34,7 @@ import ( "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" ) @@ -41,6 +42,7 @@ import ( const ( DefaultAntiEntropyInterval = 10 * time.Minute DefaultPollingInterval = 60 * time.Second + DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Server represents a holder wrapped by a running HTTP server. @@ -59,14 +61,16 @@ type Server struct { // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". - Network string - URI *URI - Cluster *Cluster + Network string + URI *URI + Cluster *Cluster + diagnostics *diagnostics.Diagnostics // Background monitoring intervals. AntiEntropyInterval time.Duration PollingInterval time.Duration MetricInterval time.Duration + DiagnosticInterval time.Duration // TLS configuration TLS *tls.Config @@ -88,12 +92,14 @@ func NewServer() *Server { Handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, + diagnostics: diagnostics.New(DefaultDiagnosticServer), Network: "tcp", AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, + DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval LogOutput: os.Stderr, } @@ -191,10 +197,11 @@ func (s *Server) Open() error { }() // Start background monitoring. - s.wg.Add(3) + s.wg.Add(4) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() go func() { defer s.wg.Done(); s.monitorMaxSlices() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() + go func() { defer s.wg.Done(); s.monitorDiagnostics() }() return nil } @@ -507,14 +514,57 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint return pb.MaxSlices, nil } +// monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info. +func (s *Server) monitorDiagnostics() { + if s.DiagnosticInterval <= 0 { + return + } + + s.diagnostics.SetLogger(s.LogOutput) + s.diagnostics.SetVersion(Version) + s.diagnostics.Set("Host", s.Host) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) + s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) + s.diagnostics.Set("NumCPU", runtime.NumCPU()) + // TODO: unique cluster ID + + ticker := time.NewTicker(s.DiagnosticInterval) + defer ticker.Stop() + + for { + // Wait for tick or a close. + select { + case <-s.closing: + return + case <-ticker.C: + numFrames := 0 + numSlices := uint64(0) + for _, index := range s.Holder.Indexes() { + numSlices += index.MaxSlice() + 1 + for _, f := range index.Frames() { + numFrames++ + if f.rangeEnabled { + s.diagnostics.Set("BSIEnabled", true) + } + if f.timeQuantum != "" { + s.diagnostics.Set("TimeQuantumEnabled", true) + } + } + } + + s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) + s.diagnostics.Set("NumFrames", numFrames) + s.diagnostics.Set("NumSlices", numSlices) + s.diagnostics.Set("OpenFiles", CountOpenFiles()) + s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) + s.diagnostics.CheckVersion() + s.diagnostics.Flush() + } + } +} + // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - s.Holder.Stats.Set("Host", s.Host, 1.0) - s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) - s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) - s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) - // TODO should we force this to run for diagnostics? - // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return diff --git a/server/server.go b/server/server.go index 0ce1680c2..d1063f306 100644 --- a/server/server.go +++ b/server/server.go @@ -31,7 +31,6 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -42,8 +41,7 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" + DefaultDataDir = "~/.pilosa" ) // Command represents the state of the pilosa server command. @@ -247,20 +245,12 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { - ms := make(pilosa.MultiStatsClient, 1) - d := diagnostics.New(DefaultDiagnosticServer) - d.SetVersion(pilosa.Version) - ms[0] = d - switch name { case "expvar": - ms = append(ms, pilosa.NewExpvarStatsClient()) + return pilosa.NewExpvarStatsClient(), nil case "statsd": - r, err := statsd.NewStatsClient(host) - if err != nil { - return nil, err - } - ms = append(ms, r) + return statsd.NewStatsClient(host) + default: + return pilosa.NopStatsClient, nil } - return ms, nil } From 32f1558737e9652f85725c713e3e97152ba1b1d2 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:21 -0500 Subject: [PATCH 30/48] Using MultiStatsClient add Diagnostics --- server/server.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/server.go b/server/server.go index d1063f306..0ce1680c2 100644 --- a/server/server.go +++ b/server/server.go @@ -31,6 +31,7 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -41,7 +42,8 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" + DefaultDataDir = "~/.pilosa" + DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" ) // Command represents the state of the pilosa server command. @@ -245,12 +247,20 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { + ms := make(pilosa.MultiStatsClient, 1) + d := diagnostics.New(DefaultDiagnosticServer) + d.SetVersion(pilosa.Version) + ms[0] = d + switch name { case "expvar": - return pilosa.NewExpvarStatsClient(), nil + ms = append(ms, pilosa.NewExpvarStatsClient()) case "statsd": - return statsd.NewStatsClient(host) - default: - return pilosa.NopStatsClient, nil + r, err := statsd.NewStatsClient(host) + if err != nil { + return nil, err + } + ms = append(ms, r) } + return ms, nil } From f575605bbea79c3b5d834c4a3f61b4bfef3284b9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 26 Sep 2017 22:42:35 -0500 Subject: [PATCH 31/48] Add some new Diagnostics metrics --- server.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index e6bbfe1c0..2276e4e6e 100644 --- a/server.go +++ b/server.go @@ -565,6 +565,12 @@ func (s *Server) monitorDiagnostics() { // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { + s.Holder.Stats.Set("Host", s.Host, 1.0) + s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) + s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) + s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) + // TODO should we force this to run for diagnostics? + // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return @@ -614,7 +620,7 @@ func (s *Server) createDefaultClient() { s.defaultClient = &http.Client{Transport: transport} } -// CountOpenFiles on opperating systems that support lsof +// CountOpenFiles on operating systems that support lsof. func CountOpenFiles() int { count := 0 From 3b4892f9267e89160de94529bc27201289f2aa58 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 16 Oct 2017 12:08:48 -0500 Subject: [PATCH 32/48] simplifying the diagnostics client. Using circuit breaker to manage the diagnostics http connection. --- server.go | 6 ------ server/server.go | 20 +++++--------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/server.go b/server.go index 2276e4e6e..cab93b4b5 100644 --- a/server.go +++ b/server.go @@ -565,12 +565,6 @@ func (s *Server) monitorDiagnostics() { // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { - s.Holder.Stats.Set("Host", s.Host, 1.0) - s.Holder.Stats.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","), 1.0) - s.Holder.Stats.Set("NumNodes", strconv.Itoa(len(s.Cluster.Nodes)), 1.0) - s.Holder.Stats.Set("NumCPU", strconv.Itoa(runtime.NumCPU()), 1.0) - // TODO should we force this to run for diagnostics? - // Disable metrics when poll interval is zero. if s.MetricInterval <= 0 { return diff --git a/server/server.go b/server/server.go index 0ce1680c2..d1063f306 100644 --- a/server/server.go +++ b/server/server.go @@ -31,7 +31,6 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" ) @@ -42,8 +41,7 @@ func init() { const ( // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - DefaultDiagnosticServer = "https://requestb.in/w3uukzw3" + DefaultDataDir = "~/.pilosa" ) // Command represents the state of the pilosa server command. @@ -247,20 +245,12 @@ func (m *Command) Close() error { // NewStatsClient creates a stats client from the config func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { - ms := make(pilosa.MultiStatsClient, 1) - d := diagnostics.New(DefaultDiagnosticServer) - d.SetVersion(pilosa.Version) - ms[0] = d - switch name { case "expvar": - ms = append(ms, pilosa.NewExpvarStatsClient()) + return pilosa.NewExpvarStatsClient(), nil case "statsd": - r, err := statsd.NewStatsClient(host) - if err != nil { - return nil, err - } - ms = append(ms, r) + return statsd.NewStatsClient(host) + default: + return pilosa.NopStatsClient, nil } - return ms, nil } From 4dc333be2b5d16f2f41242880896809bff9d2dae Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 16 Oct 2017 14:51:15 -0500 Subject: [PATCH 33/48] Flush diagnostics at startup, and then on each interval --- server.go | 57 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/server.go b/server.go index cab93b4b5..430bce7dc 100644 --- a/server.go +++ b/server.go @@ -62,7 +62,7 @@ type Server struct { // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". Network string - URI *URI + URI *URI Cluster *Cluster diagnostics *diagnostics.Diagnostics @@ -99,7 +99,7 @@ func NewServer() *Server { AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, - DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval + DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval, LogOutput: os.Stderr, } @@ -522,43 +522,48 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.SetLogger(s.LogOutput) s.diagnostics.SetVersion(Version) - s.diagnostics.Set("Host", s.Host) + s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) // TODO: unique cluster ID + // Flush the diagnostics metrics at startup, then on each tick interval + flush := func() { + numFrames := 0 + numSlices := uint64(0) + for _, index := range s.Holder.Indexes() { + numSlices += index.MaxSlice() + 1 + for _, f := range index.Frames() { + numFrames++ + if f.rangeEnabled { + s.diagnostics.Set("BSIEnabled", true) + } + if f.timeQuantum != "" { + s.diagnostics.Set("TimeQuantumEnabled", true) + } + } + } + + s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) + s.diagnostics.Set("NumFrames", numFrames) + s.diagnostics.Set("NumSlices", numSlices) + s.diagnostics.Set("OpenFiles", CountOpenFiles()) + s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) + s.diagnostics.CheckVersion() + s.diagnostics.Flush() + } + ticker := time.NewTicker(s.DiagnosticInterval) defer ticker.Stop() - + flush() for { // Wait for tick or a close. select { case <-s.closing: return case <-ticker.C: - numFrames := 0 - numSlices := uint64(0) - for _, index := range s.Holder.Indexes() { - numSlices += index.MaxSlice() + 1 - for _, f := range index.Frames() { - numFrames++ - if f.rangeEnabled { - s.diagnostics.Set("BSIEnabled", true) - } - if f.timeQuantum != "" { - s.diagnostics.Set("TimeQuantumEnabled", true) - } - } - } - - s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) - s.diagnostics.Set("NumFrames", numFrames) - s.diagnostics.Set("NumSlices", numSlices) - s.diagnostics.Set("OpenFiles", CountOpenFiles()) - s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) - s.diagnostics.CheckVersion() - s.diagnostics.Flush() + flush() } } } From 96ab16d2e09692ff95d86716576b7874aa976673 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 17 Oct 2017 13:49:35 -0500 Subject: [PATCH 34/48] Set the diagnostics interval and circuit breaker timeout at Open() --- diagnostics/diagnostics.go | 25 ++++++++++++++----------- diagnostics/diagnostics_test.go | 2 ++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index 761e7adca..35d37f6cc 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -18,10 +18,9 @@ import ( // TODO: unique Cluster ID -// Default interval to sync diagnostics metrics. +// Default version check URL. const ( - DefaultDiagnosticsInterval = 1 * time.Hour - DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" + DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" ) type versionResponse struct { @@ -51,8 +50,6 @@ type Diagnostics struct { // New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". func New(host string) *Diagnostics { - var st gobreaker.Settings - st.Timeout = DefaultDiagnosticsInterval * 2 return &Diagnostics{ closing: make(chan struct{}), @@ -62,9 +59,7 @@ func New(host string) *Diagnostics { start: time.Now(), client: http.DefaultClient, metrics: make(map[string]interface{}), - interval: DefaultDiagnosticsInterval, logOutput: ioutil.Discard, - cb: gobreaker.NewCircuitBreaker(st), } } @@ -74,6 +69,11 @@ func (d *Diagnostics) SetVersion(v string) { d.Set("Version", v) } +// SetInterval of the diagnostic go routine and match with the circuit breaker timeout. +func (d *Diagnostics) SetInterval(i time.Duration) { + d.interval = i +} + // schedule start the diagnostics service ticker. func (d *Diagnostics) schedule() { ticker := time.NewTicker(d.interval) @@ -117,10 +117,13 @@ func (d *Diagnostics) Flush() error { return err } -// Open starts the diagnostics metric go routine. +// Open configures the circuit breaker used by the HTTP client. func (d *Diagnostics) Open() { - d.wg.Add(1) - go func() { defer d.wg.Done(); d.schedule() }() + var st gobreaker.Settings + if d.interval > 0 { + st.Timeout = d.interval * 2 + } + d.cb = gobreaker.NewCircuitBreaker(st) } // Close notify goroutine to stop. @@ -169,7 +172,7 @@ func (d *Diagnostics) CompareVersion(value string) error { return nil } -// Encode metrics maps into the json message format +// Encode metrics maps into the json message format. func (d *Diagnostics) Encode() ([]byte, error) { return json.Marshal(d.metrics) } diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index d780a276f..81d6b7cf6 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -21,6 +21,7 @@ func TestDiagnosticsClient(t *testing.T) { // Create a new client. d := diagnostics.New(server.URL) d.SetLogger(ioutil.Discard) + d.Open() defer d.Close() d.Set("gg", 10) @@ -69,6 +70,7 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { func TestDiagnosticsVersion_Compare(t *testing.T) { d := diagnostics.New("localhost:10101") + d.Open() defer d.Close() version := "0.1.1" From 8b4d2d6c85b6e735b26212ccf8b7cc1a12c7ee91 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 17 Oct 2017 13:50:28 -0500 Subject: [PATCH 35/48] Config option for diagnostics interval. Default to 1 hour --- config.go | 11 ++++++++--- ctl/server.go | 1 + server.go | 4 +++- server/server.go | 2 ++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index fe93be19e..68a351523 100644 --- a/config.go +++ b/config.go @@ -43,6 +43,9 @@ const ( // DefaultMaxWritesPerRequest is the default number of writes per request. DefaultMaxWritesPerRequest = 5000 + + // DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. + DefaultDiagnosticsInterval = 1 * time.Hour ) // ClusterTypes set of cluster types. @@ -88,9 +91,10 @@ type Config struct { LogPath string `toml:"log-path"` Metric struct { - Service string `toml:"service"` - Host string `toml:"host"` - PollInterval Duration `toml:"poll-interval"` + Service string `toml:"service"` + Host string `toml:"host"` + PollInterval Duration `toml:"poll-interval"` + DiagnosticInterval Duration `toml:"diagnostics"` } `toml:"metric"` TLS TLSConfig @@ -108,6 +112,7 @@ func NewConfig() *Config { c.Cluster.Hosts = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) c.Metric.Service = DefaultMetrics + c.Metric.DiagnosticInterval = Duration(DefaultDiagnosticsInterval) c.TLS = TLSConfig{} return c } diff --git a/ctl/server.go b/ctl/server.go index a0c080729..a39c86e3e 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -41,6 +41,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") + flags.DurationVarP((*time.Duration)(&srv.Config.Metric.DiagnosticInterval), "metric.diagnostics", "", time.Hour*1, "Diagnostic reporting interval back to Pilosa.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) } diff --git a/server.go b/server.go index 430bce7dc..d9b4209fc 100644 --- a/server.go +++ b/server.go @@ -99,7 +99,7 @@ func NewServer() *Server { AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, MetricInterval: 0, - DiagnosticInterval: diagnostics.DefaultDiagnosticsInterval, + DiagnosticInterval: 0, LogOutput: os.Stderr, } @@ -522,6 +522,8 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.SetLogger(s.LogOutput) s.diagnostics.SetVersion(Version) + s.diagnostics.SetInterval(s.DiagnosticInterval) + s.diagnostics.Open() s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) diff --git a/server/server.go b/server/server.go index d1063f306..fb18137ad 100644 --- a/server/server.go +++ b/server/server.go @@ -30,6 +30,7 @@ import ( "time" "crypto/tls" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statsd" @@ -143,6 +144,7 @@ func (m *Command) SetupServer() error { m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + m.Server.DiagnosticInterval = time.Duration(m.Config.Metric.DiagnosticInterval) if err != nil { return err } From 0dc10ad1e9c93dc3fa841a18b1c381ad62dc7ffb Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 18 Oct 2017 09:22:55 -0500 Subject: [PATCH 36/48] diagnostics docs --- docs/administration.md | 21 +++++++++++++++++++++ docs/configuration.md | 13 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/administration.md b/docs/administration.md index ed4da61ee..861d7f6a1 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -112,6 +112,27 @@ Note: This will only work when the replication factor is >= 2 - Restart the cluster - Wait for the 1st sync (10 minutes) to validate Index connections +#### Diagnostics + +Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. + +Version: Version string of the build. +Host: Host URI. +Cluster: List of nodes in the Cluster. +NumNodes: Number of nodes in the Cluster. +NumCPU: Number of Cores per Node +BSIEnabled: Bit Slice Index Frames in use. +TimeQuantumEnabled: Time Quantum Frames in use. +InverseEnabled: Inverse Frames in use. +NumIndexes: Number of Indexes in the Cluster. +NumFrames: Number of Frames in the Cluster. +NumSlices: Number of Slices in the Cluster. +NumViews: Number of Views in the Cluster. +OpenFiles: Open file handle count. +GoRoutines: Go routine count. + +You can opt-out of the Pilosa diagnostics reporting by setting the `diagnostics` configuration option under `metric` to `0m0s`. + #### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. diff --git a/docs/configuration.md b/docs/configuration.md index 6741bc74b..d786d0513 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,6 +206,19 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "0m15s" ``` +##### Metric Diagnostics Interval + +* Description: Diagnostic reporting interval. To disable diagnostics set to zero. +* Flag: `metric.diagnostics=”60m0s”` +* Env: `PILOSA_METRIC_DIAGNOSTICS=60m0s` +* Config: + + ```toml + [metric] + diagnostics = "60m0s" + ``` + + ##### TLS Certificate * Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions. From 642bf3180fdbc64689b82341ce158bee1b649ae9 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 18 Oct 2017 21:17:25 +0300 Subject: [PATCH 37/48] Rename NewClientFromURI to NewInternalHTTPClientFromURI --- client.go | 4 ++-- executor.go | 2 +- handler.go | 2 +- server.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index 3c9ff0709..185b09f93 100644 --- a/client.go +++ b/client.go @@ -61,11 +61,11 @@ func NewInternalHTTPClient(host string, options *ClientOptions) (*InternalHTTPCl return nil, err } - client := NewClientFromURI(uri, options) + client := NewInternalHTTPClientFromURI(uri, options) return client, nil } -func NewClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient { +func NewInternalHTTPClientFromURI(defaultURI *URI, options *ClientOptions) *InternalHTTPClient { if options == nil { options = &ClientOptions{} } diff --git a/executor.go b/executor.go index 6cb4aad0d..2f5acf3c0 100644 --- a/executor.go +++ b/executor.go @@ -56,7 +56,7 @@ func NewExecutor(clientOptions *ClientOptions) *Executor { clientOptions = &ClientOptions{} } return &Executor{ - client: NewClientFromURI(nil, clientOptions), + client: NewInternalHTTPClientFromURI(nil, clientOptions), } } diff --git a/handler.go b/handler.go index 811ab14f1..0f224c31d 100644 --- a/handler.go +++ b/handler.go @@ -1506,7 +1506,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Create a client for the remote cluster. - client := NewClientFromURI(host, h.ClientOptions) + client := NewInternalHTTPClientFromURI(host, h.ClientOptions) // Determine the maximum number of slices. maxSlices, err := client.MaxSliceByIndex(r.Context()) diff --git a/server.go b/server.go index 7c255079e..3460bec46 100644 --- a/server.go +++ b/server.go @@ -537,7 +537,7 @@ func (s *Server) createDefaultClient() { if s.TLS != nil { transport.TLSClientConfig = s.TLS } - s.defaultClient = NewClientFromURI(nil, &ClientOptions{TLS: s.TLS}) + s.defaultClient = NewInternalHTTPClientFromURI(nil, &ClientOptions{TLS: s.TLS}) } // CountOpenFiles on opperating systems that support lsof From c9cb92c96b8dcaff4ae9c2bc44df5c765fd1ca00 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 23 Oct 2017 17:34:04 +0300 Subject: [PATCH 38/48] Added Travis's note about InteraClient interface --- client.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client.go b/client.go index 185b09f93..9297f4718 100644 --- a/client.go +++ b/client.go @@ -32,6 +32,7 @@ import ( "time" "crypto/tls" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -1206,6 +1207,12 @@ func nodePathToURL(node *Node, path string) url.URL { } } +// InternalClient should be implemented by any struct that enables any transport between nodes +// TODO: Refactor +// Note from Travis: Typically an interface containing more than two or three methods is an indication that +// something hasn't been architected correctly. +// While I understand that putting the entire Client behind an interface might require this many methods, +// I don't want to let it go unquestioned. type InternalClient interface { MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) From 197e05b04a0baef81a11cc2e7ade1294d0d0b4c1 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 24 Oct 2017 01:37:41 +0300 Subject: [PATCH 39/48] Adds scheme to the node status --- internal/private.pb.go | 867 +++++++++++++++++++++++++++++------------ internal/private.proto | 1 + internal/public.pb.go | 477 +++++++++++++++++------ server.go | 1 + 4 files changed, 972 insertions(+), 374 deletions(-) diff --git a/internal/private.pb.go b/internal/private.pb.go index 786279cf6..e5cc6516f 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -63,6 +62,20 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *IndexMeta) GetColumnLabel() string { + if m != nil { + return m.ColumnLabel + } + return "" +} + +func (m *IndexMeta) GetTimeQuantum() string { + if m != nil { + return m.TimeQuantum + } + return "" +} + type FrameMeta struct { RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` @@ -78,6 +91,48 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +func (m *FrameMeta) GetRowLabel() string { + if m != nil { + return m.RowLabel + } + return "" +} + +func (m *FrameMeta) GetInverseEnabled() bool { + if m != nil { + return m.InverseEnabled + } + return false +} + +func (m *FrameMeta) GetCacheType() string { + if m != nil { + return m.CacheType + } + return "" +} + +func (m *FrameMeta) GetCacheSize() uint32 { + if m != nil { + return m.CacheSize + } + return 0 +} + +func (m *FrameMeta) GetTimeQuantum() string { + if m != nil { + return m.TimeQuantum + } + return "" +} + +func (m *FrameMeta) GetRangeEnabled() bool { + if m != nil { + return m.RangeEnabled + } + return false +} + func (m *FrameMeta) GetFields() []*Field { if m != nil { return m.Fields @@ -94,6 +149,13 @@ func (m *ImportResponse) String() string { return proto.CompactTextSt func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{2} } +func (m *ImportResponse) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -107,6 +169,41 @@ func (m *BlockDataRequest) String() string { return proto.CompactText func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} } +func (m *BlockDataRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *BlockDataRequest) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *BlockDataRequest) GetView() string { + if m != nil { + return m.View + } + return "" +} + +func (m *BlockDataRequest) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *BlockDataRequest) GetBlock() uint64 { + if m != nil { + return m.Block + } + return 0 +} + type BlockDataResponse struct { RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` @@ -117,6 +214,20 @@ func (m *BlockDataResponse) String() string { return proto.CompactTex func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} } +func (m *BlockDataResponse) GetRowIDs() []uint64 { + if m != nil { + return m.RowIDs + } + return nil +} + +func (m *BlockDataResponse) GetColumnIDs() []uint64 { + if m != nil { + return m.ColumnIDs + } + return nil +} + type Cache struct { IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` } @@ -126,6 +237,13 @@ func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{5} } +func (m *Cache) GetIDs() []uint64 { + if m != nil { + return m.IDs + } + return nil +} + type MaxSlicesResponse struct { MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } @@ -153,6 +271,27 @@ func (m *CreateSliceMessage) String() string { return proto.CompactTe func (*CreateSliceMessage) ProtoMessage() {} func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } +func (m *CreateSliceMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateSliceMessage) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *CreateSliceMessage) GetIsInverse() bool { + if m != nil { + return m.IsInverse + } + return false +} + type DeleteIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } @@ -162,6 +301,13 @@ func (m *DeleteIndexMessage) String() string { return proto.CompactTe func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } +func (m *DeleteIndexMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` @@ -172,6 +318,13 @@ func (m *CreateIndexMessage) String() string { return proto.CompactTe func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } +func (m *CreateIndexMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + func (m *CreateIndexMessage) GetMeta() *IndexMeta { if m != nil { return m.Meta @@ -190,6 +343,20 @@ func (m *CreateFrameMessage) String() string { return proto.CompactTe func (*CreateFrameMessage) ProtoMessage() {} func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } +func (m *CreateFrameMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateFrameMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + func (m *CreateFrameMessage) GetMeta() *FrameMeta { if m != nil { return m.Meta @@ -207,6 +374,20 @@ func (m *DeleteFrameMessage) String() string { return proto.CompactTe func (*DeleteFrameMessage) ProtoMessage() {} func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } +func (m *DeleteFrameMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteFrameMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + type Frame struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` @@ -217,6 +398,13 @@ func (m *Frame) String() string { return proto.CompactTextString(m) } func (*Frame) ProtoMessage() {} func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (m *Frame) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *Frame) GetMeta() *FrameMeta { if m != nil { return m.Meta @@ -238,6 +426,13 @@ func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (m *Index) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *Index) GetMeta() *IndexMeta { if m != nil { return m.Meta @@ -245,6 +440,13 @@ func (m *Index) GetMeta() *IndexMeta { return nil } +func (m *Index) GetMaxSlice() uint64 { + if m != nil { + return m.MaxSlice + } + return 0 +} + func (m *Index) GetFrames() []*Frame { if m != nil { return m.Frames @@ -252,6 +454,13 @@ func (m *Index) GetFrames() []*Frame { return nil } +func (m *Index) GetSlices() []uint64 { + if m != nil { + return m.Slices + } + return nil +} + func (m *Index) GetInputDefinitions() []*InputDefinition { if m != nil { return m.InputDefinitions @@ -270,6 +479,13 @@ func (m *InputDefinition) String() string { return proto.CompactTextS func (*InputDefinition) ProtoMessage() {} func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (m *InputDefinition) GetName() string { + if m != nil { + return m.Name + } + return "" +} + func (m *InputDefinition) GetFrames() []*Frame { if m != nil { return m.Frames @@ -295,6 +511,20 @@ func (m *InputDefinitionField) String() string { return proto.Compact func (*InputDefinitionField) ProtoMessage() {} func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (m *InputDefinitionField) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *InputDefinitionField) GetPrimaryKey() bool { + if m != nil { + return m.PrimaryKey + } + return false +} + func (m *InputDefinitionField) GetInputDefinitionActions() []*InputDefinitionAction { if m != nil { return m.InputDefinitionActions @@ -314,6 +544,20 @@ func (m *InputDefinitionAction) String() string { return proto.Compac func (*InputDefinitionAction) ProtoMessage() {} func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (m *InputDefinitionAction) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *InputDefinitionAction) GetValueDestination() string { + if m != nil { + return m.ValueDestination + } + return "" +} + func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { if m != nil { return m.ValueMap @@ -321,6 +565,13 @@ func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { return nil } +func (m *InputDefinitionAction) GetRowID() uint64 { + if m != nil { + return m.RowID + } + return 0 +} + type CreateInputDefinitionMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Definition *InputDefinition `protobuf:"bytes,3,opt,name=Definition" json:"Definition,omitempty"` @@ -333,6 +584,13 @@ func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (m *CreateInputDefinitionMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + func (m *CreateInputDefinitionMessage) GetDefinition() *InputDefinition { if m != nil { return m.Definition @@ -352,10 +610,25 @@ func (*DeleteInputDefinitionMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } +func (m *DeleteInputDefinitionMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteInputDefinitionMessage) GetName() string { + if m != nil { + return m.Name + } + return "" +} + type NodeStatus struct { Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` + Scheme string `protobuf:"bytes,4,opt,name=Scheme,proto3" json:"Scheme,omitempty"` } func (m *NodeStatus) Reset() { *m = NodeStatus{} } @@ -363,6 +636,20 @@ func (m *NodeStatus) String() string { return proto.CompactTextString func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (m *NodeStatus) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *NodeStatus) GetState() string { + if m != nil { + return m.State + } + return "" +} + func (m *NodeStatus) GetIndexes() []*Index { if m != nil { return m.Indexes @@ -370,6 +657,13 @@ func (m *NodeStatus) GetIndexes() []*Index { return nil } +func (m *NodeStatus) GetScheme() string { + if m != nil { + return m.Scheme + } + return "" +} + type ClusterStatus struct { Nodes []*NodeStatus `protobuf:"bytes,1,rep,name=Nodes" json:"Nodes,omitempty"` } @@ -414,6 +708,34 @@ func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (m *Field) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Field) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *Field) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *Field) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -425,6 +747,27 @@ func (m *DeleteViewMessage) String() string { return proto.CompactTex func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (m *DeleteViewMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteViewMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *DeleteViewMessage) GetView() string { + if m != nil { + return m.View + } + return "" +} + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -1274,6 +1617,12 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } + if len(m.Scheme) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) + i += copy(dAtA[i:], m.Scheme) + } return i, nil } @@ -1413,24 +1762,6 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1801,6 +2132,10 @@ func (m *NodeStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + l = len(m.Scheme) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -2498,7 +2833,24 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RowIDs = append(m.RowIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2539,7 +2891,11 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } m.RowIDs = append(m.RowIDs, v) } - } else if wireType == 0 { + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) + } + case 2: + if wireType == 0 { var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2555,12 +2911,8 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.RowIDs = append(m.RowIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) - } - case 2: - if wireType == 2 { + m.ColumnIDs = append(m.ColumnIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2601,23 +2953,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } m.ColumnIDs = append(m.ColumnIDs, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.ColumnIDs = append(m.ColumnIDs, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } @@ -2672,7 +3007,24 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2713,23 +3065,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } m.IDs = append(m.IDs, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.IDs = append(m.IDs, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) } @@ -2809,51 +3144,14 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.MaxSlices == nil { m.MaxSlices = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -2863,31 +3161,69 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.MaxSlices[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.MaxSlices[mapkey] = mapvalue } + m.MaxSlices[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -3722,7 +4058,24 @@ func (m *Index) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 5: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -3763,23 +4116,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Slices = append(m.Slices, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) } @@ -4219,51 +4555,14 @@ func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.ValueMap == nil { m.ValueMap = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -4273,31 +4572,69 @@ func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.ValueMap[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.ValueMap[mapkey] = mapvalue } + m.ValueMap[mapkey] = mapvalue iNdEx = postIndex case 4: if wireType != 0 { @@ -4677,6 +5014,35 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Scheme", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Scheme = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -5251,64 +5617,65 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 940 bytes of a gzipped FileDescriptorProto + // 948 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, 0x10, 0x65, 0x3c, 0x63, 0xaf, 0x5d, 0x26, 0x1b, 0xa7, 0x09, 0x2b, 0x6f, 0x14, 0x19, 0xab, 0x0f, 0x6c, 0x88, 0x44, 0x0e, 0x41, 0x5a, 0x01, 0xcb, 0x01, 0x36, 0xce, 0x2a, 0x16, 0x78, 0x81, 0xf6, 0x6a, 0xb9, 0x21, 0x75, 0x9c, 0x62, 0x77, 0x94, 0xf1, 0x8c, 0x99, 0x69, 0x27, 0x31, 0x07, 0x8e, 0x7c, 0x03, 0x12, 0x47, 0x7e, 0x86, 0x23, 0x9f, 0x80, 0xc2, 0x85, 0x3f, 0x40, 0xe2, 0x84, 0xba, - 0xba, 0x7b, 0x66, 0x6c, 0xc7, 0x8e, 0xb2, 0xb7, 0xae, 0xd7, 0xd5, 0x55, 0xaf, 0xdf, 0x54, 0xd5, - 0x34, 0x6c, 0x4c, 0xd2, 0xf0, 0x42, 0x2a, 0x3c, 0x98, 0xa4, 0x89, 0x4a, 0x58, 0x3d, 0x8c, 0x15, - 0xa6, 0xb1, 0x8c, 0xf8, 0xd7, 0xd0, 0xe8, 0xc7, 0x67, 0x78, 0x35, 0x40, 0x25, 0x59, 0x17, 0x9a, - 0x47, 0x49, 0x34, 0x1d, 0xc7, 0x5f, 0xc9, 0x53, 0x8c, 0xda, 0x5e, 0xd7, 0xdb, 0x6b, 0x88, 0x32, - 0xa4, 0x3d, 0x5e, 0x84, 0x63, 0xfc, 0x76, 0x2a, 0x63, 0x35, 0x1d, 0xb7, 0x2b, 0xc6, 0xa3, 0x04, - 0xf1, 0xff, 0x3c, 0x68, 0x3c, 0x4b, 0xe5, 0x18, 0x29, 0xe2, 0x0e, 0xd4, 0x45, 0x72, 0x59, 0x0e, - 0x97, 0xdb, 0xec, 0x7d, 0xb8, 0xdf, 0x8f, 0x2f, 0x30, 0xcd, 0xf0, 0x38, 0x96, 0xa7, 0x11, 0x9e, - 0x51, 0xb8, 0xba, 0x58, 0x40, 0xd9, 0x2e, 0x34, 0x8e, 0xe4, 0xe8, 0x35, 0xbe, 0x98, 0x4d, 0xb0, - 0xed, 0x53, 0x90, 0x02, 0xc8, 0x77, 0x87, 0xe1, 0x4f, 0xd8, 0x0e, 0xba, 0xde, 0xde, 0x86, 0x28, - 0x80, 0x45, 0xbe, 0xd5, 0x25, 0xbe, 0x8c, 0xc3, 0xdb, 0x42, 0xc6, 0xaf, 0x72, 0x0e, 0x35, 0xe2, - 0x30, 0x87, 0xb1, 0x47, 0x50, 0x7b, 0x16, 0x62, 0x74, 0x96, 0xb5, 0xef, 0x75, 0xfd, 0xbd, 0xe6, - 0xe1, 0xe6, 0x81, 0xd3, 0xef, 0x80, 0x70, 0x61, 0xb7, 0x39, 0x87, 0xfb, 0xfd, 0xf1, 0x24, 0x49, - 0x95, 0xc0, 0x6c, 0x92, 0xc4, 0x19, 0xb2, 0x16, 0xf8, 0xc7, 0x69, 0x6a, 0xef, 0xae, 0x97, 0xfc, - 0x67, 0x68, 0x3d, 0x8d, 0x92, 0xd1, 0x79, 0x4f, 0x2a, 0x29, 0xf0, 0xc7, 0x29, 0x66, 0x8a, 0x6d, - 0x43, 0x95, 0xbe, 0x82, 0xf5, 0x33, 0x86, 0x46, 0x49, 0x49, 0x2b, 0xb3, 0x31, 0x34, 0x4a, 0xe7, - 0x49, 0x8a, 0x40, 0x18, 0x43, 0xa3, 0xc3, 0x28, 0x1c, 0x19, 0x09, 0x02, 0x61, 0x0c, 0xc6, 0x20, - 0x78, 0x19, 0xe2, 0xa5, 0xbd, 0x37, 0xad, 0x79, 0x1f, 0xb6, 0x4a, 0xf9, 0x2d, 0xcd, 0x07, 0x50, - 0x13, 0xc9, 0x65, 0xbf, 0x97, 0xb5, 0xbd, 0xae, 0xbf, 0x17, 0x08, 0x6b, 0x91, 0xba, 0xf4, 0xf9, - 0xf5, 0x56, 0x85, 0xb6, 0x0a, 0x80, 0x3f, 0x84, 0x2a, 0x49, 0xad, 0x6f, 0x59, 0x9c, 0xd5, 0x4b, - 0xfe, 0x9b, 0x07, 0x5b, 0x03, 0x79, 0x45, 0x34, 0xb2, 0x3c, 0xcd, 0x09, 0x34, 0x72, 0x90, 0xbc, - 0x9b, 0x87, 0xfb, 0x85, 0x96, 0x4b, 0xfe, 0x05, 0x72, 0x1c, 0xab, 0x74, 0x26, 0x8a, 0xc3, 0x3b, - 0x9f, 0xc1, 0xfd, 0xf9, 0x4d, 0xcd, 0xe1, 0x1c, 0x67, 0x4e, 0xe9, 0x73, 0x9c, 0x69, 0x4d, 0x2e, - 0x64, 0x34, 0x35, 0xfa, 0x05, 0xc2, 0x18, 0x9f, 0x56, 0x3e, 0xf6, 0xf8, 0xf7, 0xc0, 0x8e, 0x52, - 0x94, 0x0a, 0x29, 0xc0, 0x00, 0xb3, 0x4c, 0xbe, 0xc2, 0xd5, 0x5f, 0xc1, 0x28, 0x5b, 0x29, 0x2b, - 0xbb, 0x0b, 0x8d, 0x7e, 0x66, 0x0b, 0x95, 0xbe, 0x44, 0x5d, 0x14, 0x00, 0xdf, 0x07, 0xd6, 0xc3, - 0x08, 0x15, 0xda, 0xde, 0x5a, 0x13, 0x9f, 0x0f, 0x1d, 0x97, 0xdb, 0x7d, 0xd9, 0x23, 0x08, 0x74, - 0x5b, 0x11, 0x95, 0xe6, 0xe1, 0x3b, 0x85, 0x74, 0x79, 0x0f, 0x0b, 0x72, 0xe0, 0xa1, 0x0b, 0x6a, - 0x5b, 0xf1, 0x96, 0x0b, 0xde, 0x50, 0x66, 0x2e, 0x95, 0xbf, 0x98, 0x2a, 0x6f, 0x6e, 0x9b, 0xea, - 0x73, 0x77, 0xd7, 0x37, 0x4d, 0xc5, 0x7b, 0x16, 0xd5, 0xe5, 0xfa, 0x5c, 0xef, 0x9a, 0x33, 0xb4, - 0x5e, 0x7d, 0xe5, 0x45, 0x1e, 0xff, 0x78, 0x36, 0xe5, 0xdd, 0xc2, 0x2c, 0x28, 0xa7, 0x27, 0x96, - 0x2b, 0x2c, 0xdb, 0x61, 0xb9, 0x4d, 0x73, 0x40, 0x67, 0xcd, 0xda, 0xc1, 0xd2, 0x1c, 0xd0, 0xb8, - 0xb0, 0xdb, 0xba, 0x9d, 0x6c, 0x91, 0x57, 0x4d, 0x3b, 0x19, 0x8b, 0x1d, 0x43, 0xab, 0x1f, 0x4f, - 0xa6, 0xaa, 0x87, 0x3f, 0x84, 0x71, 0xa8, 0xc2, 0x24, 0xce, 0xda, 0x35, 0x0a, 0xf5, 0xb0, 0xcc, - 0x68, 0xce, 0x43, 0x2c, 0x1d, 0xe1, 0xbf, 0x78, 0xb0, 0xb9, 0x00, 0xae, 0xb8, 0xb4, 0xe3, 0x5b, - 0x59, 0xcf, 0xf7, 0x71, 0x3e, 0xe0, 0x7c, 0x72, 0xec, 0xac, 0x64, 0x33, 0x3f, 0xef, 0x7e, 0xf7, - 0x60, 0xfb, 0x26, 0x87, 0x1b, 0xd9, 0x74, 0x00, 0xbe, 0x49, 0xc3, 0xb1, 0x4c, 0x67, 0x5f, 0xe2, - 0xcc, 0xce, 0xfa, 0x12, 0xc2, 0xbe, 0x83, 0x07, 0x0b, 0xb1, 0xbe, 0x18, 0x19, 0x89, 0x0c, 0xa9, - 0xf7, 0x56, 0x92, 0x32, 0x7e, 0x62, 0xc5, 0x71, 0xfe, 0xaf, 0x07, 0xef, 0xde, 0xb8, 0x55, 0xd4, - 0xa3, 0x57, 0x2e, 0xfd, 0x7d, 0x68, 0xbd, 0xd4, 0xa3, 0xa2, 0x87, 0x99, 0x0a, 0x63, 0xa9, 0x3d, - 0x6d, 0xc1, 0x2e, 0xe1, 0xac, 0x0f, 0x75, 0xc2, 0x06, 0x72, 0x62, 0x69, 0x7e, 0x78, 0x0b, 0xcd, - 0x03, 0xe7, 0x6f, 0x66, 0x5a, 0x7e, 0x5c, 0x93, 0xa1, 0xa9, 0xeb, 0x46, 0x38, 0x19, 0x3b, 0x4f, - 0x60, 0x63, 0xee, 0xc0, 0x9d, 0xe6, 0x5c, 0x02, 0xbb, 0x6e, 0xb6, 0xcc, 0x31, 0x59, 0xdf, 0xa5, - 0x9f, 0x00, 0x14, 0xae, 0x76, 0x00, 0xac, 0xa9, 0xcf, 0x92, 0x33, 0x3f, 0x81, 0x5d, 0x37, 0xf8, - 0xee, 0x90, 0xd0, 0x55, 0x4b, 0xa5, 0xa8, 0x16, 0x2e, 0x01, 0x9e, 0x27, 0x67, 0x38, 0x54, 0x52, - 0x4d, 0x33, 0xed, 0x71, 0x92, 0x64, 0xca, 0xd5, 0x93, 0x5e, 0xd3, 0x60, 0x56, 0x52, 0xe5, 0xc3, - 0x84, 0x0c, 0xf6, 0x01, 0xdc, 0xa3, 0xa0, 0xe8, 0xca, 0x66, 0x73, 0xa1, 0xd7, 0x85, 0xdb, 0xe7, - 0x4f, 0x60, 0xe3, 0x28, 0x9a, 0x66, 0x0a, 0x53, 0x9b, 0x65, 0x1f, 0xaa, 0x3a, 0xa7, 0xfb, 0x35, - 0x6d, 0x17, 0x27, 0x0b, 0x2a, 0xc2, 0xb8, 0xf0, 0xc7, 0xd0, 0xa4, 0x6a, 0x19, 0x8e, 0x5e, 0xe3, - 0x58, 0x96, 0x9e, 0x08, 0xde, 0xfa, 0x27, 0xc2, 0x10, 0xaa, 0xab, 0x5b, 0x84, 0x41, 0x40, 0xaf, - 0x1c, 0x2b, 0x04, 0x3d, 0x70, 0x5a, 0xe0, 0x0f, 0x42, 0xf3, 0x19, 0x7c, 0xa1, 0x97, 0x84, 0xc8, - 0x2b, 0x2a, 0x13, 0x8d, 0x48, 0xfd, 0x0f, 0xd9, 0x32, 0xb2, 0xeb, 0x3f, 0xfc, 0x9b, 0x4c, 0x7b, - 0xf7, 0x50, 0xf0, 0x8b, 0x87, 0xc2, 0xd3, 0xd6, 0x1f, 0xd7, 0x1d, 0xef, 0xcf, 0xeb, 0x8e, 0xf7, - 0xd7, 0x75, 0xc7, 0xfb, 0xf5, 0xef, 0xce, 0x5b, 0xa7, 0x35, 0x7a, 0x3d, 0x7e, 0xf4, 0x7f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x59, 0x39, 0x2e, 0xa5, 0x4e, 0x0a, 0x00, 0x00, + 0xba, 0x7b, 0x66, 0x6c, 0xc7, 0x8e, 0xb2, 0xb7, 0xae, 0x57, 0xd5, 0x55, 0xaf, 0xab, 0xab, 0xaa, + 0x1b, 0x36, 0x26, 0x69, 0x78, 0x21, 0x15, 0x1e, 0x4c, 0xd2, 0x44, 0x25, 0xac, 0x1e, 0xc6, 0x0a, + 0xd3, 0x58, 0x46, 0xfc, 0x6b, 0x68, 0xf4, 0xe3, 0x33, 0xbc, 0x1a, 0xa0, 0x92, 0xac, 0x0b, 0xcd, + 0xa3, 0x24, 0x9a, 0x8e, 0xe3, 0xaf, 0xe4, 0x29, 0x46, 0x6d, 0xaf, 0xeb, 0xed, 0x35, 0x44, 0x19, + 0xd2, 0x16, 0x2f, 0xc2, 0x31, 0x7e, 0x3b, 0x95, 0xb1, 0x9a, 0x8e, 0xdb, 0x15, 0x63, 0x51, 0x82, + 0xf8, 0x7f, 0x1e, 0x34, 0x9e, 0xa5, 0x72, 0x8c, 0xe4, 0x71, 0x07, 0xea, 0x22, 0xb9, 0x2c, 0xbb, + 0xcb, 0x65, 0xf6, 0x3e, 0xdc, 0xef, 0xc7, 0x17, 0x98, 0x66, 0x78, 0x1c, 0xcb, 0xd3, 0x08, 0xcf, + 0xc8, 0x5d, 0x5d, 0x2c, 0xa0, 0x6c, 0x17, 0x1a, 0x47, 0x72, 0xf4, 0x1a, 0x5f, 0xcc, 0x26, 0xd8, + 0xf6, 0xc9, 0x49, 0x01, 0xe4, 0xda, 0x61, 0xf8, 0x13, 0xb6, 0x83, 0xae, 0xb7, 0xb7, 0x21, 0x0a, + 0x60, 0x91, 0x6f, 0x75, 0x89, 0x2f, 0xe3, 0xf0, 0xb6, 0x90, 0xf1, 0xab, 0x9c, 0x43, 0x8d, 0x38, + 0xcc, 0x61, 0xec, 0x11, 0xd4, 0x9e, 0x85, 0x18, 0x9d, 0x65, 0xed, 0x7b, 0x5d, 0x7f, 0xaf, 0x79, + 0xb8, 0x79, 0xe0, 0xf2, 0x77, 0x40, 0xb8, 0xb0, 0x6a, 0xce, 0xe1, 0x7e, 0x7f, 0x3c, 0x49, 0x52, + 0x25, 0x30, 0x9b, 0x24, 0x71, 0x86, 0xac, 0x05, 0xfe, 0x71, 0x9a, 0xda, 0xb3, 0xeb, 0x25, 0xff, + 0x19, 0x5a, 0x4f, 0xa3, 0x64, 0x74, 0xde, 0x93, 0x4a, 0x0a, 0xfc, 0x71, 0x8a, 0x99, 0x62, 0xdb, + 0x50, 0xa5, 0x5b, 0xb0, 0x76, 0x46, 0xd0, 0x28, 0x65, 0xd2, 0xa6, 0xd9, 0x08, 0x1a, 0xa5, 0xfd, + 0x94, 0x8a, 0x40, 0x18, 0x41, 0xa3, 0xc3, 0x28, 0x1c, 0x99, 0x14, 0x04, 0xc2, 0x08, 0x8c, 0x41, + 0xf0, 0x32, 0xc4, 0x4b, 0x7b, 0x6e, 0x5a, 0xf3, 0x3e, 0x6c, 0x95, 0xe2, 0x5b, 0x9a, 0x0f, 0xa0, + 0x26, 0x92, 0xcb, 0x7e, 0x2f, 0x6b, 0x7b, 0x5d, 0x7f, 0x2f, 0x10, 0x56, 0xa2, 0xec, 0xd2, 0xf5, + 0x6b, 0x55, 0x85, 0x54, 0x05, 0xc0, 0x1f, 0x42, 0x95, 0x52, 0xad, 0x4f, 0x59, 0xec, 0xd5, 0x4b, + 0xfe, 0x9b, 0x07, 0x5b, 0x03, 0x79, 0x45, 0x34, 0xb2, 0x3c, 0xcc, 0x09, 0x34, 0x72, 0x90, 0xac, + 0x9b, 0x87, 0xfb, 0x45, 0x2e, 0x97, 0xec, 0x0b, 0xe4, 0x38, 0x56, 0xe9, 0x4c, 0x14, 0x9b, 0x77, + 0x3e, 0x83, 0xfb, 0xf3, 0x4a, 0xcd, 0xe1, 0x1c, 0x67, 0x2e, 0xd3, 0xe7, 0x38, 0xd3, 0x39, 0xb9, + 0x90, 0xd1, 0xd4, 0xe4, 0x2f, 0x10, 0x46, 0xf8, 0xb4, 0xf2, 0xb1, 0xc7, 0xbf, 0x07, 0x76, 0x94, + 0xa2, 0x54, 0x48, 0x0e, 0x06, 0x98, 0x65, 0xf2, 0x15, 0xae, 0xbe, 0x05, 0x93, 0xd9, 0x4a, 0x39, + 0xb3, 0xbb, 0xd0, 0xe8, 0x67, 0xb6, 0x50, 0xe9, 0x26, 0xea, 0xa2, 0x00, 0xf8, 0x3e, 0xb0, 0x1e, + 0x46, 0xa8, 0xd0, 0xf6, 0xd6, 0x1a, 0xff, 0x7c, 0xe8, 0xb8, 0xdc, 0x6e, 0xcb, 0x1e, 0x41, 0xa0, + 0xdb, 0x8a, 0xa8, 0x34, 0x0f, 0xdf, 0x29, 0x52, 0x97, 0xf7, 0xb0, 0x20, 0x03, 0x1e, 0x3a, 0xa7, + 0xb6, 0x15, 0x6f, 0x39, 0xe0, 0x0d, 0x65, 0xe6, 0x42, 0xf9, 0x8b, 0xa1, 0xf2, 0xe6, 0xb6, 0xa1, + 0x3e, 0x77, 0x67, 0x7d, 0xd3, 0x50, 0xbc, 0x67, 0x51, 0x5d, 0xae, 0xcf, 0xb5, 0xd6, 0xec, 0xa1, + 0xf5, 0xea, 0x23, 0x2f, 0xf2, 0xf8, 0xc7, 0xb3, 0x21, 0xef, 0xe6, 0x66, 0x21, 0x73, 0x7a, 0x62, + 0xb9, 0xc2, 0xb2, 0x1d, 0x96, 0xcb, 0x34, 0x07, 0x74, 0xd4, 0xac, 0x1d, 0x2c, 0xcd, 0x01, 0x8d, + 0x0b, 0xab, 0xd6, 0xed, 0x64, 0x8b, 0xbc, 0x6a, 0xda, 0xc9, 0x48, 0xec, 0x18, 0x5a, 0xfd, 0x78, + 0x32, 0x55, 0x3d, 0xfc, 0x21, 0x8c, 0x43, 0x15, 0x26, 0x71, 0xd6, 0xae, 0x91, 0xab, 0x87, 0x65, + 0x46, 0x73, 0x16, 0x62, 0x69, 0x0b, 0xff, 0xc5, 0x83, 0xcd, 0x05, 0x70, 0xc5, 0xa1, 0x1d, 0xdf, + 0xca, 0x7a, 0xbe, 0x8f, 0xf3, 0x01, 0xe7, 0x93, 0x61, 0x67, 0x25, 0x9b, 0xf9, 0x79, 0xf7, 0xbb, + 0x07, 0xdb, 0x37, 0x19, 0xdc, 0xc8, 0xa6, 0x03, 0xf0, 0x4d, 0x1a, 0x8e, 0x65, 0x3a, 0xfb, 0x12, + 0x67, 0x76, 0xd6, 0x97, 0x10, 0xf6, 0x1d, 0x3c, 0x58, 0xf0, 0xf5, 0xc5, 0xc8, 0xa4, 0xc8, 0x90, + 0x7a, 0x6f, 0x25, 0x29, 0x63, 0x27, 0x56, 0x6c, 0xe7, 0xff, 0x7a, 0xf0, 0xee, 0x8d, 0xaa, 0xa2, + 0x1e, 0xbd, 0x72, 0xe9, 0xef, 0x43, 0xeb, 0xa5, 0x1e, 0x15, 0x3d, 0xcc, 0x54, 0x18, 0x4b, 0x6d, + 0x69, 0x0b, 0x76, 0x09, 0x67, 0x7d, 0xa8, 0x13, 0x36, 0x90, 0x13, 0x4b, 0xf3, 0xc3, 0x5b, 0x68, + 0x1e, 0x38, 0x7b, 0x33, 0xd3, 0xf2, 0xed, 0x9a, 0x0c, 0x4d, 0x5d, 0x37, 0xc2, 0x49, 0xd8, 0x79, + 0x02, 0x1b, 0x73, 0x1b, 0xee, 0x34, 0xe7, 0x12, 0xd8, 0x75, 0xb3, 0x65, 0x8e, 0xc9, 0xfa, 0x2e, + 0xfd, 0x04, 0xa0, 0x30, 0xb5, 0x03, 0x60, 0x4d, 0x7d, 0x96, 0x8c, 0xf9, 0x09, 0xec, 0xba, 0xc1, + 0x77, 0x87, 0x80, 0xae, 0x5a, 0x2a, 0x45, 0xb5, 0xf0, 0x19, 0xc0, 0xf3, 0xe4, 0x0c, 0x87, 0x4a, + 0xaa, 0x69, 0xa6, 0x2d, 0x4e, 0x92, 0x4c, 0xb9, 0x7a, 0xd2, 0x6b, 0x1a, 0xcc, 0x4a, 0xaa, 0x7c, + 0x98, 0x90, 0xc0, 0x3e, 0x80, 0x7b, 0xe4, 0x14, 0x5d, 0xd9, 0x6c, 0x2e, 0xf4, 0xba, 0x70, 0x7a, + 0xea, 0xd2, 0xd1, 0x6b, 0x1c, 0x9b, 0x47, 0xb3, 0x21, 0xac, 0xc4, 0x9f, 0xc0, 0xc6, 0x51, 0x34, + 0xcd, 0x14, 0xa6, 0x36, 0xfa, 0x3e, 0x54, 0x35, 0x17, 0xf7, 0x64, 0x6d, 0x17, 0x1e, 0x0b, 0x8a, + 0xc2, 0x98, 0xf0, 0xc7, 0xd0, 0xa4, 0x2a, 0x22, 0x5f, 0xb2, 0xf4, 0x75, 0xf0, 0xd6, 0x7f, 0x1d, + 0x86, 0x50, 0x5d, 0xdd, 0x3a, 0x0c, 0x02, 0xfa, 0xfd, 0xd8, 0x04, 0xd1, 0xc7, 0xa7, 0x05, 0xfe, + 0x20, 0x34, 0xd7, 0xe3, 0x0b, 0xbd, 0x24, 0x44, 0x5e, 0xd1, 0x61, 0x34, 0x22, 0xf5, 0xdb, 0xb2, + 0x65, 0xae, 0x43, 0xbf, 0xfc, 0x6f, 0xf2, 0x0a, 0xb8, 0x0f, 0x84, 0x5f, 0x7c, 0x20, 0x9e, 0xb6, + 0xfe, 0xb8, 0xee, 0x78, 0x7f, 0x5e, 0x77, 0xbc, 0xbf, 0xae, 0x3b, 0xde, 0xaf, 0x7f, 0x77, 0xde, + 0x3a, 0xad, 0xd1, 0xaf, 0xf2, 0xa3, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x47, 0xdd, 0xdd, 0x8e, + 0x66, 0x0a, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index e37ca48b6..083316ddd 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -116,6 +116,7 @@ message NodeStatus { string Host = 1; string State = 2; repeated Index Indexes = 3; + string Scheme = 4; } message ClusterStatus { diff --git a/internal/public.pb.go b/internal/public.pb.go index 33987fd10..81fb2267b 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import encoding_binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -51,6 +52,13 @@ func (m *Bitmap) String() string { return proto.CompactTextString(m) func (*Bitmap) ProtoMessage() {} func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} } +func (m *Bitmap) GetBits() []uint64 { + if m != nil { + return m.Bits + } + return nil +} + func (m *Bitmap) GetAttrs() []*Attr { if m != nil { return m.Attrs @@ -68,6 +76,20 @@ func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } +func (m *Pair) GetKey() uint64 { + if m != nil { + return m.Key + } + return 0 +} + +func (m *Pair) GetCount() uint64 { + if m != nil { + return m.Count + } + return 0 +} + type SumCount struct { Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` @@ -78,6 +100,20 @@ func (m *SumCount) String() string { return proto.CompactTextString(m func (*SumCount) ProtoMessage() {} func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (m *SumCount) GetSum() int64 { + if m != nil { + return m.Sum + } + return 0 +} + +func (m *SumCount) GetCount() int64 { + if m != nil { + return m.Count + } + return 0 +} + type Bit struct { RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` @@ -89,6 +125,27 @@ func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (m *Bit) GetRowID() uint64 { + if m != nil { + return m.RowID + } + return 0 +} + +func (m *Bit) GetColumnID() uint64 { + if m != nil { + return m.ColumnID + } + return 0 +} + +func (m *Bit) GetTimestamp() int64 { + if m != nil { + return m.Timestamp + } + return 0 +} + type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` @@ -99,6 +156,13 @@ func (m *ColumnAttrSet) String() string { return proto.CompactTextStr func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (m *ColumnAttrSet) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + func (m *ColumnAttrSet) GetAttrs() []*Attr { if m != nil { return m.Attrs @@ -120,6 +184,48 @@ func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (m *Attr) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *Attr) GetType() uint64 { + if m != nil { + return m.Type + } + return 0 +} + +func (m *Attr) GetStringValue() string { + if m != nil { + return m.StringValue + } + return "" +} + +func (m *Attr) GetIntValue() int64 { + if m != nil { + return m.IntValue + } + return 0 +} + +func (m *Attr) GetBoolValue() bool { + if m != nil { + return m.BoolValue + } + return false +} + +func (m *Attr) GetFloatValue() float64 { + if m != nil { + return m.FloatValue + } + return 0 +} + type AttrMap struct { Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } @@ -150,6 +256,48 @@ func (m *QueryRequest) String() string { return proto.CompactTextStri func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (m *QueryRequest) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + +func (m *QueryRequest) GetSlices() []uint64 { + if m != nil { + return m.Slices + } + return nil +} + +func (m *QueryRequest) GetColumnAttrs() bool { + if m != nil { + return m.ColumnAttrs + } + return false +} + +func (m *QueryRequest) GetRemote() bool { + if m != nil { + return m.Remote + } + return false +} + +func (m *QueryRequest) GetExcludeAttrs() bool { + if m != nil { + return m.ExcludeAttrs + } + return false +} + +func (m *QueryRequest) GetExcludeBits() bool { + if m != nil { + return m.ExcludeBits + } + return false +} + type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` @@ -161,6 +309,13 @@ func (m *QueryResponse) String() string { return proto.CompactTextStr func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (m *QueryResponse) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { return m.Results @@ -195,6 +350,13 @@ func (m *QueryResult) GetBitmap() *Bitmap { return nil } +func (m *QueryResult) GetN() uint64 { + if m != nil { + return m.N + } + return 0 +} + func (m *QueryResult) GetPairs() []*Pair { if m != nil { return m.Pairs @@ -209,6 +371,13 @@ func (m *QueryResult) GetSumCount() *SumCount { return nil } +func (m *QueryResult) GetChanged() bool { + if m != nil { + return m.Changed + } + return false +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -223,6 +392,48 @@ func (m *ImportRequest) String() string { return proto.CompactTextStr func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (m *ImportRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *ImportRequest) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *ImportRequest) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *ImportRequest) GetRowIDs() []uint64 { + if m != nil { + return m.RowIDs + } + return nil +} + +func (m *ImportRequest) GetColumnIDs() []uint64 { + if m != nil { + return m.ColumnIDs + } + return nil +} + +func (m *ImportRequest) GetTimestamps() []int64 { + if m != nil { + return m.Timestamps + } + return nil +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -237,6 +448,48 @@ func (m *ImportValueRequest) String() string { return proto.CompactTe func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (m *ImportValueRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *ImportValueRequest) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *ImportValueRequest) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +func (m *ImportValueRequest) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *ImportValueRequest) GetColumnIDs() []uint64 { + if m != nil { + return m.ColumnIDs + } + return nil +} + +func (m *ImportValueRequest) GetValues() []uint64 { + if m != nil { + return m.Values + } + return nil +} + func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") @@ -472,7 +725,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 } return i, nil } @@ -863,24 +1117,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1194,7 +1430,24 @@ func (m *Bitmap) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -1235,23 +1488,6 @@ func (m *Bitmap) Unmarshal(dAtA []byte) error { } m.Bits = append(m.Bits, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Bits = append(m.Bits, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) } @@ -1843,15 +2079,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex @@ -2014,7 +2243,24 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { m.Query = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2055,23 +2301,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } m.Slices = append(m.Slices, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) } @@ -2610,7 +2839,24 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } } case 4: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RowIDs = append(m.RowIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2651,7 +2897,11 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.RowIDs = append(m.RowIDs, v) } - } else if wireType == 0 { + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) + } + case 5: + if wireType == 0 { var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2667,12 +2917,8 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.RowIDs = append(m.RowIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) - } - case 5: - if wireType == 2 { + m.ColumnIDs = append(m.ColumnIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2713,8 +2959,12 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.ColumnIDs = append(m.ColumnIDs, v) } - } else if wireType == 0 { - var v uint64 + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) + } + case 6: + if wireType == 0 { + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -2724,17 +2974,13 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } } - m.ColumnIDs = append(m.ColumnIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) - } - case 6: - if wireType == 2 { + m.Timestamps = append(m.Timestamps, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2775,23 +3021,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.Timestamps = append(m.Timestamps, v) } - } else if wireType == 0 { - var v int64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Timestamps = append(m.Timestamps, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Timestamps", wireType) } @@ -2952,7 +3181,24 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 5: - if wireType == 2 { + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ColumnIDs = append(m.ColumnIDs, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -2993,7 +3239,11 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.ColumnIDs = append(m.ColumnIDs, v) } - } else if wireType == 0 { + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) + } + case 6: + if wireType == 0 { var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -3009,12 +3259,8 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { break } } - m.ColumnIDs = append(m.ColumnIDs, v) - } else { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) - } - case 6: - if wireType == 2 { + m.Values = append(m.Values, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -3055,23 +3301,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.Values = append(m.Values, v) } - } else if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Values = append(m.Values, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field Values", wireType) } diff --git a/server.go b/server.go index 3460bec46..e94475d02 100644 --- a/server.go +++ b/server.go @@ -386,6 +386,7 @@ func (s *Server) LocalStatus() (proto.Message, error) { } ns := internal.NodeStatus{ + Scheme: s.URI.Scheme(), Host: s.URI.HostPort(), State: NodeStateUp, Indexes: EncodeIndexes(s.Holder.Indexes()), From c14f72b66a8183ae5736f6a7825134368017d977 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 24 Oct 2017 10:40:21 -0500 Subject: [PATCH 40/48] changed the diagnostics configuration option to a boolean --- config.go | 13 +++++-------- ctl/server.go | 2 +- docs/administration.md | 2 +- docs/configuration.md | 10 +++++----- server.go | 1 + server/server.go | 7 ++++++- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/config.go b/config.go index 68a351523..07dcd94b5 100644 --- a/config.go +++ b/config.go @@ -43,9 +43,6 @@ const ( // DefaultMaxWritesPerRequest is the default number of writes per request. DefaultMaxWritesPerRequest = 5000 - - // DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. - DefaultDiagnosticsInterval = 1 * time.Hour ) // ClusterTypes set of cluster types. @@ -91,10 +88,10 @@ type Config struct { LogPath string `toml:"log-path"` Metric struct { - Service string `toml:"service"` - Host string `toml:"host"` - PollInterval Duration `toml:"poll-interval"` - DiagnosticInterval Duration `toml:"diagnostics"` + Service string `toml:"service"` + Host string `toml:"host"` + PollInterval Duration `toml:"poll-interval"` + Diagnostics bool `toml:"diagnostics"` } `toml:"metric"` TLS TLSConfig @@ -112,7 +109,7 @@ func NewConfig() *Config { c.Cluster.Hosts = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) c.Metric.Service = DefaultMetrics - c.Metric.DiagnosticInterval = Duration(DefaultDiagnosticsInterval) + c.Metric.Diagnostics = true c.TLS = TLSConfig{} return c } diff --git a/ctl/server.go b/ctl/server.go index a39c86e3e..46777df48 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -41,7 +41,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") - flags.DurationVarP((*time.Duration)(&srv.Config.Metric.DiagnosticInterval), "metric.diagnostics", "", time.Hour*1, "Diagnostic reporting interval back to Pilosa.") + flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) } diff --git a/docs/administration.md b/docs/administration.md index 861d7f6a1..8d8762086 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -131,7 +131,7 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi OpenFiles: Open file handle count. GoRoutines: Go routine count. -You can opt-out of the Pilosa diagnostics reporting by setting the `diagnostics` configuration option under `metric` to `0m0s`. +You can opt-out of the Pilosa diagnostics reporting by setting either the `metric.diagnostics` configuration option to false, using the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. #### Metrics diff --git a/docs/configuration.md b/docs/configuration.md index d786d0513..e74f6798f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,16 +206,16 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "0m15s" ``` -##### Metric Diagnostics Interval +##### Metric Diagnostics -* Description: Diagnostic reporting interval. To disable diagnostics set to zero. -* Flag: `metric.diagnostics=”60m0s”` -* Env: `PILOSA_METRIC_DIAGNOSTICS=60m0s` +* Description: Enable diagnostic reporting. To disable diagnostics set to false. +* Flag: `metric.diagnostics` +* Env: `PILOSA_METRIC_DIAGNOSTICS` * Config: ```toml [metric] - diagnostics = "60m0s" + diagnostics = true ``` diff --git a/server.go b/server.go index d9b4209fc..ff6a57808 100644 --- a/server.go +++ b/server.go @@ -517,6 +517,7 @@ func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint // monitorDiagnostics periodically polls the the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { if s.DiagnosticInterval <= 0 { + s.Logger().Printf("diagnostics disabled") return } diff --git a/server/server.go b/server/server.go index fb18137ad..7c4ad8081 100644 --- a/server/server.go +++ b/server/server.go @@ -43,6 +43,9 @@ func init() { const ( // DefaultDataDir is the default data directory. DefaultDataDir = "~/.pilosa" + + // DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. + DefaultDiagnosticsInterval = 1 * time.Hour ) // Command represents the state of the pilosa server command. @@ -143,8 +146,10 @@ func (m *Command) SetupServer() error { m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) + if m.Config.Metric.Diagnostics { + m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval) + } m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) - m.Server.DiagnosticInterval = time.Duration(m.Config.Metric.DiagnosticInterval) if err != nil { return err } From 030b0721c8c1125b31dc2ff8e8640053e75b95be Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 27 Oct 2017 16:58:00 -0500 Subject: [PATCH 41/48] Added a last checked version I'D. Fixed error messages. --- diagnostics/diagnostics.go | 29 +++++++++++++++++++---------- diagnostics/diagnostics_test.go | 10 +++++++--- docs/administration.md | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index 35d37f6cc..b836e4b37 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -30,14 +30,15 @@ type versionResponse struct { // Diagnostics represents a client to the Pilosa cluster. type Diagnostics struct { - mu sync.Mutex - wg sync.WaitGroup - closing chan struct{} - host string - VersionURL string - version string - startTime int64 - start time.Time + mu sync.Mutex + wg sync.WaitGroup + closing chan struct{} + host string + VersionURL string + version string + lastVersion string + startTime int64 + start time.Time metrics map[string]interface{} @@ -124,6 +125,8 @@ func (d *Diagnostics) Open() { st.Timeout = d.interval * 2 } d.cb = gobreaker.NewCircuitBreaker(st) + + d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/") } // Close notify goroutine to stop. @@ -149,6 +152,12 @@ func (d *Diagnostics) CheckVersion() error { return fmt.Errorf("json decode: %s", err) } + // Same a version as last test + if rsp.Version == d.lastVersion { + return nil + } + + d.lastVersion = rsp.Version if err := d.CompareVersion(rsp.Version); err != nil { d.logger().Printf("%s\n", err.Error()) } @@ -162,9 +171,9 @@ func (d *Diagnostics) CompareVersion(value string) error { localVersion := VersionSegments(d.version) if localVersion[0] < currentVersion[0] { //Major - return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Major release is %s", d.version, value) + return fmt.Errorf("Warning: You are running Pilosa %s, but a newer version is available %s", d.version, value) } else if localVersion[1] < currentVersion[1] { // Minor - return fmt.Errorf("Warning: You are running an older version of Pilosa %s. The latest Minor release is %s", d.version, value) + return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s", d.version, value) } else if localVersion[2] < currentVersion[2] { // Patch return fmt.Errorf("There is a new patch relese of Pilosa availbale: %s", value) } diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 81d6b7cf6..18ea8d38e 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -73,12 +73,16 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { d.Open() defer d.Close() - version := "0.1.1" + version := "v0.1.1" d.SetVersion(version) err := d.CompareVersion("1.7.0") - if !strings.Contains(err.Error(), "The latest Major release is") { - t.Fatalf("Expected Major Version Missmatch, actual error: %s", err) + if !strings.Contains(err.Error(), "a newer version is available ") { + t.Fatalf("Expected a newer version is available, actual error: %s", err) + } + err = d.CompareVersion("1.7.0") + if !strings.Contains(err.Error(), "a newer version is available ") { + t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.CompareVersion("0.7.0") if !strings.Contains(err.Error(), "The latest Minor release is") { diff --git a/docs/administration.md b/docs/administration.md index 8d8762086..23b5d17f0 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -131,7 +131,7 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi OpenFiles: Open file handle count. GoRoutines: Go routine count. -You can opt-out of the Pilosa diagnostics reporting by setting either the `metric.diagnostics` configuration option to false, using the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. +You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. #### Metrics From 27748259a2d2b49e797796c0485b9be65e3f1c83 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Oct 2017 13:32:26 -0500 Subject: [PATCH 42/48] Fix field value import: Use signed int and respect field minimum. Fixes #875 --- client.go | 6 +++--- client_test.go | 6 +++--- ctl/import.go | 2 +- frame.go | 9 +++++++-- index.go | 2 +- internal/public.pb.go | 23 ++++++++++++----------- internal/public.proto | 2 +- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/client.go b/client.go index 9297f4718..0a893ddf9 100644 --- a/client.go +++ b/client.go @@ -1131,7 +1131,7 @@ func (p Bits) GroupBySlice() map[uint64][]Bit { // range-encoded frame. type FieldValue struct { ColumnID uint64 - Value uint64 + Value int64 } // FieldValues represents a slice of field values. @@ -1154,8 +1154,8 @@ func (p FieldValues) ColumnIDs() []uint64 { } // Values returns a slice of all the values. -func (p FieldValues) Values() []uint64 { - other := make([]uint64, len(p)) +func (p FieldValues) Values() []int64 { + other := make([]int64, len(p)) for i := range p { other[i] = p[i].Value } diff --git a/client_test.go b/client_test.go index 20f601fe7..7e769381b 100644 --- a/client_test.go +++ b/client_test.go @@ -299,7 +299,7 @@ func TestClient_ImportValue(t *testing.T) { fld := pilosa.Field{ Name: "fld", Type: pilosa.FieldTypeInt, - Min: 0, + Min: -100, Max: 100, } @@ -320,7 +320,7 @@ func TestClient_ImportValue(t *testing.T) { // Send import request. c := test.MustNewClient(s.Host()) if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ - {ColumnID: 1, Value: 10}, + {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, {ColumnID: 3, Value: 40}, }); err != nil { @@ -333,7 +333,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify data. - if sum != 70 || cnt != 3 { + if sum != 50 || cnt != 3 { t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt) } } diff --git a/ctl/import.go b/ctl/import.go index d2f888956..179385db5 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -293,7 +293,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er val.ColumnID = columnID // Parse field value. - value, err := strconv.ParseUint(record[1], 10, 64) + value, err := strconv.ParseInt(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid value on row %d: %q", rnum, record[1]) } diff --git a/frame.go b/frame.go index 21f160a64..3da55cc93 100644 --- a/frame.go +++ b/frame.go @@ -882,7 +882,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro } // ImportValue bulk imports range-encoded value data. -func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error { +func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { // Verify that this frame is range-encoded. if !f.RangeEnabled() { return fmt.Errorf("Frame not RangeEnabled: %s", f.name) @@ -930,7 +930,12 @@ func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error return err } - if err := frag.ImportValue(data.ColumnIDs, data.Values, field.BitDepth()); err != nil { + baseValues := make([]uint64, len(data.Values)) + for i, value := range data.Values { + baseValues[i] = uint64(value - field.Min) + } + + if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil { return err } } diff --git a/index.go b/index.go index 3e6cb2518..2031ec5fe 100644 --- a/index.go +++ b/index.go @@ -668,7 +668,7 @@ type importData struct { type importValueData struct { ColumnIDs []uint64 - Values []uint64 + Values []int64 } // CreateInputDefinition creates a new input definition. diff --git a/internal/public.pb.go b/internal/public.pb.go index 81fb2267b..2b39cf4fb 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -440,7 +440,7 @@ type ImportValueRequest struct { Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - Values []uint64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` + Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` } func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } @@ -483,7 +483,7 @@ func (m *ImportValueRequest) GetColumnIDs() []uint64 { return nil } -func (m *ImportValueRequest) GetValues() []uint64 { +func (m *ImportValueRequest) GetValues() []int64 { if m != nil { return m.Values } @@ -1100,7 +1100,8 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { if len(m.Values) > 0 { dAtA16 := make([]byte, len(m.Values)*10) var j15 int - for _, num := range m.Values { + for _, num1 := range m.Values { + num := uint64(num1) for num >= 1<<7 { dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -3244,7 +3245,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } case 6: if wireType == 0 { - var v uint64 + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3254,7 +3255,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3284,7 +3285,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } for iNdEx < postIndex { - var v uint64 + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3294,7 +3295,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3433,7 +3434,7 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 653 bytes of a gzipped FileDescriptorProto + // 651 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x6e, 0xd3, 0x40, 0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0x14, 0x59, 0x2c, 0xbc, 0x4a, 0xa5, 0xf0, 0x01, 0x08, 0xb7, 0xa9, 0x64, 0x21, 0x2a, 0x98, 0x14, 0xf6, 0x6e, 0x3b, 0x2a, @@ -3472,7 +3473,7 @@ var fileDescriptorPublic = []byte{ 0x67, 0x5c, 0x79, 0xae, 0x51, 0x8e, 0x41, 0xa8, 0x73, 0x7b, 0xc5, 0x95, 0xd7, 0x27, 0x53, 0x4b, 0xa0, 0xce, 0xb7, 0x67, 0x8c, 0xda, 0x70, 0x02, 0x47, 0x74, 0x18, 0xff, 0x23, 0x03, 0x6e, 0x2a, 0x25, 0xdd, 0xff, 0xbf, 0x72, 0xd1, 0x37, 0x91, 0xa9, 0x19, 0x25, 0xfa, 0x22, 0xf8, 0x4b, 0xb1, - 0x87, 0x30, 0xa0, 0x2a, 0x4c, 0xa1, 0xae, 0x68, 0x50, 0x78, 0xf0, 0x6d, 0x33, 0x63, 0xdf, 0x37, - 0x33, 0xf6, 0x63, 0x33, 0x63, 0x1f, 0x7e, 0xce, 0xee, 0x5c, 0x0c, 0xe8, 0x07, 0xfd, 0xf8, 0x57, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x73, 0x96, 0xb9, 0xb0, 0x05, 0x00, 0x00, + 0x87, 0x30, 0xa0, 0x2a, 0x6c, 0xa1, 0x0d, 0x0a, 0x0f, 0xbe, 0x6d, 0x66, 0xec, 0xfb, 0x66, 0xc6, + 0x7e, 0x6c, 0x66, 0xec, 0xc3, 0xcf, 0xd9, 0x9d, 0x8b, 0x01, 0xfd, 0xa0, 0x1f, 0xff, 0x0a, 0x00, + 0x00, 0xff, 0xff, 0x4d, 0x1e, 0xdf, 0xba, 0xb0, 0x05, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 9a94fd469..47b04d8e7 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -79,5 +79,5 @@ message ImportValueRequest { uint64 Slice = 3; string Field = 4; repeated uint64 ColumnIDs = 5; - repeated uint64 Values = 6; + repeated int64 Values = 6; } From 98bd7b3ba25cb6712163ea04551f179672800711 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 1 Nov 2017 11:59:59 -0500 Subject: [PATCH 43/48] improvements to the version check message --- diagnostics/diagnostics.go | 8 ++++---- diagnostics/diagnostics_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index b836e4b37..c653a7052 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -126,7 +126,7 @@ func (d *Diagnostics) Open() { } d.cb = gobreaker.NewCircuitBreaker(st) - d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/") + d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics") } // Close notify goroutine to stop. @@ -171,11 +171,11 @@ func (d *Diagnostics) CompareVersion(value string) error { localVersion := VersionSegments(d.version) if localVersion[0] < currentVersion[0] { //Major - return fmt.Errorf("Warning: You are running Pilosa %s, but a newer version is available %s", d.version, value) + return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[1] < currentVersion[1] { // Minor - return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s", d.version, value) + return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[2] < currentVersion[2] { // Patch - return fmt.Errorf("There is a new patch relese of Pilosa availbale: %s", value) + return fmt.Errorf("There is a new patch release of Pilosa availbale: %s: https://github.com/pilosa/pilosa/releases", value) } return nil diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 18ea8d38e..6ad0cb773 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -76,12 +76,12 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { version := "v0.1.1" d.SetVersion(version) - err := d.CompareVersion("1.7.0") - if !strings.Contains(err.Error(), "a newer version is available ") { + err := d.CompareVersion("v1.7.0") + if !strings.Contains(err.Error(), "A newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.CompareVersion("1.7.0") - if !strings.Contains(err.Error(), "a newer version is available ") { + if !strings.Contains(err.Error(), "A newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.CompareVersion("0.7.0") @@ -89,7 +89,7 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) } err = d.CompareVersion("0.1.2") - if !strings.Contains(err.Error(), "There is a new patch relese of Pilosa") { + if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") { t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) } err = d.CompareVersion("0.1.1") From d50a0e241f0cf7b27e1c5add6778904071479db5 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 1 Nov 2017 12:00:20 -0500 Subject: [PATCH 44/48] gobreaker dep --- Gopkg.lock | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index 19483791d..09793a7e2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -161,6 +161,12 @@ packages = ["."] revision = "e2103e2c35297fb7e17febb81e49b312087a2372" +[[projects]] + name = "github.com/sony/gobreaker" + packages = ["."] + revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36" + version = "0.3.0" + [[projects]] branch = "master" name = "github.com/spf13/afero" @@ -230,6 +236,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e8e78a7c61547d8f4d967c8deac9334b3151e7c10c4f7909e80758956e9c8204" + inputs-digest = "0a7eaef315413bc55acf42c5566982fb71be8c11f33162127953572738967b35" solver-name = "gps-cdcl" solver-version = 1 From d10677bb6f20dd3b4a32312a61b75ef2bb125925 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 3 Nov 2017 18:02:21 -0500 Subject: [PATCH 45/48] Fix error format on slices. These tests were failing on `Go:master` (passing on `Go:1.8` and `Go:1.9`) --- client_test.go | 2 +- executor_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client_test.go b/client_test.go index 7e769381b..613d37659 100644 --- a/client_test.go +++ b/client_test.go @@ -90,7 +90,7 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %s", num, s[i].Host(), owns) + t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns) } } diff --git a/executor_test.go b/executor_test.go index bf7550b25..2a5266371 100644 --- a/executor_test.go +++ b/executor_test.go @@ -781,7 +781,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %s", result[0].(*pilosa.Bitmap).Bits()) + t.Fatalf("unexpected result: %v", result[0].(*pilosa.Bitmap).Bits()) } }) From 721baba105b8c9db90897dd437301aed1908a409 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 3 Nov 2017 18:02:21 -0500 Subject: [PATCH 46/48] Fix error format on slices. These tests were failing on `Go:master` (passing on `Go:1.8` and `Go:1.9`) --- client_test.go | 2 +- executor_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client_test.go b/client_test.go index 7e769381b..613d37659 100644 --- a/client_test.go +++ b/client_test.go @@ -90,7 +90,7 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %s", num, s[i].Host(), owns) + t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns) } } diff --git a/executor_test.go b/executor_test.go index bf7550b25..2a5266371 100644 --- a/executor_test.go +++ b/executor_test.go @@ -781,7 +781,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Bitmap).Bits()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %s", result[0].(*pilosa.Bitmap).Bits()) + t.Fatalf("unexpected result: %v", result[0].(*pilosa.Bitmap).Bits()) } }) From 1283263b707fc8a71f9ed288ea8e8351a44ca033 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 6 Nov 2017 14:01:09 -0600 Subject: [PATCH 47/48] missing formatting directive --- executor_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor_test.go b/executor_test.go index 2a5266371..4eb81e492 100644 --- a/executor_test.go +++ b/executor_test.go @@ -305,7 +305,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } else if value != 25 { - t.Fatal("unexpected value: %v", value) + t.Fatalf("unexpected value: %v", value) } if value, exists, err := f.FieldValue(10, "field1"); err != nil { @@ -313,7 +313,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } else if value != 2 { - t.Fatal("unexpected value: %v", value) + t.Fatalf("unexpected value: %v", value) } if value, exists, err := f.FieldValue(100, "field0"); err != nil { @@ -321,7 +321,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } else if value != 10 { - t.Fatal("unexpected value: %v", value) + t.Fatalf("unexpected value: %v", value) } }) From 64da894e7eda3803827a4b1de61807f299ee8f02 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 6 Nov 2017 14:08:01 -0600 Subject: [PATCH 48/48] adding uri to string method --- uri.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uri.go b/uri.go index ee5d23d75..21028fe44 100644 --- a/uri.go +++ b/uri.go @@ -134,6 +134,11 @@ func (u *URI) Normalize() string { return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port) } +// String returns the address as a string. +func (u URI) String() string { + return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port) +} + // Equals returns true if the checked URI is equivalent to this URI. func (u URI) Equals(other *URI) bool { if other == nil {