From b808c0f61016aeb331276209758831098724d9ad Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Mar 2014 16:42:36 -0500 Subject: [PATCH 1/5] Update nexter - add statsd and logfile support. --- commands/pilosa-nexter/nexter.go | 28 ++++++++++++++++++++++++---- deps.json | 6 ++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/commands/pilosa-nexter/nexter.go b/commands/pilosa-nexter/nexter.go index f2821c40e..b0bd84efd 100644 --- a/commands/pilosa-nexter/nexter.go +++ b/commands/pilosa-nexter/nexter.go @@ -6,9 +6,11 @@ import ( "fmt" "log" "net/http" + "os" "strconv" "strings" + "github.com/cactus/go-statsd-client/statsd" "github.com/coreos/go-etcd/etcd" ) @@ -26,11 +28,15 @@ func (self *stringslice) Set(value string) error { var port uint var blocksize uint64 var etcd_nodes stringslice +var stats string +var logfile string func init() { flag.UintVar(&port, "port", 9000, "Port to run HTTP server on") flag.Uint64Var(&blocksize, "blocksize", 64, "Block size") flag.Var(&etcd_nodes, "etcd", "Etcd server") + flag.String(stats, "statsd", "Statsd server") + flag.String(logfile, "log", "Log file name") flag.Parse() } @@ -48,6 +54,7 @@ type DelReq struct { type Nexter struct { reqchan chan Req done chan bool + stats *statsd.Client } func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { @@ -85,6 +92,7 @@ func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { if err != nil { continue } + self.stats.Gauge(path, int64(end), 1.0) for c := start; c < end; c += 1 { ch <- c } @@ -132,15 +140,27 @@ func (self *Nexter) Stop() { self.done <- true } -func NewNexter() *Nexter { - nexter := &Nexter{make(chan Req), make(chan bool)} +func NewNexter() (*Nexter, error) { + stats, err := statsd.New(stats, "") + if err != nil { + return nil, err + } + nexter := &Nexter{make(chan Req), make(chan bool), stats} go nexter.loop() - return nexter + return nexter, nil } func main() { + logf, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Println("Error opening file: %v", err) + } + log.SetOutput(logf) log.Println("Starting Nexter...") - nexter := NewNexter() + nexter, err := NewNexter() + if err != nil { + log.Fatal(err) + } http.HandleFunc("/nexter/", func(w http.ResponseWriter, r *http.Request) { url := r.URL.Path splits := strings.Split(url, "/") diff --git a/deps.json b/deps.json index db940d04e..a0cf0e7bc 100644 --- a/deps.json +++ b/deps.json @@ -73,5 +73,11 @@ "repo": "github.com/gorilla/websocket", "version": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c", "type": "git" + }, + "github.com/cactus/go-statsd-client/statsd": { + "repo": "github.com/cactus/go-statsd-client", + "version": "912f30c35e9cdf51f50bae24f071e227cca152fb", + "type": "git" } + } From 7724eba9d7b0397dd7b5d28b7deb33b384916311 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Mar 2014 16:45:55 -0500 Subject: [PATCH 2/5] oops --- deps.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps.json b/deps.json index a0cf0e7bc..feba75ef1 100644 --- a/deps.json +++ b/deps.json @@ -74,8 +74,8 @@ "version": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c", "type": "git" }, - "github.com/cactus/go-statsd-client/statsd": { - "repo": "github.com/cactus/go-statsd-client", + "statsd": { + "repo": "github.com/cactus/go-statsd-client/statsd", "version": "912f30c35e9cdf51f50bae24f071e227cca152fb", "type": "git" } From 33473bfaa0de21721e63f8c42c7c286faa3bb9a3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Mar 2014 17:01:47 -0500 Subject: [PATCH 3/5] oops --- commands/pilosa-nexter/nexter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/pilosa-nexter/nexter.go b/commands/pilosa-nexter/nexter.go index b0bd84efd..96c4e3085 100644 --- a/commands/pilosa-nexter/nexter.go +++ b/commands/pilosa-nexter/nexter.go @@ -35,8 +35,8 @@ func init() { flag.UintVar(&port, "port", 9000, "Port to run HTTP server on") flag.Uint64Var(&blocksize, "blocksize", 64, "Block size") flag.Var(&etcd_nodes, "etcd", "Etcd server") - flag.String(stats, "statsd", "Statsd server") - flag.String(logfile, "log", "Log file name") + flag.StringVar(&stats, "statsd", "127.0.0.1:8125", "Statsd server") + flag.StringVar(&logfile, "log", "/tmp/nexter.log", "Log file name") flag.Parse() } From b22760a5a09da0002c05e9fb2a79550eb073c844 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Mar 2014 17:07:28 -0500 Subject: [PATCH 4/5] use dot for statsd --- commands/pilosa-nexter/nexter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/pilosa-nexter/nexter.go b/commands/pilosa-nexter/nexter.go index 96c4e3085..83b0a5454 100644 --- a/commands/pilosa-nexter/nexter.go +++ b/commands/pilosa-nexter/nexter.go @@ -92,7 +92,7 @@ func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { if err != nil { continue } - self.stats.Gauge(path, int64(end), 1.0) + self.stats.Gauge("nexter."+strconv.Itoa(id), int64(end), 1.0) for c := start; c < end; c += 1 { ch <- c } From 0a480f8511d0697d0a06f4b1a253696df9067242 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 26 Mar 2014 10:47:39 -0500 Subject: [PATCH 5/5] Add extra logging --- commands/pilosa-nexter/nexter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/pilosa-nexter/nexter.go b/commands/pilosa-nexter/nexter.go index 83b0a5454..25926f429 100644 --- a/commands/pilosa-nexter/nexter.go +++ b/commands/pilosa-nexter/nexter.go @@ -90,9 +90,11 @@ func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) { } _, err = client.CompareAndSwap(path, strconv.FormatUint(end, 10), 0, strconv.FormatUint(start, 10), 0) if err != nil { + log.Println("CAS failure", path, start, end) continue } self.stats.Gauge("nexter."+strconv.Itoa(id), int64(end), 1.0) + log.Println("Allocated", id, start, end) for c := start; c < end; c += 1 { ch <- c }