From beb280624d99ef50ad9b07e15a767b0c593619d9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 25 Apr 2017 11:08:28 -0500 Subject: [PATCH 1/3] create/delete index/frame tag --- holder.go | 8 +++----- index.go | 6 ++---- stats_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/holder.go b/holder.go index ad329522f..2cfed1afc 100644 --- a/holder.go +++ b/holder.go @@ -94,8 +94,6 @@ func (h *Holder) Open() error { return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) } h.indexes[index.Name()] = index - - h.Stats.Count("indexN", 1) } // Periodically flush cache. @@ -233,8 +231,8 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index - - h.Stats.Count("indexN", 1) + fmt.Println("HERE") + h.Stats.Count("createIndex", 1) return index, nil } @@ -274,7 +272,7 @@ func (h *Holder) DeleteIndex(name string) error { // Remove reference. delete(h.indexes, name) - h.Stats.Count("indexN", -1) + h.Stats.Count("deleteIndex", 1) return nil } diff --git a/index.go b/index.go index 10790a079..ffcb53b61 100644 --- a/index.go +++ b/index.go @@ -164,8 +164,6 @@ func (i *Index) openFrames() error { return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) } i.frames[fr.Name()] = fr - - i.Stats.Count("frameN", 1) } return nil } @@ -405,7 +403,7 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { // Add to index's frame lookup. i.frames[name] = f - i.Stats.Count("frameN", 1) + i.Stats.Count("createFrame", 1) return f, nil } @@ -445,7 +443,7 @@ func (i *Index) DeleteFrame(name string) error { // Remove reference. delete(i.frames, name) - i.Stats.Count("frameN", -1) + i.Stats.Count("deleteFrame", 1) return nil } diff --git a/stats_test.go b/stats_test.go index 3875b6a0f..0c4e21f32 100644 --- a/stats_test.go +++ b/stats_test.go @@ -135,6 +135,38 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } } +func TestStatsCount_CreateIndex(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + + called := false + e := NewExecutor(hldr.Holder, NewCluster(1)) + idx := e.Holder.Index("d") + if idx == nil { + t.Fatal("index not found") + } + + e.Holder.Stats = &MockStats{ + mockCount: func(name string, value int64) { + if name != "createIndex" { + t.Errorf("Expected createIndex, Results %s", name) + } + + called = true + return + }, + } + if _, err := e.Execute(context.Background(), "d", MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + t.Fatal(err) + } + if !called { + t.Error("Count isn't called") + } +} + type MockStats struct { mockCount func(name string, value int64) mockCountWithTags func(name string, value int64, tags []string) From d18ee46d7654efcdc8e30b1d620fcec9a5fd2978 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 25 Apr 2017 14:32:11 -0500 Subject: [PATCH 2/3] index, frame statsD and tests --- holder.go | 1 - server.go | 1 - stats_test.go | 88 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/holder.go b/holder.go index 2cfed1afc..552b13d2a 100644 --- a/holder.go +++ b/holder.go @@ -231,7 +231,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index - fmt.Println("HERE") h.Stats.Count("createIndex", 1) return index, nil diff --git a/server.go b/server.go index fc1f11020..9ccb0ee84 100644 --- a/server.go +++ b/server.go @@ -201,7 +201,6 @@ func (s *Server) monitorAntiEntropy() { continue } - // Record successful sync in log. s.logger().Printf("holder sync complete") } diff --git a/stats_test.go b/stats_test.go index 0c4e21f32..62876cbd9 100644 --- a/stats_test.go +++ b/stats_test.go @@ -138,18 +138,8 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { func TestStatsCount_CreateIndex(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) - called := false - e := NewExecutor(hldr.Holder, NewCluster(1)) - idx := e.Holder.Index("d") - if idx == nil { - t.Fatal("index not found") - } - - e.Holder.Stats = &MockStats{ + hldr.Holder.Stats = &MockStats{ mockCount: func(name string, value int64) { if name != "createIndex" { t.Errorf("Expected createIndex, Results %s", name) @@ -159,9 +149,81 @@ func TestStatsCount_CreateIndex(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { - t.Fatal(err) + hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) + if !called { + t.Error("Count isn't called") } +} + +func TestStatsCount_DeleteIndex(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + called := false + hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) + hldr.Holder.Stats = &MockStats{ + mockCount: func(name string, value int64) { + if name != "deleteIndex" { + t.Errorf("Expected deleteIndex, Results %s", name) + } + + called = true + return + }, + } + hldr.DeleteIndex("d") + if !called { + t.Error("Count isn't called") + } +} + +func TestStatsCount_CreateFrame(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + called := false + hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) + e := NewExecutor(hldr.Holder, NewCluster(1)) + idx := e.Holder.Index("d") + if idx == nil { + t.Fatal("index not found") + } + idx.Stats = &MockStats{ + mockCount: func(name string, value int64) { + if name != "createFrame" { + t.Errorf("Expected createFrame, Results %s", name) + } + + called = true + return + }, + } + idx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}) + if !called { + t.Error("Count isn't called") + } +} + +func TestStatsCount_DeleteFrame(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + called := false + hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) + e := NewExecutor(hldr.Holder, NewCluster(1)) + idx := e.Holder.Index("d") + if idx == nil { + t.Fatal("index not found") + } + idx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}) + idx.Stats = &MockStats{ + mockCount: func(name string, value int64) { + if name != "deleteFrame" { + t.Errorf("Expected deleteFrame, Results %s", name) + } + + called = true + return + }, + } + idx.DeleteFrame("test") if !called { t.Error("Count isn't called") } From e67c9c051c2b87ad7f586c47b67f9540ce5269e0 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 27 Apr 2017 11:29:33 -0500 Subject: [PATCH 3/3] move statsd count to handler --- handler.go | 9 +++++++ holder.go | 3 --- index.go | 4 --- stats_test.go | 73 +++++++++++++++++++++++++++++++++------------------ 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/handler.go b/handler.go index 77dcb5700..1d2faf89f 100644 --- a/handler.go +++ b/handler.go @@ -338,6 +338,8 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } + + h.Holder.Stats.Count("deleteIndex", 1) } type deleteIndexResponse struct{} @@ -381,6 +383,8 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } + + h.Holder.Stats.Count("createIndex", 1) } // handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. @@ -531,6 +535,9 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } + + h.Holder.Stats.CountWithCustomTags("createFrame", 1, []string{fmt.Sprintf("index:%s", indexName)}) + } type _postFrameRequest postFrameRequest @@ -612,6 +619,8 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } + + h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, []string{fmt.Sprintf("index:%s", indexName)}) } type deleteFrameResponse struct{} diff --git a/holder.go b/holder.go index 552b13d2a..8e71ebe5b 100644 --- a/holder.go +++ b/holder.go @@ -231,7 +231,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index - h.Stats.Count("createIndex", 1) return index, nil } @@ -271,8 +270,6 @@ func (h *Holder) DeleteIndex(name string) error { // Remove reference. delete(h.indexes, name) - h.Stats.Count("deleteIndex", 1) - return nil } diff --git a/index.go b/index.go index ffcb53b61..516064f64 100644 --- a/index.go +++ b/index.go @@ -403,8 +403,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { // Add to index's frame lookup. i.frames[name] = f - i.Stats.Count("createFrame", 1) - return f, nil } @@ -443,8 +441,6 @@ func (i *Index) DeleteFrame(name string) error { // Remove reference. delete(i.frames, name) - i.Stats.Count("deleteFrame", 1) - return nil } diff --git a/stats_test.go b/stats_test.go index 62876cbd9..526b1deeb 100644 --- a/stats_test.go +++ b/stats_test.go @@ -2,10 +2,11 @@ package pilosa_test import ( "context" + "github.com/pilosa/pilosa" + "net/http" + "strings" "testing" "time" - - "github.com/pilosa/pilosa" ) func TestStatsCount_TopN(t *testing.T) { @@ -138,8 +139,11 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { func TestStatsCount_CreateIndex(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() + s := NewServer() + s.Handler.Holder = hldr.Holder + defer s.Close() called := false - hldr.Holder.Stats = &MockStats{ + s.Handler.Holder.Stats = &MockStats{ mockCount: func(name string, value int64) { if name != "createIndex" { t.Errorf("Expected createIndex, Results %s", name) @@ -149,7 +153,7 @@ func TestStatsCount_CreateIndex(t *testing.T) { return }, } - hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", nil)) if !called { t.Error("Count isn't called") } @@ -158,9 +162,17 @@ func TestStatsCount_CreateIndex(t *testing.T) { func TestStatsCount_DeleteIndex(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() + + s := NewServer() + s.Handler.Holder = hldr.Holder + defer s.Close() + + // Create index. + if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } called := false - hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) - hldr.Holder.Stats = &MockStats{ + s.Handler.Holder.Stats = &MockStats{ mockCount: func(name string, value int64) { if name != "deleteIndex" { t.Errorf("Expected deleteIndex, Results %s", name) @@ -170,7 +182,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { return }, } - hldr.DeleteIndex("d") + http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if !called { t.Error("Count isn't called") } @@ -179,24 +191,30 @@ func TestStatsCount_DeleteIndex(t *testing.T) { func TestStatsCount_CreateFrame(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - called := false - hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) - e := NewExecutor(hldr.Holder, NewCluster(1)) - idx := e.Holder.Index("d") - if idx == nil { - t.Fatal("index not found") + + s := NewServer() + s.Handler.Holder = hldr.Holder + defer s.Close() + + // Create index. + if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) } - idx.Stats = &MockStats{ - mockCount: func(name string, value int64) { + called := false + s.Handler.Holder.Stats = &MockStats{ + mockCountWithTags: func(name string, value int64, index []string) { if name != "createFrame" { t.Errorf("Expected createFrame, Results %s", name) } + if index[0] != "index:i" { + t.Errorf("Expected index:i, Results %s", index) + } called = true return }, } - idx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil)) if !called { t.Error("Count isn't called") } @@ -205,25 +223,30 @@ func TestStatsCount_CreateFrame(t *testing.T) { func TestStatsCount_DeleteFrame(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() + + s := NewServer() + s.Handler.Holder = hldr.Holder + defer s.Close() called := false - hldr.CreateIndexIfNotExists("d", pilosa.IndexOptions{}) - e := NewExecutor(hldr.Holder, NewCluster(1)) - idx := e.Holder.Index("d") - if idx == nil { - t.Fatal("index not found") + // Create index. + indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) } - idx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}) - idx.Stats = &MockStats{ - mockCount: func(name string, value int64) { + s.Handler.Holder.Stats = &MockStats{ + mockCountWithTags: func(name string, value int64, index []string) { if name != "deleteFrame" { t.Errorf("Expected deleteFrame, Results %s", name) } + if index[0] != "index:i" { + t.Errorf("Expected index:i, Results %s", index) + } called = true return }, } - idx.DeleteFrame("test") + http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader(""))) if !called { t.Error("Count isn't called") }