From 222206fc3f3fb1a756d01826ab854e2e7f3daf7c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 10:10:09 -0500 Subject: [PATCH 01/14] Add stats reporting in handler --- handler.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/handler.go b/handler.go index a6309e44a..be49a032f 100644 --- a/handler.go +++ b/handler.go @@ -46,6 +46,18 @@ type Handler struct { LogOutput io.Writer } +// Endpoints that are intended to be exposed to clients +var externalPrefixFlag = map[string]bool{ + "schema": true, + "query": true, + "import": true, + "export": true, + "db": true, + "frame": true, + "nodes": true, + "version": true, +} + // NewHandler returns a new instance of Handler with a default logger. func NewHandler() *Handler { handler := &Handler{ @@ -100,7 +112,31 @@ func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request // ServeHTTP handles an HTTP request. func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + t := time.Now() h.Router.ServeHTTP(w, r) + dif := time.Since(t).Seconds() + + // handle some stats tagging + statsTags := make([]string, 0, 3) + + if dif > 90 { + h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "longrunning") + } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, ".") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // internal = useragent:pilosa + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + + stats := h.Index.Stats.WithTags(statsTags...) + stats.Count("http.count"+endpointName, 1) + stats.Histogram("http.duration"+endpointName, dif) } // handleGetSchema handles GET /schema requests. From 985c0b4cdf308151c1dc81278abb18206ec52f77 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 10:11:19 -0500 Subject: [PATCH 02/14] Update tests to work with handler stats --- executor_test.go | 5 +++ handler_test.go | 82 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/executor_test.go b/executor_test.go index 7a0c95622..a54e394ed 100644 --- a/executor_test.go +++ b/executor_test.go @@ -489,6 +489,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() + s.Handler.Index = idx.Index idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(idx.Index, c) @@ -516,6 +517,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() + s.Handler.Index = idx.Index idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2) @@ -552,6 +554,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create local executor data. idx := MustOpenIndex() defer idx.Close() + s.Handler.Index = idx.Index // Create frame. if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -597,6 +600,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. idx := MustOpenIndex() defer idx.Close() + s.Handler.Index = idx.Index // Create frame. if f, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -664,6 +668,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 1 & 3. idx := MustOpenIndex() defer idx.Close() + s.Handler.Index = idx.Index idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2) diff --git a/handler_test.go b/handler_test.go index 478720e23..20944de7c 100644 --- a/handler_test.go +++ b/handler_test.go @@ -22,8 +22,14 @@ import ( // Ensure the handler returns "not found" for invalid paths. func TestHandler_NotFound(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil)) + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil)) if w.Code != http.StatusNotFound { t.Fatalf("invalid status: %d", w.Code) } @@ -130,7 +136,11 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) @@ -153,7 +163,11 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) @@ -188,7 +202,13 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Ensure the handler returns an error when parsing bad arguments. func TestHandler_Query_Args_Err(t *testing.T) { w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { @@ -198,7 +218,11 @@ func TestHandler_Query_Args_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -214,7 +238,11 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -237,7 +265,11 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -287,7 +319,11 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as protobuf. func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -386,7 +422,11 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -405,7 +445,11 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -431,7 +475,11 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -447,7 +495,11 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -470,8 +522,13 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { // Ensure the handler returns "method not allowed" for non-POST queries. func TestHandler_Query_MethodNotAllowed(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/db/d/query", nil)) + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/db/d/query", nil)) if w.Code != http.StatusMethodNotAllowed { t.Fatalf("invalid status: %d", w.Code) } @@ -479,7 +536,11 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { // Ensure the handler returns an error if there is a parsing error.. func TestHandler_Query_ErrParse(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -739,22 +800,29 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() - h.Version = "1.0.0" + h.Index = idx.Index w := httptest.NewRecorder() r := MustNewHTTPRequest("GET", "/version", nil) h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"version":"1.0.0"}`+"\n" { + } else if w.Body.String() != `{"version":"`+pilosa.Version+`"}`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } // Ensure the handler can return a list of nodes for a fragment. func TestHandler_Fragment_Nodes(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index h.Cluster = NewCluster(3) h.Cluster.ReplicaN = 2 @@ -770,7 +838,11 @@ func TestHandler_Fragment_Nodes(t *testing.T) { // Ensure the handler can return expvars without panicking. func TestHandler_Expvars(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + h := NewHandler() + h.Index = idx.Index w := httptest.NewRecorder() r := MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) From 83e13bec747e5d4c06c85ed6b77cb491b1c3f0ce Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 10:14:21 -0500 Subject: [PATCH 03/14] Move version vars to pilosa package --- Makefile | 2 +- cmd/root.go | 21 ++++----------------- cmd/server.go | 4 ++-- handler.go | 5 +---- version.go | 17 +++++++++++++++++ 5 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 version.go diff --git a/Makefile b/Makefile index 78c3e71f6..6641541fd 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa BUILD_TIME=`date -u +%FT%T%z` -LDFLAGS=-ldflags "-X github.com/pilosa/pilosa/cmd.Version=$(VERSION) -X github.com/pilosa/pilosa/cmd.BuildTime=$(BUILD_TIME)" +LDFLAGS=-ldflags "-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" default: test pilosa diff --git a/cmd/root.go b/cmd/root.go index 7cee0e77f..a918b0841 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,22 +5,18 @@ import ( "io" "strings" + "github.com/pilosa/pilosa" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" ) -var ( - Version string - BuildTime string -) - // TODO maybe give this an Add method which will ensure two command // with same name aren't added var subcommandFns = map[string]func(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command{} func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - setupVersionBuild() + // pilosa.SetupVersionBuild() // want to remove - see version.go rc := &cobra.Command{ Use: "pilosa", Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.", @@ -32,8 +28,8 @@ tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available at http://pilosa.com/docs -Version: ` + Version + ` -Build Time: ` + BuildTime + "\n", +Version: ` + pilosa.Version + ` +Build Time: ` + pilosa.BuildTime + "\n", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { v := viper.New() err := setAllConfig(v, cmd.Flags(), "PILOSA") @@ -63,15 +59,6 @@ Build Time: ` + BuildTime + "\n", return rc } -func setupVersionBuild() { - if Version == "" { - Version = "v0.0.0" - } - if BuildTime == "" { - BuildTime = "not recorded" - } -} - // setAllConfig takes a FlagSet to be the definition of all configuration // options, as well as their defaults. It then reads from the command line, the // environment, and a config file (if specified), and applies the configuration diff --git a/cmd/server.go b/cmd/server.go index 4e5d96114..d9863c47a 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" ) @@ -27,8 +28,7 @@ It will load existing data from the configured directory, and start listening client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - Server.Server.Handler.Version = Version - fmt.Fprintf(Server.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) + fmt.Fprintf(Server.Stderr, "Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) // Start CPU profiling. if Server.CPUProfile != "" { diff --git a/handler.go b/handler.go index be49a032f..014afea17 100644 --- a/handler.go +++ b/handler.go @@ -39,9 +39,6 @@ type Handler struct { Execute(context context.Context, db string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } - // The version to report on the /version endpoint. - Version string - // The writer for any logging. LogOutput io.Writer } @@ -1232,7 +1229,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(struct { Version string `json:"version"` }{ - Version: h.Version, + Version: Version, }); err != nil { h.logger().Printf("write version response error: %s", err) } diff --git a/version.go b/version.go new file mode 100644 index 000000000..42129f64a --- /dev/null +++ b/version.go @@ -0,0 +1,17 @@ +package pilosa + +var Version = "v0.0.0" +var BuildTime = "not recorded" + +// ldflags works without this - removing it allows TestHandler_Version to work simply +func SetupVersionBuild() { + /* + if Version == "" { + Version = "v0.0.0" + } + if BuildTime == "" { + BuildTime = "not recorded" + } + */ + +} From ef8892ad0187302614436c47fc121031ffeb6acf Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 10:15:58 -0500 Subject: [PATCH 04/14] Add versioned user-agent to requests --- client.go | 23 +++++++++++++++++++++++ executor.go | 1 + httpbroadcast/messenger.go | 1 + server.go | 1 + 4 files changed, 26 insertions(+) diff --git a/client.go b/client.go index e4401a69a..0955be3b0 100644 --- a/client.go +++ b/client.go @@ -73,6 +73,8 @@ func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[stri return nil, err } + req.Header.Set("User-Agent", "pilosa/"+Version) + // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -105,6 +107,8 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) { return nil, err } + req.Header.Set("User-Agent", "pilosa/"+Version) + // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -140,6 +144,7 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -181,6 +186,8 @@ func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([] return nil, err } + req.Header.Set("User-Agent", "pilosa/"+Version) + // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -228,6 +235,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -269,6 +277,8 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, if err != nil { return nil, err } + req.Header.Set("User-Agent", "pilosa/"+Version) + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -347,6 +357,7 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -423,6 +434,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string return err } req.Header.Set("Accept", "text/csv") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -557,6 +569,8 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, sl return nil, err } + req.Header.Set("User-Agent", "pilosa/"+Version) + // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -642,6 +656,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi return err } req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("User-Agent", "pilosa/"+Version) resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -681,6 +696,7 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -723,6 +739,7 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error return err } req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -752,6 +769,7 @@ func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, er return nil, err } req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -799,6 +817,8 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli return nil, err } + req.Header.Set("User-Agent", "pilosa/"+Version) + // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -844,6 +864,7 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui req.Header.Set("Content-Type", "application/protobuf") req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Accept", "application/protobuf") + req.Header.Set("User-Agent", "pilosa/"+Version) resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { @@ -890,6 +911,7 @@ func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBloc return nil, err } req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) @@ -933,6 +955,7 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At return nil, err } req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) diff --git a/executor.go b/executor.go index 9c156ec56..ae7cf2ed9 100644 --- a/executor.go +++ b/executor.go @@ -966,6 +966,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query // Require protobuf encoding. req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+Version) // Send request to remote node. resp, err := e.HTTPClient.Do(req) diff --git a/httpbroadcast/messenger.go b/httpbroadcast/messenger.go index ebd4b2e6a..6ef42f5be 100644 --- a/httpbroadcast/messenger.go +++ b/httpbroadcast/messenger.go @@ -83,6 +83,7 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *pilosa.Node, msg []byte) error { // Require protobuf encoding. req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Send request to remote node. resp, err := client.Do(req) diff --git a/server.go b/server.go index 68337049d..d8a2e458b 100644 --- a/server.go +++ b/server.go @@ -340,6 +340,7 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) { // Require protobuf encoding. req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+Version) // Send request to remote node. resp, err := http.DefaultClient.Do(req) From 78704dc55a5c2c8bf627e3651ddaaf37614423e1 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 14:55:07 -0500 Subject: [PATCH 05/14] Remove SetupVersionBuild --- cmd/root.go | 1 - version.go | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a918b0841..d7da24a0b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,6 @@ import ( var subcommandFns = map[string]func(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command{} func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - // pilosa.SetupVersionBuild() // want to remove - see version.go rc := &cobra.Command{ Use: "pilosa", Short: "Pilosa - A Distributed In-memory Binary Bitmap Index.", diff --git a/version.go b/version.go index 42129f64a..3c7d19ca3 100644 --- a/version.go +++ b/version.go @@ -2,16 +2,3 @@ package pilosa var Version = "v0.0.0" var BuildTime = "not recorded" - -// ldflags works without this - removing it allows TestHandler_Version to work simply -func SetupVersionBuild() { - /* - if Version == "" { - Version = "v0.0.0" - } - if BuildTime == "" { - BuildTime = "not recorded" - } - */ - -} From 3b415ef3fd3dacc5300bd47e8d2010c3088a2576 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 15:01:31 -0500 Subject: [PATCH 06/14] Simplify stats naming --- handler.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/handler.go b/handler.go index 014afea17..55ca9199d 100644 --- a/handler.go +++ b/handler.go @@ -122,7 +122,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, ".") + endpointName := strings.Join(pathParts, "_") if externalPrefixFlag[pathParts[1]] { statsTags = append(statsTags, "external") @@ -132,8 +132,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statsTags = append(statsTags, "useragent:"+r.UserAgent()) stats := h.Index.Stats.WithTags(statsTags...) - stats.Count("http.count"+endpointName, 1) - stats.Histogram("http.duration"+endpointName, dif) + stats.Histogram("http_"+endpointName, dif) } // handleGetSchema handles GET /schema requests. From a1ba58398d60e4c266af402c9fb3ffa33cef9b67 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 15:10:16 -0500 Subject: [PATCH 07/14] Improve comments --- handler.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/handler.go b/handler.go index 55ca9199d..542de01c1 100644 --- a/handler.go +++ b/handler.go @@ -43,7 +43,8 @@ type Handler struct { LogOutput io.Writer } -// Endpoints that are intended to be exposed to clients +// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. +// This is used for stats tagging. var externalPrefixFlag = map[string]bool{ "schema": true, "query": true, @@ -113,7 +114,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Router.ServeHTTP(w, r) dif := time.Since(t).Seconds() - // handle some stats tagging + // Handle some stats tagging statsTags := make([]string, 0, 3) if dif > 90 { @@ -128,7 +129,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statsTags = append(statsTags, "external") } - // internal = useragent:pilosa + // useragent tag identifies internal/external endpoints statsTags = append(statsTags, "useragent:"+r.UserAgent()) stats := h.Index.Stats.WithTags(statsTags...) From 6a162c4be29709e62d755fa14cc820f52e8e9e5b Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 21 Apr 2017 18:50:25 -0500 Subject: [PATCH 08/14] Support LongQueryTime as config option --- cmd/server.go | 1 + config.go | 5 +++-- handler.go | 15 ++++++++++----- server.go | 4 +++- server/server.go | 1 + 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index d9863c47a..14ec2e7a3 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -75,6 +75,7 @@ on the configured port.`, flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") + flags.DurationVarP((*time.Duration)(&Server.Config.LongQueryTime), "long-query-time", "", 10*time.Second, "Threshold for logging long-running queries (0 to disable)") flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&Server.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.StringSliceVarP(&Server.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") diff --git a/config.go b/config.go index a0b51a503..b0c0b9ee5 100644 --- a/config.go +++ b/config.go @@ -13,8 +13,9 @@ const ( // Config represents the configuration for the command. type Config struct { - DataDir string `toml:"data-dir"` - Host string `toml:"host"` + DataDir string `toml:"data-dir"` + Host string `toml:"host"` + LongQueryTime Duration `toml:"long-query-time"` Cluster struct { ReplicaN int `toml:"replicas"` diff --git a/handler.go b/handler.go index 542de01c1..3083d4695 100644 --- a/handler.go +++ b/handler.go @@ -41,6 +41,9 @@ type Handler struct { // The writer for any logging. LogOutput io.Writer + + // Threshold for logging long-running queries + LongQueryTime time.Duration } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -112,14 +115,16 @@ func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { t := time.Now() h.Router.ServeHTTP(w, r) - dif := time.Since(t).Seconds() + dif := time.Since(t) // Handle some stats tagging statsTags := make([]string, 0, 3) - if dif > 90 { - h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "longrunning") + fmt.Printf("long query time: %v\n", h.LongQueryTime) + + if h.LongQueryTime > 0 && dif > h.LongQueryTime { + h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), float64(dif)) + statsTags = append(statsTags, "slow_query") } pathParts := strings.Split(r.URL.Path, "/") @@ -133,7 +138,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statsTags = append(statsTags, "useragent:"+r.UserAgent()) stats := h.Index.Stats.WithTags(statsTags...) - stats.Histogram("http_"+endpointName, dif) + stats.Histogram("http_"+endpointName, float64(dif)) } // handleGetSchema handles GET /schema requests. diff --git a/server.go b/server.go index d8a2e458b..36aac1708 100644 --- a/server.go +++ b/server.go @@ -50,6 +50,9 @@ type Server struct { PollingInterval time.Duration MetricInterval time.Duration + // Threshold for logging long queries + LongQueryTime time.Duration + LogOutput io.Writer } @@ -71,7 +74,6 @@ func NewServer() *Server { } s.Handler.Index = s.Index - return s } diff --git a/server/server.go b/server/server.go index 7e4171d3d..9ce842376 100644 --- a/server/server.go +++ b/server/server.go @@ -174,6 +174,7 @@ func (m *Command) SetupServer() error { // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) + m.Server.Handler.LongQueryTime = time.Duration(m.Config.LongQueryTime) return nil } From 615c830ecc7965afced33607d6aa444815c753e7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 24 Apr 2017 11:31:50 -0500 Subject: [PATCH 09/14] Move LongQueryTime to Cluster --- cluster.go | 4 ++++ handler.go | 7 +------ server/server.go | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cluster.go b/cluster.go index f92b8c6e9..0d9b60fe3 100644 --- a/cluster.go +++ b/cluster.go @@ -3,6 +3,7 @@ package pilosa import ( "encoding/binary" "hash/fnv" + "time" ) const ( @@ -97,6 +98,9 @@ type Cluster struct { // The number of replicas a partition has. ReplicaN int + + // Threshold for logging long-running queries + LongQueryTime time.Duration } // NewCluster returns a new instance of Cluster with defaults. diff --git a/handler.go b/handler.go index 3083d4695..d75a60290 100644 --- a/handler.go +++ b/handler.go @@ -41,9 +41,6 @@ type Handler struct { // The writer for any logging. LogOutput io.Writer - - // Threshold for logging long-running queries - LongQueryTime time.Duration } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -120,9 +117,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Handle some stats tagging statsTags := make([]string, 0, 3) - fmt.Printf("long query time: %v\n", h.LongQueryTime) - - if h.LongQueryTime > 0 && dif > h.LongQueryTime { + if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), float64(dif)) statsTags = append(statsTags, "slow_query") } diff --git a/server/server.go b/server/server.go index 9ce842376..88ac54296 100644 --- a/server/server.go +++ b/server/server.go @@ -174,7 +174,7 @@ func (m *Command) SetupServer() error { // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - m.Server.Handler.LongQueryTime = time.Duration(m.Config.LongQueryTime) + m.Server.Cluster.LongQueryTime = time.Duration(m.Config.LongQueryTime) return nil } From bbffc8f5003243ab090ab7f6bfdd0c79949674fc Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 24 Apr 2017 11:32:22 -0500 Subject: [PATCH 10/14] Add Clusters to Handlers in tests --- handler_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/handler_test.go b/handler_test.go index 20944de7c..583ce6ace 100644 --- a/handler_test.go +++ b/handler_test.go @@ -28,6 +28,8 @@ func TestHandler_NotFound(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil)) if w.Code != http.StatusNotFound { @@ -61,6 +63,7 @@ func TestHandler_Schema(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { @@ -85,6 +88,7 @@ func TestHandler_MaxSlices(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { @@ -125,6 +129,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { @@ -140,6 +145,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { defer idx.Close() h := NewHandler() + h.Cluster = NewCluster(1) h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != "db0" { @@ -167,6 +173,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { defer idx.Close() h := NewHandler() + h.Cluster = NewCluster(1) h.Index = idx.Index h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != "db0" { @@ -206,6 +213,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { defer idx.Close() h := NewHandler() + h.Cluster = NewCluster(1) h.Index = idx.Index h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) @@ -223,6 +231,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -243,6 +252,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -270,6 +280,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -302,6 +313,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -324,6 +336,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -369,6 +382,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -427,6 +441,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -450,6 +465,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -480,6 +496,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -500,6 +517,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -527,6 +545,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/db/d/query", nil)) if w.Code != http.StatusMethodNotAllowed { @@ -541,6 +560,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -597,6 +617,7 @@ func TestHandler_DeleteFrame(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/db/d0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -616,6 +637,7 @@ func TestHandler_SetDBTimeQuantum(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -639,6 +661,7 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -805,6 +828,7 @@ func TestHandler_Version(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() r := MustNewHTTPRequest("GET", "/version", nil) @@ -843,6 +867,7 @@ func TestHandler_Expvars(t *testing.T) { h := NewHandler() h.Index = idx.Index + h.Cluster = NewCluster(1) w := httptest.NewRecorder() r := MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) From 5413597998f87f209f6016b6669d8c58c195bac2 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 24 Apr 2017 15:20:08 -0500 Subject: [PATCH 11/14] Move LongQueryTime option to cluster section of config file --- config.go | 6 +++--- handler.go | 1 + server/server.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index b0c0b9ee5..dcceb370e 100644 --- a/config.go +++ b/config.go @@ -13,9 +13,8 @@ const ( // Config represents the configuration for the command. type Config struct { - DataDir string `toml:"data-dir"` - Host string `toml:"host"` - LongQueryTime Duration `toml:"long-query-time"` + DataDir string `toml:"data-dir"` + Host string `toml:"host"` Cluster struct { ReplicaN int `toml:"replicas"` @@ -25,6 +24,7 @@ type Config struct { PollingInterval Duration `toml:"polling-interval"` InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` + LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` Plugins struct { diff --git a/handler.go b/handler.go index d75a60290..0c85d71d3 100644 --- a/handler.go +++ b/handler.go @@ -117,6 +117,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Handle some stats tagging statsTags := make([]string, 0, 3) + fmt.Printf("long query time: %v\n", h.Cluster.LongQueryTime) if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), float64(dif)) statsTags = append(statsTags, "slow_query") diff --git a/server/server.go b/server/server.go index 88ac54296..6914b49ff 100644 --- a/server/server.go +++ b/server/server.go @@ -174,7 +174,7 @@ func (m *Command) SetupServer() error { // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - m.Server.Cluster.LongQueryTime = time.Duration(m.Config.LongQueryTime) + m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) return nil } From 9bf6e1b3b7fbf0ea0f164a73f2a9c9626b926728 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 24 Apr 2017 15:22:12 -0500 Subject: [PATCH 12/14] Move LongQueryTime option to cluster section of config file --- cmd/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/server.go b/cmd/server.go index 14ec2e7a3..657a129a0 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -75,11 +75,11 @@ on the configured port.`, flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") - flags.DurationVarP((*time.Duration)(&Server.Config.LongQueryTime), "long-query-time", "", 10*time.Second, "Threshold for logging long-running queries (0 to disable)") flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&Server.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.StringSliceVarP(&Server.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? + flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.LongQueryTime), "long-query-time", "", 10*time.Second, "Threshold for logging long-running queries (0 to disable)") flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path") flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") From 684a8afa23ee4518baa334e58f1f9cd4a7b57fc6 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 25 Apr 2017 11:48:20 -0500 Subject: [PATCH 13/14] missed these db to index changes in the merge conflicts --- handler.go | 3 +-- handler_test.go | 35 +++++++++++++++++------------------ 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/handler.go b/handler.go index 9cf5af3c3..46713f0ba 100644 --- a/handler.go +++ b/handler.go @@ -121,7 +121,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Handle some stats tagging statsTags := make([]string, 0, 3) - fmt.Printf("long query time: %v\n", h.Cluster.LongQueryTime) if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { h.logger().Printf("%s %s %.03fs", r.Method, r.URL.String(), float64(dif)) statsTags = append(statsTags, "slow_query") @@ -137,7 +136,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // useragent tag identifies internal/external endpoints statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.Index.Stats.WithTags(statsTags...) + stats := h.Holder.Stats.WithTags(statsTags...) stats.Histogram("http_"+endpointName, float64(dif)) } diff --git a/handler_test.go b/handler_test.go index 86ad8538d..efa552653 100644 --- a/handler_test.go +++ b/handler_test.go @@ -22,13 +22,12 @@ import ( // Ensure the handler returns "not found" for invalid paths. func TestHandler_NotFound(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() h := NewHandler() - h.Index = idx.Index - h.Cluster = NewCluster(1) + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -215,7 +214,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { h := NewHandler() h.Cluster = NewCluster(1) h.Holder = hldr.Holder - + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) @@ -459,7 +458,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { } // Ensure the handler can execute a query that returns pairs as protobuf. -func TestHandler_Query_Pairs_Protobuf(t *testing.T) { +func TestHandler_Query_Pairs_Protobuf(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() @@ -555,12 +554,12 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { // Ensure the handler returns an error if there is a parsing error.. func TestHandler_Query_ErrParse(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() h := NewHandler() - h.Index = idx.Index h.Cluster = NewCluster(1) + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -823,12 +822,12 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() h := NewHandler() - h.Index = idx.Index h.Cluster = NewCluster(1) + h.Holder = hldr.Holder w := httptest.NewRecorder() r := MustNewHTTPRequest("GET", "/version", nil) @@ -842,11 +841,11 @@ func TestHandler_Version(t *testing.T) { // Ensure the handler can return a list of nodes for a fragment. func TestHandler_Fragment_Nodes(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder h.Cluster = NewCluster(3) h.Cluster.ReplicaN = 2 @@ -862,12 +861,12 @@ func TestHandler_Fragment_Nodes(t *testing.T) { // Ensure the handler can return expvars without panicking. func TestHandler_Expvars(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() h := NewHandler() - h.Index = idx.Index h.Cluster = NewCluster(1) + h.Holder = hldr.Holder w := httptest.NewRecorder() r := MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) From 4bb31d071eb84affb6b767da025a06403a9e30a0 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 9 May 2017 11:07:09 -0500 Subject: [PATCH 14/14] Change db to index --- handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler.go b/handler.go index 46713f0ba..c3a548a58 100644 --- a/handler.go +++ b/handler.go @@ -53,7 +53,7 @@ var externalPrefixFlag = map[string]bool{ "query": true, "import": true, "export": true, - "db": true, + "index": true, "frame": true, "nodes": true, "version": true,