From 99eb00bbdc7787c551996b14575fd981463a8435 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 23 Mar 2018 14:20:23 -0500 Subject: [PATCH 01/21] Remove obsolete coverage tools from Makefile Go 1.10 allows -coverprofile when testing multiple packages, so these hacks are no longer needed. --- Makefile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8fdbcea3d..9eb2df7e6 100644 --- a/Makefile +++ b/Makefile @@ -40,20 +40,10 @@ test: vendor go test $(PKGS) $(TESTFLAGS) cover: vendor - mkdir -p build/coverage - echo "mode: set" > build/coverage/all.out - for pkg in $(PKGS) ; do \ - make cover-pkg PKG=$$pkg ; \ - done - -cover-pkg: - mkdir -p build/coverage - touch build/coverage/$(subst /,-,$(PKG)).out - go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) - tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + make test TESTFLAGS="-coverprofile=build/coverage.out" cover-viz: cover - go tool cover -html=build/coverage/all.out + go tool cover -html=build/coverage.out pilosa: vendor go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From 2a8172b2a3100ead64830a93159cc490a73710e4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 26 Mar 2018 14:48:56 -0500 Subject: [PATCH 02/21] Remove RowLabel support --- executor.go | 47 ++++----- executor_test.go | 110 ++++++++++---------- fragment.go | 4 +- frame.go | 41 +------- frame_test.go | 43 -------- handler_internal_test.go | 9 +- handler_test.go | 4 +- index.go | 10 -- index_test.go | 34 +------ input_definition_test.go | 8 +- internal/private.pb.go | 214 +++++++++++++++------------------------ internal/private.proto | 1 - pilosa.go | 1 - server/cluster_test.go | 14 +-- server/server_test.go | 66 ++++++------ stats_test.go | 4 +- 16 files changed, 216 insertions(+), 394 deletions(-) diff --git a/executor.go b/executor.go index 854a9d8b9..de0d65df0 100644 --- a/executor.go +++ b/executor.go @@ -80,7 +80,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // MaxSlice can differ between inverse and standard views, so we need // to send queries to different slices based on orientation. var inverseSlices []uint64 - rowLabel := DefaultRowLabel columnLabel := DefaultColumnLabel // If slices aren't specified, then include all of them. @@ -131,10 +130,9 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic if f == nil { return nil, ErrFrameNotFound } - rowLabel = f.RowLabel() // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(rowLabel, columnLabel) { + if call.IsInverse(DefaultRowLabel, columnLabel) { slices = inverseSlices } } @@ -280,8 +278,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { frame, _ := c.Args["frame"].(string) if fr := idx.Frame(frame); fr != nil { - rowLabel := fr.RowLabel() - rowID, _, err := c.UintArg(rowLabel) + rowID, _, err := c.UintArg(DefaultRowLabel) if err != nil { return nil, err } @@ -536,18 +533,17 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() // Return an error if both the row and column label are specified. - rowID, rowOK, rowErr := c.UintArg(rowLabel) + rowID, rowOK, rowErr := c.UintArg(DefaultRowLabel) columnID, columnOK, columnErr := c.UintArg(columnLabel) if rowErr != nil || columnErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) } if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, columnLabel) } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, columnLabel) } // Determine row or column orientation. @@ -613,14 +609,13 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() // Read row & column id. columnID, columnOK, err := c.UintArg(columnLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) } - rowID, rowOK, err := c.UintArg(rowLabel) + rowID, rowOK, err := c.UintArg(DefaultRowLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } @@ -629,9 +624,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C var id uint64 var viewName string if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel) + return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, DefaultRowLabel) } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel) + return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, DefaultRowLabel) } else if columnOK { viewName, id = ViewInverse, columnID } else { @@ -906,14 +901,13 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal // Retrieve labels. columnLabel := idx.ColumnLabel() - rowLabel := f.RowLabel() // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("ClearBit() row field '%v' required", DefaultRowLabel) } colID, ok, err := c.UintArg(columnLabel) @@ -1000,14 +994,13 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, // Retrieve labels. columnLabel := idx.ColumnLabel() - rowLabel := f.RowLabel() // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { return false, fmt.Errorf("reading SetBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("SetBit() row field '%v' required", DefaultRowLabel) } colID, ok, err := c.UintArg(columnLabel) @@ -1171,20 +1164,19 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if frame == nil { return ErrFrameNotFound } - rowLabel := frame.RowLabel() // Parse labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetRowAttrs() row field '%v' required", rowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required", DefaultRowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, rowLabel) + delete(attrs, DefaultRowLabel) // Set attributes. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { @@ -1232,19 +1224,18 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { - return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) + return nil, fmt.Errorf("reading SetRowAttrs() row: %v", DefaultRowLabel) } else if !ok { - return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required", DefaultRowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, rowLabel) + delete(attrs, DefaultRowLabel) // Create frame group, if not exists. frameMap := m[frame] diff --git a/executor_test.go b/executor_test.go index c98239137..d90c8048f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit bits. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected bits: %+v", bits) @@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -122,7 +122,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { t.Fatalf("unexpected bits: %+v", bits) @@ -154,7 +154,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -184,7 +184,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -217,7 +217,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -233,7 +233,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -384,16 +384,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -419,15 +419,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=1) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, rowID=10, columnID=0) - SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, rowID=0, columnID=0) + SetBit(frame=f, row=0, columnID=0) + SetBit(frame=f, row=0, columnID=1) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, row=10, columnID=0) + SetBit(frame=f, row=10, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=20, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, row=0, columnID=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -543,7 +543,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -590,7 +590,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -630,8 +630,8 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, columnID=0) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) @@ -652,7 +652,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -679,22 +679,22 @@ func TestExecutor_Execute_Range(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, rowID=1, columnID=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, rowID=1, columnID=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, rowID=1, columnID=7, timestamp="2002-01-01T02:00") + SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-31T00:00") + SetBit(frame=f, row=1, columnID=3, timestamp="2000-01-01T00:00") + SetBit(frame=f, row=1, columnID=4, timestamp="2000-01-02T00:00") + SetBit(frame=f, row=1, columnID=5, timestamp="2000-02-01T00:00") + SetBit(frame=f, row=1, columnID=6, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, columnID=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, rowID=1, columnID=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, rowID=10, columnID=2, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-30T00:00") + SetBit(frame=f, row=1, columnID=2, timestamp="2002-02-01T00:00") + SetBit(frame=f, row=10, columnID=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) @@ -751,8 +751,8 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, columnID=0) + SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) @@ -908,7 +908,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", rowID=10)` { + } else if query.String() != `Bitmap(frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -931,7 +931,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected bits: %+v", bits) @@ -966,7 +966,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -994,7 +994,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10)` { + } else if query.String() != `SetBit(columnID=2, frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1012,7 +1012,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -1046,7 +1046,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(columnID=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1066,7 +1066,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -1169,7 +1169,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=10)"), nil, nil) + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=10)"), nil, nil) if err != nil { t.Fatal(err) } @@ -1186,7 +1186,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=20)"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=20)"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index ab397af47..8b32182d6 100644 --- a/fragment.go +++ b/fragment.go @@ -1841,11 +1841,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/frame.go b/frame.go index 466d7ae96..060125385 100644 --- a/frame.go +++ b/frame.go @@ -32,7 +32,7 @@ import ( // Default frame settings. const ( - DefaultRowLabel = "rowID" + DefaultRowLabel = "row" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false DefaultRangeEnabled = false @@ -87,7 +87,6 @@ func NewFrame(path, index, name string) (*Frame, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, @@ -139,39 +138,6 @@ func (f *Frame) MaxInverseSlice() uint64 { return view.MaxSlice() } -// SetRowLabel sets the row labels. Persists to meta file on update. -func (f *Frame) SetRowLabel(v string) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || f.rowLabel == v { - return nil - } - - // Make sure rowLabel is valid name - err := ValidateLabel(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - f.rowLabel = v - if err := f.saveMeta(); err != nil { - return err - } - - return nil -} - -// RowLabel returns the row label. -func (f *Frame) RowLabel() string { - f.mu.RLock() - v := f.rowLabel - f.mu.RUnlock() - return v -} - // CacheType returns the caching mode for the frame. func (f *Frame) CacheType() string { return f.cacheType @@ -224,7 +190,6 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ - RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, @@ -319,7 +284,6 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled f.cacheType = pb.CacheType if f.cacheType == "" { @@ -1030,7 +994,6 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` RangeEnabled bool `json:"rangeEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` @@ -1049,7 +1012,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { return nil } return &internal.FrameMeta{ - RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, @@ -1064,7 +1026,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { return nil } return &FrameOptions{ - RowLabel: options.RowLabel, InverseEnabled: options.InverseEnabled, RangeEnabled: options.RangeEnabled, CacheType: options.CacheType, diff --git a/frame_test.go b/frame_test.go index 6e4ade84a..2107bd703 100644 --- a/frame_test.go +++ b/frame_test.go @@ -267,49 +267,6 @@ func TestFrame_NameValidation(t *testing.T) { } } -// Ensure that frame RowLable validation is consistent. -func TestFrame_RowLabelValidation(t *testing.T) { - validRowLabels := []string{ - "", - "foo", - "hyphen-ated", - "under_score", - "abc123", - "trailing_", - "camelCase", - "UPPERCASE", - } - invalidRowLabels := []string{ - "123abc", - "x.y", - "_foo", - "-bar", - "abc def", - "a12345678901234567890123456789012345678901234567890123456789012345", - } - - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - f, err := pilosa.NewFrame(path, "i", "f") - if err != nil { - t.Fatalf("unexpected frame error: %s", err) - } - - for _, label := range validRowLabels { - if err := f.SetRowLabel(label); err != nil { - t.Fatalf("unexpected row label: %s %s", label, err) - } - } - for _, label := range invalidRowLabels { - if err := f.SetRowLabel(label); err == nil { - t.Fatalf("expected error on row label: %s", label) - } - } - -} - // Ensure frame can open and retrieve a view. func TestFrame_DeleteView(t *testing.T) { f := test.MustOpenFrame() diff --git a/handler_internal_test.go b/handler_internal_test.go index 169267c38..fb6226ea9 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -66,11 +66,10 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}}, - {json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, - {json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, + {json: `{"options": {"rowLabel": "test"}}`, err: "Unknown key: rowLabel:test"}, + {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, + {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { actual := &postFrameRequest{} diff --git a/handler_test.go b/handler_test.go index 551c81d66..3c9c08094 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1433,7 +1433,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { // Test input definition is deleted. index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -1470,7 +1470,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { h.Holder = hldr.Holder h.Cluster = test.NewCluster(1) - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} diff --git a/index.go b/index.go index 707b87213..968bac847 100644 --- a/index.go +++ b/index.go @@ -446,11 +446,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrInvalidCacheType } - // Validate that row label does not match column label. - if i.columnLabel == opt.RowLabel || (opt.RowLabel == "" && i.columnLabel == DefaultRowLabel) { - return nil, ErrColumnRowLabelEqual - } - // Validate mutually exclusive options if ranges are enabled. if opt.RangeEnabled { if opt.InverseEnabled { @@ -496,10 +491,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } f.cacheType = opt.CacheType - // Set options. - if opt.RowLabel != "" { - f.rowLabel = opt.RowLabel - } if opt.CacheSize != 0 { f.cacheSize = opt.CacheSize } @@ -694,7 +685,6 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin for _, fr := range pb.Frames { opt := FrameOptions{ // Deprecating row labels per #810. So, setting the default row label here. - RowLabel: DefaultRowLabel, InverseEnabled: fr.Meta.InverseEnabled, CacheType: fr.Meta.CacheType, CacheSize: fr.Meta.CacheSize, diff --git a/index_test.go b/index_test.go index 23a820f2d..c8cd238e8 100644 --- a/index_test.go +++ b/index_test.go @@ -215,32 +215,6 @@ func TestIndex_CreateFrame(t *testing.T) { } }) }) - - // Ensure frame cannot be created with a matching row label. - t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { - t.Run("Explicit", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - _, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel}) - if err != pilosa.ErrColumnRowLabelEqual { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("Default", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil { - t.Fatal(err) - } - - _, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != pilosa.ErrColumnRowLabelEqual { - t.Fatalf("unexpected error: %s", err) - } - }) - }) } // Ensure index can delete a frame. @@ -303,7 +277,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} @@ -330,7 +304,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { } // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -350,7 +324,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -381,7 +355,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} diff --git a/input_definition_test.go b/input_definition_test.go index 93d0e393d..efd13bec3 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -30,7 +30,7 @@ func TestInputDefinition_Open(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -115,14 +115,14 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err) } - frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{}} action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} @@ -131,7 +131,7 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err) } - frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}} action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} diff --git a/internal/private.pb.go b/internal/private.pb.go index 1b7f71ac4..488aca0fe 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -92,7 +92,6 @@ func (m *IndexMeta) GetTimeQuantum() string { } type FrameMeta struct { - RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` @@ -106,13 +105,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } -func (m *FrameMeta) GetRowLabel() string { - if m != nil { - return m.RowLabel - } - return "" -} - func (m *FrameMeta) GetInverseEnabled() bool { if m != nil { return m.InverseEnabled @@ -1278,12 +1270,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.RowLabel) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.RowLabel))) - i += copy(dAtA[i:], m.RowLabel) - } if m.InverseEnabled { dAtA[i] = 0x10 i++ @@ -2788,10 +2774,6 @@ func (m *IndexMeta) Size() (n int) { func (m *FrameMeta) Size() (n int) { var l int _ = l - l = len(m.RowLabel) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } if m.InverseEnabled { n += 2 } @@ -3583,35 +3565,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.RowLabel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) @@ -8667,89 +8620,88 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1334 bytes of a gzipped FileDescriptorProto + // 1326 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf6, 0x71, 0x9c, 0x38, 0xd3, 0x34, 0xaf, 0x13, 0x45, 0xae, 0x19, - 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0x34, 0x50, 0xa9, 0xc4, 0x76, 0xd5, 0x85, 0x26, 0x94, - 0x71, 0x12, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x24, 0xee, - 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x4f, 0x40, 0xe1, 0x47, 0x20, - 0x71, 0x03, 0x9a, 0xaf, 0xdd, 0xf5, 0x57, 0xd2, 0x04, 0xee, 0xf6, 0x3c, 0x73, 0xce, 0x99, 0x67, - 0xce, 0xd7, 0xcc, 0x42, 0xb9, 0x1f, 0xba, 0xa7, 0x94, 0xb3, 0xcd, 0x7e, 0x18, 0xf0, 0x00, 0x15, - 0x5c, 0x9f, 0xb3, 0xd0, 0xa7, 0x1e, 0xfe, 0x14, 0x8a, 0x8e, 0xdf, 0x65, 0xe7, 0xbb, 0x8c, 0x53, - 0x54, 0x87, 0x52, 0x23, 0xf0, 0x06, 0x3d, 0xff, 0x39, 0x3d, 0x62, 0x5e, 0xd5, 0xaa, 0x5b, 0x1b, - 0x45, 0x92, 0x86, 0x84, 0xc6, 0xbe, 0xdb, 0x63, 0x9f, 0x0d, 0xa8, 0xcf, 0x07, 0xbd, 0x6a, 0x46, - 0x69, 0xa4, 0x20, 0xfc, 0x97, 0x05, 0xc5, 0xa7, 0x21, 0xed, 0x31, 0xe9, 0x71, 0x0d, 0x0a, 0x24, - 0x38, 0x4b, 0xbb, 0x8b, 0x65, 0xf4, 0x16, 0x2c, 0x38, 0xfe, 0x29, 0x0b, 0x23, 0xd6, 0xf2, 0xe9, - 0x91, 0xc7, 0xba, 0xd2, 0x5d, 0x81, 0x8c, 0xa1, 0x68, 0x1d, 0x8a, 0x0d, 0xda, 0x79, 0xc9, 0xf6, - 0x87, 0x7d, 0x56, 0xb5, 0xa5, 0x93, 0x04, 0x88, 0x57, 0xdb, 0xee, 0x2b, 0x56, 0xcd, 0xd6, 0xad, - 0x8d, 0x32, 0x49, 0x80, 0x71, 0xbe, 0xb9, 0x09, 0xbe, 0x08, 0xc3, 0x3c, 0xa1, 0xfe, 0x71, 0xcc, - 0x21, 0x2f, 0x39, 0x8c, 0x60, 0xe8, 0x1e, 0xe4, 0x9f, 0xba, 0xcc, 0xeb, 0x46, 0xd5, 0xb9, 0xba, - 0xbd, 0x51, 0xda, 0x5a, 0xdc, 0x34, 0xf1, 0xdb, 0x94, 0x38, 0xd1, 0xcb, 0x18, 0xc3, 0x82, 0xd3, - 0xeb, 0x07, 0x21, 0x27, 0x2c, 0xea, 0x07, 0x7e, 0xc4, 0x50, 0x05, 0xec, 0x56, 0x18, 0xea, 0xb3, - 0x8b, 0x4f, 0xfc, 0x2d, 0x54, 0x76, 0xbc, 0xa0, 0x73, 0xd2, 0xa4, 0x9c, 0x12, 0xf6, 0xcd, 0x80, - 0x45, 0x1c, 0x2d, 0x43, 0x4e, 0x66, 0x41, 0xeb, 0x29, 0x41, 0xa0, 0x32, 0x92, 0x3a, 0xcc, 0x4a, - 0x10, 0xa8, 0xb4, 0x97, 0xa1, 0xc8, 0x12, 0x25, 0x08, 0xb4, 0xed, 0xb9, 0x1d, 0x15, 0x82, 0x2c, - 0x51, 0x02, 0x42, 0x90, 0x3d, 0x74, 0xd9, 0x99, 0x3e, 0xb7, 0xfc, 0xc6, 0x0e, 0x2c, 0xa5, 0xf6, - 0xd7, 0x34, 0x57, 0x20, 0x4f, 0x82, 0x33, 0xa7, 0x19, 0x55, 0xad, 0xba, 0xbd, 0x91, 0x25, 0x5a, - 0x92, 0xd1, 0x95, 0xe9, 0x17, 0x4b, 0x19, 0xb9, 0x94, 0x00, 0x78, 0x15, 0x72, 0x32, 0xd4, 0xe2, - 0x94, 0x89, 0xad, 0xf8, 0xc4, 0x7f, 0x5b, 0x50, 0xdc, 0xa5, 0xe7, 0x92, 0x46, 0x84, 0x1e, 0x43, - 0xa1, 0xcd, 0xa9, 0xdf, 0xa5, 0x61, 0x57, 0x2a, 0x95, 0xb6, 0xde, 0x48, 0x42, 0x18, 0xab, 0x6d, - 0x1a, 0x9d, 0x96, 0xcf, 0xc3, 0x21, 0x89, 0x4d, 0xd0, 0x36, 0xcc, 0xe9, 0x9a, 0x90, 0x1c, 0x4a, - 0x5b, 0xf5, 0x69, 0xd6, 0x71, 0xd9, 0x08, 0x63, 0x63, 0xb0, 0xf6, 0x01, 0x94, 0x47, 0xdc, 0x0a, - 0xae, 0x27, 0x6c, 0x68, 0x32, 0x72, 0xc2, 0x86, 0x22, 0x76, 0xa7, 0xd4, 0x1b, 0xa8, 0x38, 0x67, - 0x89, 0x12, 0xb6, 0x33, 0xef, 0x5b, 0x6b, 0xdb, 0x30, 0x9f, 0xf6, 0x7a, 0x1d, 0x5b, 0xfc, 0x15, - 0xa0, 0x46, 0xc8, 0x28, 0x67, 0x92, 0xde, 0x2e, 0x8b, 0x22, 0x7a, 0xcc, 0x66, 0x67, 0x5a, 0x65, - 0x2f, 0x93, 0xce, 0xde, 0x3a, 0x14, 0x9d, 0xc8, 0x1c, 0xdc, 0x96, 0x75, 0x99, 0x00, 0xf8, 0x3e, - 0xa0, 0x26, 0xf3, 0x18, 0x67, 0xba, 0x7f, 0x2f, 0xf1, 0x8f, 0xdb, 0x86, 0xcb, 0xd5, 0xba, 0xe8, - 0x1e, 0x64, 0x45, 0xeb, 0x4a, 0x2a, 0xa5, 0xad, 0x5b, 0x49, 0xa4, 0xe3, 0x39, 0x41, 0xa4, 0x02, - 0x76, 0x8d, 0x53, 0xdd, 0xee, 0x57, 0x1c, 0x70, 0x4a, 0x29, 0x9b, 0xad, 0xec, 0xf1, 0xad, 0xe2, - 0x01, 0xa2, 0xb7, 0x7a, 0x62, 0xce, 0x7a, 0xd3, 0xad, 0xf0, 0x71, 0x4c, 0x56, 0x74, 0xea, 0x4d, - 0xc8, 0xbe, 0x09, 0x39, 0x69, 0xab, 0xd9, 0x4e, 0xcc, 0x00, 0xb5, 0x8a, 0x0f, 0x63, 0xaa, 0x37, - 0xdd, 0x68, 0x39, 0xbd, 0x51, 0xd1, 0xf8, 0xfd, 0x42, 0xeb, 0x8a, 0x9e, 0xde, 0x13, 0x36, 0xca, - 0x93, 0xfc, 0x9e, 0x9d, 0xb3, 0xb1, 0x40, 0x0a, 0xdf, 0x62, 0x08, 0x44, 0x55, 0xbb, 0x6e, 0x0b, - 0xdf, 0x52, 0xc0, 0x0f, 0x21, 0xdf, 0xee, 0xbc, 0x64, 0x3d, 0x8a, 0xde, 0x16, 0x9d, 0xd6, 0x65, - 0xe7, 0x2c, 0xd2, 0x7d, 0xba, 0x38, 0x96, 0x7f, 0x62, 0xd6, 0xf1, 0x0f, 0x96, 0x3e, 0xd3, 0x0c, - 0x46, 0x79, 0xb9, 0x77, 0x54, 0xcd, 0x4e, 0x8c, 0x4c, 0x81, 0x13, 0xbd, 0x8c, 0x5a, 0x50, 0x71, - 0xfc, 0xfe, 0x80, 0x37, 0xd9, 0xd7, 0xae, 0xef, 0x72, 0x37, 0xf0, 0xa3, 0x6a, 0x5e, 0x9a, 0xac, - 0xa6, 0xb7, 0x1e, 0xd1, 0x20, 0x13, 0x26, 0xf8, 0x3b, 0x0b, 0x16, 0xc7, 0xc0, 0x2b, 0x78, 0x65, - 0x2e, 0xe7, 0xf5, 0x5e, 0x3c, 0xf3, 0x6d, 0xa9, 0x58, 0x9b, 0xc9, 0x66, 0xf4, 0x0a, 0xf8, 0xc5, - 0x82, 0xe5, 0x69, 0x0a, 0x53, 0xd9, 0xd4, 0x00, 0x5e, 0x84, 0x6e, 0x8f, 0x86, 0xc3, 0x4f, 0xd8, - 0x50, 0x5f, 0x7f, 0x29, 0x04, 0x7d, 0x0e, 0x2b, 0x63, 0xbe, 0x3e, 0xea, 0xa8, 0x10, 0x29, 0x52, - 0x77, 0x66, 0x92, 0x52, 0x7a, 0x64, 0x86, 0x39, 0xfe, 0xd3, 0x82, 0xdb, 0x53, 0x97, 0x92, 0x9a, - 0xb4, 0xd2, 0x35, 0x79, 0x1f, 0x2a, 0x87, 0x62, 0xb2, 0x35, 0x59, 0xc4, 0x5d, 0x9f, 0x0a, 0x4d, - 0x5d, 0xb4, 0x13, 0x38, 0x72, 0xa0, 0x20, 0xb1, 0x5d, 0xda, 0xd7, 0x34, 0xdf, 0xb9, 0x82, 0xe6, - 0xa6, 0xd1, 0xd7, 0x83, 0xdf, 0x88, 0x82, 0x8c, 0xbc, 0x88, 0xcc, 0xad, 0x26, 0x05, 0x31, 0xd2, - 0x47, 0x0c, 0xae, 0x35, 0x96, 0x03, 0x58, 0x37, 0xa3, 0x70, 0x84, 0xc9, 0xe5, 0x9d, 0xfa, 0x08, - 0x20, 0x51, 0xd5, 0x13, 0xe0, 0x92, 0xfa, 0x4c, 0x29, 0xe3, 0x67, 0xb0, 0x6e, 0xe6, 0xf4, 0x35, - 0x36, 0x34, 0xd5, 0x92, 0x49, 0xaa, 0x05, 0xb7, 0xc0, 0x3e, 0x20, 0x8e, 0xb8, 0xab, 0x65, 0xb7, - 0x9a, 0x14, 0x69, 0x49, 0x98, 0x3c, 0x0b, 0x22, 0x6e, 0x4c, 0xc4, 0xb7, 0xc0, 0x5e, 0x04, 0x21, - 0x97, 0x8c, 0xcb, 0x44, 0x7e, 0xe3, 0x2f, 0x21, 0xbb, 0x17, 0x74, 0x19, 0x5a, 0x80, 0x8c, 0xd3, - 0xd4, 0x3e, 0x32, 0x4e, 0x13, 0xdd, 0x91, 0xee, 0xf5, 0x0c, 0x29, 0x27, 0x87, 0x3b, 0x20, 0x0e, - 0x91, 0x1b, 0xdf, 0x85, 0xb2, 0x13, 0x35, 0x82, 0x20, 0xec, 0x8a, 0x54, 0x07, 0xa1, 0xbe, 0x93, - 0x46, 0x41, 0xfc, 0x04, 0x2a, 0xc2, 0x7d, 0x9b, 0x53, 0x1e, 0x4f, 0xea, 0x15, 0xc8, 0x0b, 0x2c, - 0xde, 0x4e, 0x4b, 0xf2, 0xde, 0x13, 0x7a, 0x66, 0x00, 0x4a, 0x01, 0x3f, 0x57, 0x1e, 0x5a, 0xa7, - 0xcc, 0xe7, 0xa9, 0x28, 0x49, 0x59, 0x3a, 0x28, 0x13, 0x25, 0x20, 0xac, 0x8e, 0xa2, 0x39, 0x2f, - 0x24, 0x9c, 0x05, 0x4a, 0xe4, 0x1a, 0xfe, 0xd1, 0x02, 0x30, 0x84, 0x06, 0x51, 0x6c, 0x62, 0xcd, - 0x36, 0x41, 0xef, 0xa6, 0xde, 0x2e, 0x93, 0x33, 0x35, 0x5e, 0x22, 0xa9, 0x17, 0xce, 0x86, 0x19, - 0xa1, 0xba, 0x38, 0x2a, 0x89, 0xbe, 0xc2, 0x75, 0x9a, 0xc4, 0xb5, 0x59, 0x6e, 0x78, 0x83, 0x88, - 0xb3, 0x50, 0x33, 0x12, 0x6f, 0x2c, 0x05, 0xc4, 0xf1, 0x49, 0x80, 0xe9, 0x21, 0x42, 0x77, 0x21, - 0x27, 0x98, 0x9a, 0x39, 0x30, 0x7e, 0x0c, 0xb5, 0x88, 0xdb, 0xfa, 0x26, 0x99, 0x3a, 0x7b, 0x10, - 0x64, 0xe5, 0x8b, 0x5a, 0x97, 0x8b, 0x7c, 0x4c, 0x57, 0xc0, 0xde, 0x75, 0x55, 0x7d, 0xdb, 0x44, - 0x7c, 0x4a, 0x84, 0x9e, 0xcb, 0xfe, 0x13, 0x08, 0x15, 0x6f, 0x89, 0x25, 0xd5, 0x40, 0xe2, 0xee, - 0xb8, 0xc9, 0xfd, 0x66, 0x1e, 0xa5, 0x76, 0xea, 0x51, 0xda, 0x86, 0x25, 0xd5, 0x24, 0xff, 0xa5, - 0xd3, 0x9f, 0x33, 0xb0, 0x44, 0x58, 0xe4, 0xbe, 0x62, 0x8e, 0x1f, 0xf1, 0x70, 0x10, 0x0f, 0xb8, - 0x8f, 0x83, 0x23, 0x1d, 0x6a, 0x9b, 0x28, 0xe1, 0x75, 0x2a, 0x09, 0x3d, 0x10, 0xbf, 0x47, 0xa3, - 0xd5, 0x3f, 0xa9, 0x9a, 0x56, 0x41, 0x0f, 0x60, 0xae, 0x1d, 0x0c, 0xc2, 0x4e, 0x7c, 0x0d, 0xae, - 0x24, 0xda, 0x8a, 0x99, 0x5a, 0x26, 0x46, 0x2d, 0x55, 0x47, 0xb9, 0xcb, 0xeb, 0x08, 0x3d, 0x1e, - 0xab, 0x23, 0xf9, 0xe7, 0x52, 0xda, 0xfa, 0x7f, 0x62, 0x30, 0xb2, 0x4c, 0x46, 0xb5, 0xf1, 0xf7, - 0x16, 0xcc, 0xa7, 0x29, 0xbc, 0x56, 0x63, 0xc4, 0x19, 0xc9, 0x4c, 0xcd, 0x88, 0x3d, 0x2d, 0x23, - 0xd9, 0x24, 0x23, 0xc9, 0x3b, 0x37, 0x97, 0x7a, 0xe7, 0xe2, 0x13, 0x58, 0x9d, 0x48, 0x53, 0x23, - 0xe8, 0xf5, 0x45, 0x3d, 0xfc, 0x8b, 0x74, 0x89, 0x91, 0x11, 0x86, 0x3a, 0x51, 0x45, 0xa2, 0x04, - 0xfc, 0x08, 0x6e, 0xb7, 0x19, 0x4f, 0x25, 0xc9, 0x54, 0x5b, 0x1d, 0xec, 0x3d, 0x76, 0x36, 0xe3, - 0xf8, 0x62, 0x09, 0x7f, 0x08, 0xd5, 0x83, 0x7e, 0x97, 0x72, 0x76, 0x23, 0xeb, 0x1d, 0x28, 0xec, - 0x07, 0xfd, 0xc0, 0x0b, 0x8e, 0x87, 0x57, 0xb4, 0x7c, 0x15, 0xe6, 0xd4, 0x7c, 0x54, 0x8f, 0x94, - 0x22, 0x31, 0x22, 0xbe, 0x25, 0x0a, 0xba, 0x43, 0xbd, 0xce, 0xc0, 0x13, 0x34, 0xc4, 0xbf, 0x57, - 0xb4, 0x53, 0xf9, 0xf5, 0xa2, 0x66, 0xfd, 0x76, 0x51, 0xb3, 0x7e, 0xbf, 0xa8, 0x59, 0x3f, 0xfd, - 0x51, 0xfb, 0xdf, 0x51, 0x5e, 0xfe, 0xe5, 0x3f, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x66, 0x19, - 0x3d, 0xd2, 0xf6, 0x0f, 0x00, 0x00, + 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf1, 0x71, 0x9d, 0x3a, 0xd3, 0x36, 0xaf, 0x5b, 0x45, 0xae, 0x19, + 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0xb4, 0x50, 0xa9, 0xd4, 0x76, 0xd5, 0x85, 0xa6, 0x94, + 0x71, 0x5b, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x34, 0xee, + 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x2f, 0x40, 0xa8, 0xfc, 0x08, + 0x2e, 0x41, 0x67, 0x3e, 0x76, 0xd7, 0x5f, 0x49, 0x1b, 0xb8, 0xdb, 0xf3, 0xcc, 0x39, 0x67, 0x9e, + 0x39, 0x5f, 0x33, 0x0b, 0xb5, 0x51, 0xec, 0x1f, 0x72, 0x29, 0xb6, 0x47, 0x71, 0x24, 0x23, 0xb2, + 0xea, 0x87, 0x52, 0xc4, 0x21, 0x0f, 0xe8, 0xa7, 0x50, 0xf1, 0xc2, 0x81, 0x38, 0xda, 0x15, 0x92, + 0x93, 0x16, 0x54, 0xdb, 0x51, 0x30, 0x1e, 0x86, 0x0f, 0xf8, 0x9e, 0x08, 0x1a, 0x4e, 0xcb, 0xd9, + 0xaa, 0xb0, 0x3c, 0x84, 0x1a, 0x8f, 0xfd, 0xa1, 0xf8, 0x6c, 0xcc, 0x43, 0x39, 0x1e, 0x36, 0x0a, + 0x5a, 0x23, 0x07, 0xd1, 0xdf, 0x1d, 0xa8, 0xdc, 0x8b, 0xf9, 0x50, 0x28, 0x8f, 0x6f, 0xc1, 0x9a, + 0x17, 0x1e, 0x8a, 0x38, 0x11, 0xdd, 0x90, 0xef, 0x05, 0x62, 0xa0, 0x4c, 0x56, 0xd9, 0x0c, 0x4a, + 0x36, 0xa1, 0xd2, 0xe6, 0xfd, 0x67, 0xe2, 0xf1, 0x64, 0x24, 0x1a, 0xae, 0xf2, 0x9a, 0x01, 0xe9, + 0x6a, 0xcf, 0x7f, 0x21, 0x1a, 0xc5, 0x96, 0xb3, 0x55, 0x63, 0x19, 0x30, 0xcb, 0xa9, 0x34, 0xc7, + 0x89, 0x50, 0x38, 0xc3, 0x78, 0xb8, 0x9f, 0x72, 0x28, 0x2b, 0x0e, 0x53, 0x18, 0xb9, 0x0a, 0xe5, + 0x7b, 0xbe, 0x08, 0x06, 0x49, 0x63, 0xa5, 0xe5, 0x6e, 0x55, 0x77, 0xce, 0x6e, 0xdb, 0x18, 0x6d, + 0x2b, 0x9c, 0x99, 0x65, 0x4a, 0x61, 0xcd, 0x1b, 0x8e, 0xa2, 0x58, 0x32, 0x91, 0x8c, 0xa2, 0x30, + 0x11, 0xa4, 0x0e, 0x6e, 0x37, 0x8e, 0x4d, 0xb8, 0xf0, 0x93, 0x7e, 0x0b, 0xf5, 0xbb, 0x41, 0xd4, + 0x3f, 0xe8, 0x70, 0xc9, 0x99, 0xf8, 0x66, 0x2c, 0x12, 0x49, 0xce, 0x43, 0x49, 0x45, 0xda, 0xe8, + 0x69, 0x01, 0x51, 0x15, 0x2d, 0x13, 0x4a, 0x2d, 0x20, 0xaa, 0xec, 0x55, 0x28, 0x8a, 0x4c, 0x0b, + 0x88, 0xf6, 0x02, 0xbf, 0xaf, 0x43, 0x50, 0x64, 0x5a, 0x20, 0x04, 0x8a, 0x4f, 0x7d, 0xf1, 0xdc, + 0x9c, 0x5b, 0x7d, 0x53, 0x0f, 0xd6, 0x73, 0xfb, 0x1b, 0x9a, 0x1b, 0x50, 0x66, 0xd1, 0x73, 0xaf, + 0x93, 0x34, 0x9c, 0x96, 0xbb, 0x55, 0x64, 0x46, 0x52, 0xd1, 0x55, 0x29, 0xc6, 0xa5, 0x82, 0x5a, + 0xca, 0x00, 0x7a, 0x11, 0x4a, 0x2a, 0xd4, 0x78, 0xca, 0xcc, 0x16, 0x3f, 0xe9, 0xdf, 0x0e, 0x54, + 0x76, 0xf9, 0x91, 0xa2, 0x91, 0x90, 0xdb, 0xb0, 0xda, 0x93, 0x3c, 0x1c, 0xf0, 0x78, 0xa0, 0x94, + 0xaa, 0x3b, 0x6f, 0x64, 0x21, 0x4c, 0xd5, 0xb6, 0xad, 0x4e, 0x37, 0x94, 0xf1, 0x84, 0xa5, 0x26, + 0xe4, 0x16, 0xac, 0x98, 0x9a, 0x50, 0x1c, 0xaa, 0x3b, 0xad, 0x45, 0xd6, 0x69, 0xd9, 0xa0, 0xb1, + 0x35, 0xb8, 0xf4, 0x01, 0xd4, 0xa6, 0xdc, 0x22, 0xd7, 0x03, 0x31, 0xb1, 0x19, 0x39, 0x10, 0x13, + 0x8c, 0xdd, 0x21, 0x0f, 0xc6, 0x3a, 0xce, 0x45, 0xa6, 0x85, 0x5b, 0x85, 0xf7, 0x9d, 0x4b, 0xb7, + 0xe0, 0x4c, 0xde, 0xeb, 0xeb, 0xd8, 0xd2, 0xaf, 0x80, 0xb4, 0x63, 0xc1, 0xa5, 0x50, 0xf4, 0x76, + 0x45, 0x92, 0xf0, 0x7d, 0xb1, 0x3c, 0xd3, 0x3a, 0x7b, 0x85, 0x7c, 0xf6, 0x36, 0xa1, 0xe2, 0x25, + 0xf6, 0xe0, 0xae, 0xaa, 0xcb, 0x0c, 0xa0, 0xd7, 0x80, 0x74, 0x44, 0x20, 0xa4, 0x30, 0x3d, 0x7a, + 0x8c, 0x7f, 0xda, 0xb3, 0x5c, 0x4e, 0xd6, 0x25, 0x57, 0xa1, 0x88, 0xed, 0xa9, 0xa8, 0x54, 0x77, + 0xce, 0x65, 0x91, 0x4e, 0x67, 0x01, 0x53, 0x0a, 0xd4, 0xb7, 0x4e, 0x4d, 0x4b, 0x9f, 0x70, 0xc0, + 0x05, 0xa5, 0x6c, 0xb7, 0x72, 0x67, 0xb7, 0x4a, 0x87, 0x84, 0xd9, 0xea, 0x8e, 0x3d, 0xeb, 0x69, + 0xb7, 0xa2, 0xfb, 0x29, 0x59, 0xec, 0xd4, 0xd3, 0x90, 0x7d, 0x13, 0x4a, 0xca, 0xd6, 0xb0, 0x9d, + 0x9b, 0x01, 0x7a, 0x95, 0x3e, 0x4d, 0xa9, 0x9e, 0x76, 0xa3, 0xf3, 0xf9, 0x8d, 0x2a, 0xd6, 0xef, + 0x17, 0x46, 0x17, 0x7b, 0xfa, 0x21, 0xda, 0x68, 0x4f, 0xea, 0x7b, 0x79, 0xce, 0x66, 0x02, 0x89, + 0xbe, 0x71, 0x08, 0x24, 0x0d, 0xb7, 0xe5, 0xa2, 0x6f, 0x25, 0xd0, 0x1b, 0x50, 0xee, 0xf5, 0x9f, + 0x89, 0x21, 0x27, 0x6f, 0x63, 0xa7, 0x0d, 0xc4, 0x91, 0x48, 0x4c, 0x9f, 0x9e, 0x9d, 0xc9, 0x3f, + 0xb3, 0xeb, 0xf4, 0x07, 0xc7, 0x9c, 0x69, 0x09, 0xa3, 0xb2, 0xda, 0x3b, 0x69, 0x14, 0xe7, 0x46, + 0x26, 0xe2, 0xcc, 0x2c, 0x93, 0x2e, 0xd4, 0xbd, 0x70, 0x34, 0x96, 0x1d, 0xf1, 0xb5, 0x1f, 0xfa, + 0xd2, 0x8f, 0xc2, 0xa4, 0x51, 0x56, 0x26, 0x17, 0xf3, 0x5b, 0x4f, 0x69, 0xb0, 0x39, 0x13, 0xfa, + 0x9d, 0x03, 0x67, 0x67, 0xc0, 0x13, 0x78, 0x15, 0x8e, 0xe7, 0xf5, 0x5e, 0x3a, 0xf3, 0x5d, 0xa5, + 0xd8, 0x5c, 0xca, 0x66, 0xfa, 0x0a, 0xf8, 0xc5, 0x81, 0xf3, 0x8b, 0x14, 0x16, 0xb2, 0x69, 0x02, + 0x3c, 0x8a, 0xfd, 0x21, 0x8f, 0x27, 0x9f, 0x88, 0x89, 0xb9, 0xfe, 0x72, 0x08, 0xf9, 0x1c, 0x36, + 0x66, 0x7c, 0x7d, 0xd4, 0xd7, 0x21, 0xd2, 0xa4, 0x2e, 0x2f, 0x25, 0xa5, 0xf5, 0xd8, 0x12, 0x73, + 0xfa, 0x97, 0x03, 0x17, 0x16, 0x2e, 0x65, 0x35, 0xe9, 0xe4, 0x6b, 0xf2, 0x1a, 0xd4, 0x9f, 0xe2, + 0x64, 0xeb, 0x88, 0x44, 0xfa, 0x21, 0x47, 0x4d, 0x53, 0xb4, 0x73, 0x38, 0xf1, 0x60, 0x55, 0x61, + 0xbb, 0x7c, 0x64, 0x68, 0xbe, 0x73, 0x02, 0xcd, 0x6d, 0xab, 0x6f, 0x06, 0xbf, 0x15, 0x91, 0x8c, + 0xba, 0x88, 0xec, 0xad, 0xa6, 0x04, 0x1c, 0xe9, 0x53, 0x06, 0xaf, 0x35, 0x96, 0x23, 0xd8, 0xb4, + 0xa3, 0x70, 0x8a, 0xc9, 0xf1, 0x9d, 0x7a, 0x13, 0x20, 0x53, 0x35, 0x13, 0xe0, 0x98, 0xfa, 0xcc, + 0x29, 0xd3, 0xfb, 0xb0, 0x69, 0xe7, 0xf4, 0x6b, 0x6c, 0x68, 0xab, 0xa5, 0x90, 0x55, 0x0b, 0xed, + 0x82, 0xfb, 0x84, 0x79, 0x78, 0x57, 0xab, 0x6e, 0xb5, 0x29, 0x32, 0x12, 0x9a, 0xdc, 0x8f, 0x12, + 0x69, 0x4d, 0xf0, 0x1b, 0xb1, 0x47, 0x51, 0x2c, 0x15, 0xe3, 0x1a, 0x53, 0xdf, 0xf4, 0x4b, 0x28, + 0x3e, 0x8c, 0x06, 0x82, 0xac, 0x41, 0xc1, 0xeb, 0x18, 0x1f, 0x05, 0xaf, 0x43, 0x2e, 0x2b, 0xf7, + 0x66, 0x86, 0xd4, 0xb2, 0xc3, 0x3d, 0x61, 0x1e, 0x53, 0x1b, 0x5f, 0x81, 0x9a, 0x97, 0xb4, 0xa3, + 0x28, 0x1e, 0x60, 0xaa, 0xa3, 0xd8, 0xdc, 0x49, 0xd3, 0x20, 0xbd, 0x03, 0x75, 0x74, 0xdf, 0x93, + 0x5c, 0xa6, 0x93, 0x7a, 0x03, 0xca, 0x88, 0xa5, 0xdb, 0x19, 0x49, 0xdd, 0x7b, 0xa8, 0x67, 0x07, + 0xa0, 0x12, 0xe8, 0x03, 0xed, 0xa1, 0x7b, 0x28, 0x42, 0x99, 0x8b, 0x92, 0x92, 0x95, 0x83, 0x1a, + 0xd3, 0x02, 0xa1, 0xfa, 0x28, 0x86, 0xf3, 0x5a, 0xc6, 0x19, 0x51, 0xa6, 0xd6, 0xe8, 0x8f, 0x0e, + 0x80, 0x25, 0x34, 0x4e, 0x52, 0x13, 0x67, 0xb9, 0x09, 0x79, 0x37, 0xf7, 0x76, 0x99, 0x9f, 0xa9, + 0xe9, 0x12, 0xcb, 0xbd, 0x70, 0xb6, 0xec, 0x08, 0x35, 0xc5, 0x51, 0xcf, 0xf4, 0x35, 0x6e, 0xd2, + 0x84, 0xd7, 0x66, 0xad, 0x1d, 0x8c, 0x13, 0x29, 0x62, 0xc3, 0x08, 0xdf, 0x58, 0x1a, 0x48, 0xe3, + 0x93, 0x01, 0x8b, 0x43, 0x44, 0xae, 0x40, 0x09, 0x99, 0xda, 0x39, 0x30, 0x7b, 0x0c, 0xbd, 0x48, + 0x7b, 0xe6, 0x26, 0x59, 0x38, 0x7b, 0x08, 0x14, 0xd5, 0x8b, 0xda, 0x94, 0x8b, 0x7a, 0x4c, 0xd7, + 0xc1, 0xdd, 0xf5, 0x75, 0x7d, 0xbb, 0x0c, 0x3f, 0x15, 0xc2, 0x8f, 0x54, 0xff, 0x21, 0xc2, 0xf1, + 0x2d, 0xb1, 0xae, 0x1b, 0x08, 0xef, 0x8e, 0xd3, 0xdc, 0x6f, 0xf6, 0x51, 0xea, 0xe6, 0x1e, 0xa5, + 0x3d, 0x58, 0xd7, 0x4d, 0xf2, 0x5f, 0x3a, 0xfd, 0xb9, 0x00, 0xeb, 0x4c, 0x24, 0xfe, 0x0b, 0xe1, + 0x85, 0x89, 0x8c, 0xc7, 0xe9, 0x80, 0xfb, 0x38, 0xda, 0x33, 0xa1, 0x76, 0x99, 0x16, 0x5e, 0xa5, + 0x92, 0xc8, 0x75, 0xfc, 0x05, 0x9a, 0xae, 0xfe, 0x79, 0xd5, 0xbc, 0x0a, 0xb9, 0x0e, 0x2b, 0xbd, + 0x68, 0x1c, 0xf7, 0xd3, 0x6b, 0x70, 0x23, 0xd3, 0xd6, 0xcc, 0xf4, 0x32, 0xb3, 0x6a, 0xb9, 0x3a, + 0x2a, 0x1d, 0x5f, 0x47, 0xe4, 0xf6, 0x4c, 0x1d, 0xa9, 0x3f, 0x97, 0xea, 0xce, 0xff, 0x33, 0x83, + 0xa9, 0x65, 0x36, 0xad, 0x4d, 0xbf, 0x77, 0xe0, 0x4c, 0x9e, 0xc2, 0x2b, 0x35, 0x46, 0x9a, 0x91, + 0xc2, 0xc2, 0x8c, 0xb8, 0x8b, 0x32, 0x52, 0xcc, 0x32, 0x92, 0xbd, 0x73, 0x4b, 0xb9, 0x77, 0x2e, + 0x3d, 0x80, 0x8b, 0x73, 0x69, 0x6a, 0x47, 0xc3, 0x11, 0xd6, 0xc3, 0xbf, 0x48, 0x17, 0x8e, 0x8c, + 0x38, 0x36, 0x89, 0xaa, 0x30, 0x2d, 0xd0, 0x9b, 0x70, 0xa1, 0x27, 0x64, 0x2e, 0x49, 0xb6, 0xda, + 0x5a, 0xe0, 0x3e, 0x14, 0xcf, 0x97, 0x1c, 0x1f, 0x97, 0xe8, 0x87, 0xd0, 0x78, 0x32, 0x1a, 0x70, + 0x29, 0x4e, 0x65, 0x7d, 0x17, 0x56, 0x1f, 0x47, 0xa3, 0x28, 0x88, 0xf6, 0x27, 0x27, 0xb4, 0x7c, + 0x03, 0x56, 0xf4, 0x7c, 0xd4, 0x8f, 0x94, 0x0a, 0xb3, 0x22, 0x3d, 0x87, 0x05, 0xdd, 0xe7, 0x41, + 0x7f, 0x1c, 0x20, 0x0d, 0xfc, 0xf7, 0x4a, 0xee, 0xd6, 0x7f, 0x7d, 0xd9, 0x74, 0x7e, 0x7b, 0xd9, + 0x74, 0xfe, 0x78, 0xd9, 0x74, 0x7e, 0xfa, 0xb3, 0xf9, 0xbf, 0xbd, 0xb2, 0xfa, 0x93, 0xbf, 0xf1, + 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x13, 0x1d, 0x50, 0xda, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 8642d9a13..ca987ec51 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -8,7 +8,6 @@ message IndexMeta { } message FrameMeta { - string RowLabel = 1; bool InverseEnabled = 2; string CacheType = 3; uint32 CacheSize = 4; diff --git a/pilosa.go b/pilosa.go index 98c67203f..71b63fcaa 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,7 +36,6 @@ var ( ErrFrameExists = errors.New("frame already exists") ErrFrameNotFound = errors.New("frame not found") ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") diff --git a/server/cluster_test.go b/server/cluster_test.go index fbb784691..de623bbd4 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -153,8 +153,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", columnID=1) + SetBit(row=1, frame="f", columnID=2400000) `); err != nil { t.Fatal(err) } @@ -351,8 +351,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=1300000) + SetBit(row=1, frame="f", columnID=1) + SetBit(row=1, frame="f", columnID=1300000) `); err != nil { t.Fatal(err) } @@ -406,8 +406,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", columnID=1) + SetBit(row=1, frame="f", columnID=2400000) `); err != nil { t.Fatal(err) } @@ -560,7 +560,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setBits := "" for i := 0; i < 20; i++ { - setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) + setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setBits); err != nil { diff --git a/server/server_test.go b/server/server_test.go index 3852845a6..28155bab4 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -56,7 +56,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(rowID=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -72,7 +72,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -95,7 +95,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -131,36 +131,36 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set bits on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="x", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", columnID=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(rowID=1, frame="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -171,19 +171,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -204,9 +204,9 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=101)`); err != nil { t.Fatal(err) } @@ -216,7 +216,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -227,7 +227,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -248,9 +248,9 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { t.Fatal(err) } @@ -260,7 +260,7 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -285,19 +285,19 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m10.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=100) - SetBit(rowID=1, frame="f", columnID=1000) - SetBit(rowID=1, frame="f", columnID=100000) - SetBit(rowID=1, frame="f", columnID=200000) - SetBit(rowID=1, frame="f", columnID=400000) - SetBit(rowID=1, frame="f", columnID=600000) - SetBit(rowID=1, frame="f", columnID=800000) + SetBit(row=1, frame="f", columnID=100) + SetBit(row=1, frame="f", columnID=1000) + SetBit(row=1, frame="f", columnID=100000) + SetBit(row=1, frame="f", columnID=200000) + SetBit(row=1, frame="f", columnID=400000) + SetBit(row=1, frame="f", columnID=600000) + SetBit(row=1, frame="f", columnID=800000) `); err != nil { t.Fatal("setting bits:", err) } // Query row on first cluster. - if res, err := m10.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -335,7 +335,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Query row on second cluster. - if res, err := m20.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("another bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("2unexpected result: %s", res) @@ -401,7 +401,7 @@ func TestMain_RecalculateHashes(t *testing.T) { data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(rowID=%d, frame="f", columnID=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", columnID=%d)`, rowID, columnID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { diff --git a/stats_test.go b/stats_test.go index 93d7d9bb1..a03407a3d 100644 --- a/stats_test.go +++ b/stats_test.go @@ -143,7 +143,7 @@ func TestStatsCount_Bitmap(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -174,7 +174,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { From b2b0fd081afd96b5c4fdd9e23d7f7a20bf4f74ec Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 26 Mar 2018 15:35:56 -0500 Subject: [PATCH 03/21] Remove ColumnLabel support --- docs/examples.md | 11 +- executor.go | 70 ++++--------- executor_test.go | 120 +++++++++++----------- fragment.go | 4 +- frame.go | 2 - handler_internal_test.go | 3 +- handler_test.go | 4 +- holder.go | 1 - holder_test.go | 2 +- index.go | 46 +-------- input_definition.go | 2 +- internal/private.pb.go | 214 +++++++++++++++------------------------ internal/private.proto | 1 - pilosa.go | 1 - server.go | 1 - server/cluster_test.go | 16 +-- server/server_test.go | 66 +++--------- stats_test.go | 2 +- 18 files changed, 201 insertions(+), 365 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index cf9885a3f..225e0f07c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -267,12 +267,11 @@ First, follow the instruction in the [getting started](../getting-started/) guid The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 ``` curl localhost:10101/index/mole \ - -X POST \ - -d '{"options": {"columnLabel": "position_id"}}' + -X POST curl localhost:10101/index/mole/frame/fingerprint \ -X POST \ - -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' + -d '{"options": {"inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' ``` @@ -302,7 +301,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a * Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id ```python - bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] + bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] bitmap_string = ', '.join(bit_maps) intersection = "Intersect(%s)" % bitmap_string mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] @@ -312,7 +311,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a ```python for m in mole_ids: - mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] + mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(row=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] existed_mol = False if len(mol) == len(fp): found = m @@ -331,7 +330,7 @@ Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN * Query Pilosa’s TopN to get list of similarity chembl_id ```python - query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' + query_string = 'TopN(Bitmap(row=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) ``` diff --git a/executor.go b/executor.go index de0d65df0..0ed6dbaed 100644 --- a/executor.go +++ b/executor.go @@ -80,7 +80,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // MaxSlice can differ between inverse and standard views, so we need // to send queries to different slices based on orientation. var inverseSlices []uint64 - columnLabel := DefaultColumnLabel // If slices aren't specified, then include all of them. if len(slices) == 0 { @@ -105,9 +104,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic for i := range inverseSlices { inverseSlices[i] = uint64(i) } - - // Fetch column label from index. - columnLabel = idx.ColumnLabel() } } @@ -132,7 +128,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(DefaultRowLabel, columnLabel) { + if call.IsInverse(DefaultRowLabel, DefaultColumnLabel) { slices = inverseSlices } } @@ -266,8 +262,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { idx := e.Holder.Index(index) if idx != nil { - columnLabel := idx.ColumnLabel() - if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { + if columnID, ok, err := c.UintArg(DefaultColumnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err @@ -522,7 +517,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if idx == nil { return nil, ErrIndexNotFound } - columnLabel := idx.ColumnLabel() // Fetch frame & row label based on argument. frame, _ := c.Args["frame"].(string) @@ -536,14 +530,14 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. // Return an error if both the row and column label are specified. rowID, rowOK, rowErr := c.UintArg(DefaultRowLabel) - columnID, columnOK, columnErr := c.UintArg(columnLabel) + columnID, columnOK, columnErr := c.UintArg(DefaultColumnLabel) if rowErr != nil || columnErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) } if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, DefaultColumnLabel) } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, columnLabel) + return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, DefaultColumnLabel) } // Determine row or column orientation. @@ -602,7 +596,6 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if idx == nil { return nil, ErrIndexNotFound } - columnLabel := idx.ColumnLabel() // Retrieve base frame. f := idx.Frame(frame) @@ -611,7 +604,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Read row & column id. - columnID, columnOK, err := c.UintArg(columnLabel) + columnID, columnOK, err := c.UintArg(DefaultColumnLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) } @@ -624,9 +617,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C var id uint64 var viewName string if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() cannot contain both %q and %q", DefaultColumnLabel, DefaultRowLabel) } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() must specify either %q or %q", DefaultColumnLabel, DefaultRowLabel) } else if columnOK { viewName, id = ViewInverse, columnID } else { @@ -899,9 +892,6 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, ErrFrameNotFound } - // Retrieve labels. - columnLabel := idx.ColumnLabel() - // Read fields using labels. rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { @@ -910,11 +900,11 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("ClearBit() row field '%v' required", DefaultRowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg(DefaultColumnLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) + return false, fmt.Errorf("ClearBit col field '%v' required", DefaultColumnLabel) } // Clear bits for each view. @@ -992,9 +982,6 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, ErrFrameNotFound } - // Retrieve labels. - columnLabel := idx.ColumnLabel() - // Read fields using labels. rowID, ok, err := c.UintArg(DefaultRowLabel) if err != nil { @@ -1003,11 +990,11 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, fmt.Errorf("SetBit() row field '%v' required", DefaultRowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg(DefaultColumnLabel) if err != nil { return false, fmt.Errorf("reading SetBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) + return false, fmt.Errorf("SetBit() column field '%v' required", DefaultColumnLabel) } var timestamp *time.Time @@ -1086,13 +1073,6 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq return errors.New("SetFieldValue() frame required") } - // Retrieve column label. - idx := e.Holder.Index(index) - if idx == nil { - return ErrIndexNotFound - } - columnLabel := idx.ColumnLabel() - // Retrieve frame. frame := e.Holder.Frame(index, frameName) if frame == nil { @@ -1100,11 +1080,11 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Parse labels. - columnID, ok, err := c.UintArg(columnLabel) + columnID, ok, err := c.UintArg(DefaultColumnLabel) if err != nil { return fmt.Errorf("reading SetFieldValue() column: %v", err) } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) + return fmt.Errorf("SetFieldValue() column field '%v' required", DefaultColumnLabel) } // Copy args and remove reserved fields. @@ -1112,7 +1092,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq delete(args, "frame") // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. - delete(args, columnLabel) + delete(args, DefaultColumnLabel) // Set values. for name, value := range args { @@ -1304,28 +1284,18 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return ErrIndexNotFound } - var colName string - id, okID, errID := c.UintArg("id") - if errID != nil || !okID { - // Retrieve columnLabel - columnLabel := idx.columnLabel - col, okCol, errCol := c.UintArg(columnLabel) - if errCol != nil || !okCol { - return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) - } - id = col - colName = columnLabel - } else { - colName = "id" + col, okCol, errCol := c.UintArg(DefaultColumnLabel) + if errCol != nil || !okCol { + return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, colName) + delete(attrs, DefaultColumnLabel) delete(attrs, "frame") // Set attributes. - if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil { + if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil { return err } idx.Stats.Count("SetProfileAttrs", 1, 1.0) diff --git a/executor_test.go b/executor_test.go index d90c8048f..1a23eba14 100644 --- a/executor_test.go +++ b/executor_test.go @@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -293,9 +293,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Set field values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=100, frame=f, field0=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil { t.Fatal(err) } @@ -340,28 +340,28 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Run("ErrFrameRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { t.Fatalf("unexpected error: %s", err) } }) @@ -419,15 +419,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, columnID=0) - SetBit(frame=f, row=0, columnID=1) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, row=10, columnID=0) - SetBit(frame=f, row=10, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=20, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, row=0, columnID=0) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=1) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, row=10, col=0) + SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, row=0, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -630,15 +630,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, columnID=0) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) - SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, columnID=0) + SetFieldValue(frame=f, foo=20, bar=2000, col=0) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -679,16 +679,16 @@ func TestExecutor_Execute_Range(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, row=1, columnID=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, row=1, columnID=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, row=1, columnID=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, row=1, columnID=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, row=1, columnID=7, timestamp="2002-01-01T02:00") + SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") + SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") + SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00") + SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00") + SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, row=1, columnID=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, row=1, columnID=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, row=10, columnID=2, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00") + SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00") + SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } @@ -703,7 +703,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Run("Inverse", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { t.Fatalf("unexpected bits: %+v", bits) @@ -751,17 +751,17 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, columnID=0) - SetBit(frame=f, row=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) - SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, columnID=0) - SetFieldValue(frame=edge, foo=100, columnID=0) - SetFieldValue(frame=edge, foo=-100, columnID=1) + SetFieldValue(frame=f, foo=20, bar=2000, col=50) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, col=0) + SetFieldValue(frame=edge, foo=100, col=0) + SetFieldValue(frame=edge, foo=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } @@ -994,7 +994,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", row=10)` { + } else if query.String() != `SetBit(col=2, frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1012,7 +1012,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -1046,7 +1046,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1066,7 +1066,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -1169,11 +1169,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=10)"), nil, nil) + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', columnID=10, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } @@ -1186,11 +1186,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, columnID=20)"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(columnID=20, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index 8b32182d6..093b14d34 100644 --- a/fragment.go +++ b/fragment.go @@ -1841,11 +1841,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/frame.go b/frame.go index 060125385..f786ac0b0 100644 --- a/frame.go +++ b/frame.go @@ -57,7 +57,6 @@ type Frame struct { Stats StatsClient // Frame options. - rowLabel string inverseEnabled bool cacheType string cacheSize uint32 @@ -267,7 +266,6 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.rowLabel = DefaultRowLabel f.inverseEnabled = DefaultInverseEnabled f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize diff --git a/handler_internal_test.go b/handler_internal_test.go index fb6226ea9..395ec73b6 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -30,8 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}}, - {json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"}, + {json: `{"options": {"columnLabel": "test"}}`, err: "Unknown key: columnLabel:test"}, } for _, test := range tests { actual := &postIndexRequest{} diff --git a/handler_test.go b/handler_test.go index 3c9c08094..5e9b993f9 100644 --- a/handler_test.go +++ b/handler_test.go @@ -107,7 +107,7 @@ func TestHandler_Schema(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -1349,7 +1349,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { } // Ensure throwing error if there's no primary key - hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) unmatchColumnBody := []byte(` { "frames":[{ diff --git a/holder.go b/holder.go index 63bbbbc86..f418a145d 100644 --- a/holder.go +++ b/holder.go @@ -361,7 +361,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } // Update options. - index.SetColumnLabel(opt.ColumnLabel) index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index diff --git a/holder_test.go b/holder_test.go index 5a06a9272..267a10372 100644 --- a/holder_test.go +++ b/holder_test.go @@ -68,7 +68,7 @@ func TestHolder_Open(t *testing.T) { h := test.MustOpenHolder() defer h.Close() - if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) diff --git a/index.go b/index.go index 968bac847..f17854e1e 100644 --- a/index.go +++ b/index.go @@ -31,7 +31,7 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "columnID" + DefaultColumnLabel = "col" InputDefinitionDir = ".input-definitions" ) @@ -45,9 +45,6 @@ type Index struct { // This can be overridden by individual frames. timeQuantum TimeQuantum - // Label used for referring to columns in index. - columnLabel string - // Frames by name. frames map[string]*Frame @@ -88,8 +85,6 @@ func NewIndex(path, name string) (*Index, error) { NewAttrStore: NewNopAttrStore, columnAttrStore: NopAttrStore, - columnLabel: DefaultColumnLabel, - broadcaster: NopBroadcaster, Stats: NopStatsClient, LogOutput: ioutil.Discard, @@ -105,39 +100,6 @@ func (i *Index) Path() string { return i.path } // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } -// SetColumnLabel sets the column label. Persists to meta file on update. -func (i *Index) SetColumnLabel(v string) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || i.columnLabel == v { - return nil - } - - // Make sure columnLabel is valid name - err := ValidateLabel(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - i.columnLabel = v - if err := i.saveMeta(); err != nil { - return err - } - - return nil -} - -// ColumnLabel returns the column label. -func (i *Index) ColumnLabel() string { - i.mu.RLock() - v := i.columnLabel - i.mu.RUnlock() - return v -} - // Options returns all options for this index. func (i *Index) Options() IndexOptions { i.mu.RLock() @@ -147,7 +109,6 @@ func (i *Index) Options() IndexOptions { func (i *Index) options() IndexOptions { return IndexOptions{ - ColumnLabel: i.columnLabel, TimeQuantum: i.timeQuantum, } } @@ -217,7 +178,6 @@ func (i *Index) loadMeta() error { buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta")) if os.IsNotExist(err) { i.timeQuantum = "" - i.columnLabel = DefaultColumnLabel return nil } else if err != nil { return err @@ -229,7 +189,6 @@ func (i *Index) loadMeta() error { // Copy metadata fields. i.timeQuantum = TimeQuantum(pb.TimeQuantum) - i.columnLabel = pb.ColumnLabel return nil } @@ -239,7 +198,6 @@ func (i *Index) saveMeta() error { // Marshal metadata. buf, err := proto.Marshal(&internal.IndexMeta{ TimeQuantum: string(i.timeQuantum), - ColumnLabel: i.columnLabel, }) if err != nil { return err @@ -631,14 +589,12 @@ func encodeIndex(d *Index) *internal.Index { // IndexOptions represents options to set when initializing an index. type IndexOptions struct { - ColumnLabel string `json:"columnLabel,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { return &internal.IndexMeta{ - ColumnLabel: i.ColumnLabel, TimeQuantum: string(i.TimeQuantum), } } diff --git a/input_definition.go b/input_definition.go index f4571d0bb..b18b4980e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -281,7 +281,7 @@ func (i *InputDefinitionInfo) Validate() error { } } - // Validate columnLabel and duplicate primaryKey. + // Validate duplicate primaryKey. for _, field := range i.Fields { if field.Name == "" { return ErrInputDefinitionNameRequired diff --git a/internal/private.pb.go b/internal/private.pb.go index 488aca0fe..0d0f3124a 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -68,7 +68,6 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } @@ -77,13 +76,6 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -func (m *IndexMeta) GetColumnLabel() string { - if m != nil { - return m.ColumnLabel - } - return "" -} - func (m *IndexMeta) GetTimeQuantum() string { if m != nil { return m.TimeQuantum @@ -1240,12 +1232,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.ColumnLabel) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ColumnLabel))) - i += copy(dAtA[i:], m.ColumnLabel) - } if len(m.TimeQuantum) > 0 { dAtA[i] = 0x12 i++ @@ -2760,10 +2746,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l - l = len(m.ColumnLabel) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } l = len(m.TimeQuantum) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -3457,35 +3439,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ColumnLabel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) @@ -8620,88 +8573,87 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1326 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xd7, 0x6b, 0x3b, 0xf1, 0x71, 0x9d, 0x3a, 0xd3, 0x36, 0xaf, 0x5b, 0x45, 0xae, 0x19, - 0x15, 0x1a, 0x2a, 0x11, 0x95, 0x54, 0x42, 0xb4, 0x50, 0xa9, 0xd4, 0x76, 0xd5, 0x85, 0xa6, 0x94, - 0x71, 0x5b, 0x24, 0x24, 0x90, 0x26, 0xf6, 0x90, 0xae, 0xb2, 0xde, 0x35, 0xbb, 0xe3, 0x34, 0xee, - 0x05, 0x97, 0x08, 0x09, 0x71, 0x8f, 0xb8, 0xe5, 0xcf, 0x70, 0xc9, 0x2f, 0x40, 0xa8, 0xfc, 0x08, - 0x2e, 0x41, 0x67, 0x3e, 0x76, 0xd7, 0x5f, 0x49, 0x1b, 0xb8, 0xdb, 0xf3, 0xcc, 0x39, 0x67, 0x9e, - 0x39, 0x5f, 0x33, 0x0b, 0xb5, 0x51, 0xec, 0x1f, 0x72, 0x29, 0xb6, 0x47, 0x71, 0x24, 0x23, 0xb2, - 0xea, 0x87, 0x52, 0xc4, 0x21, 0x0f, 0xe8, 0xa7, 0x50, 0xf1, 0xc2, 0x81, 0x38, 0xda, 0x15, 0x92, - 0x93, 0x16, 0x54, 0xdb, 0x51, 0x30, 0x1e, 0x86, 0x0f, 0xf8, 0x9e, 0x08, 0x1a, 0x4e, 0xcb, 0xd9, - 0xaa, 0xb0, 0x3c, 0x84, 0x1a, 0x8f, 0xfd, 0xa1, 0xf8, 0x6c, 0xcc, 0x43, 0x39, 0x1e, 0x36, 0x0a, - 0x5a, 0x23, 0x07, 0xd1, 0xdf, 0x1d, 0xa8, 0xdc, 0x8b, 0xf9, 0x50, 0x28, 0x8f, 0x6f, 0xc1, 0x9a, - 0x17, 0x1e, 0x8a, 0x38, 0x11, 0xdd, 0x90, 0xef, 0x05, 0x62, 0xa0, 0x4c, 0x56, 0xd9, 0x0c, 0x4a, - 0x36, 0xa1, 0xd2, 0xe6, 0xfd, 0x67, 0xe2, 0xf1, 0x64, 0x24, 0x1a, 0xae, 0xf2, 0x9a, 0x01, 0xe9, - 0x6a, 0xcf, 0x7f, 0x21, 0x1a, 0xc5, 0x96, 0xb3, 0x55, 0x63, 0x19, 0x30, 0xcb, 0xa9, 0x34, 0xc7, - 0x89, 0x50, 0x38, 0xc3, 0x78, 0xb8, 0x9f, 0x72, 0x28, 0x2b, 0x0e, 0x53, 0x18, 0xb9, 0x0a, 0xe5, - 0x7b, 0xbe, 0x08, 0x06, 0x49, 0x63, 0xa5, 0xe5, 0x6e, 0x55, 0x77, 0xce, 0x6e, 0xdb, 0x18, 0x6d, - 0x2b, 0x9c, 0x99, 0x65, 0x4a, 0x61, 0xcd, 0x1b, 0x8e, 0xa2, 0x58, 0x32, 0x91, 0x8c, 0xa2, 0x30, - 0x11, 0xa4, 0x0e, 0x6e, 0x37, 0x8e, 0x4d, 0xb8, 0xf0, 0x93, 0x7e, 0x0b, 0xf5, 0xbb, 0x41, 0xd4, - 0x3f, 0xe8, 0x70, 0xc9, 0x99, 0xf8, 0x66, 0x2c, 0x12, 0x49, 0xce, 0x43, 0x49, 0x45, 0xda, 0xe8, - 0x69, 0x01, 0x51, 0x15, 0x2d, 0x13, 0x4a, 0x2d, 0x20, 0xaa, 0xec, 0x55, 0x28, 0x8a, 0x4c, 0x0b, - 0x88, 0xf6, 0x02, 0xbf, 0xaf, 0x43, 0x50, 0x64, 0x5a, 0x20, 0x04, 0x8a, 0x4f, 0x7d, 0xf1, 0xdc, - 0x9c, 0x5b, 0x7d, 0x53, 0x0f, 0xd6, 0x73, 0xfb, 0x1b, 0x9a, 0x1b, 0x50, 0x66, 0xd1, 0x73, 0xaf, - 0x93, 0x34, 0x9c, 0x96, 0xbb, 0x55, 0x64, 0x46, 0x52, 0xd1, 0x55, 0x29, 0xc6, 0xa5, 0x82, 0x5a, - 0xca, 0x00, 0x7a, 0x11, 0x4a, 0x2a, 0xd4, 0x78, 0xca, 0xcc, 0x16, 0x3f, 0xe9, 0xdf, 0x0e, 0x54, - 0x76, 0xf9, 0x91, 0xa2, 0x91, 0x90, 0xdb, 0xb0, 0xda, 0x93, 0x3c, 0x1c, 0xf0, 0x78, 0xa0, 0x94, - 0xaa, 0x3b, 0x6f, 0x64, 0x21, 0x4c, 0xd5, 0xb6, 0xad, 0x4e, 0x37, 0x94, 0xf1, 0x84, 0xa5, 0x26, - 0xe4, 0x16, 0xac, 0x98, 0x9a, 0x50, 0x1c, 0xaa, 0x3b, 0xad, 0x45, 0xd6, 0x69, 0xd9, 0xa0, 0xb1, - 0x35, 0xb8, 0xf4, 0x01, 0xd4, 0xa6, 0xdc, 0x22, 0xd7, 0x03, 0x31, 0xb1, 0x19, 0x39, 0x10, 0x13, - 0x8c, 0xdd, 0x21, 0x0f, 0xc6, 0x3a, 0xce, 0x45, 0xa6, 0x85, 0x5b, 0x85, 0xf7, 0x9d, 0x4b, 0xb7, - 0xe0, 0x4c, 0xde, 0xeb, 0xeb, 0xd8, 0xd2, 0xaf, 0x80, 0xb4, 0x63, 0xc1, 0xa5, 0x50, 0xf4, 0x76, - 0x45, 0x92, 0xf0, 0x7d, 0xb1, 0x3c, 0xd3, 0x3a, 0x7b, 0x85, 0x7c, 0xf6, 0x36, 0xa1, 0xe2, 0x25, - 0xf6, 0xe0, 0xae, 0xaa, 0xcb, 0x0c, 0xa0, 0xd7, 0x80, 0x74, 0x44, 0x20, 0xa4, 0x30, 0x3d, 0x7a, - 0x8c, 0x7f, 0xda, 0xb3, 0x5c, 0x4e, 0xd6, 0x25, 0x57, 0xa1, 0x88, 0xed, 0xa9, 0xa8, 0x54, 0x77, - 0xce, 0x65, 0x91, 0x4e, 0x67, 0x01, 0x53, 0x0a, 0xd4, 0xb7, 0x4e, 0x4d, 0x4b, 0x9f, 0x70, 0xc0, - 0x05, 0xa5, 0x6c, 0xb7, 0x72, 0x67, 0xb7, 0x4a, 0x87, 0x84, 0xd9, 0xea, 0x8e, 0x3d, 0xeb, 0x69, - 0xb7, 0xa2, 0xfb, 0x29, 0x59, 0xec, 0xd4, 0xd3, 0x90, 0x7d, 0x13, 0x4a, 0xca, 0xd6, 0xb0, 0x9d, - 0x9b, 0x01, 0x7a, 0x95, 0x3e, 0x4d, 0xa9, 0x9e, 0x76, 0xa3, 0xf3, 0xf9, 0x8d, 0x2a, 0xd6, 0xef, - 0x17, 0x46, 0x17, 0x7b, 0xfa, 0x21, 0xda, 0x68, 0x4f, 0xea, 0x7b, 0x79, 0xce, 0x66, 0x02, 0x89, - 0xbe, 0x71, 0x08, 0x24, 0x0d, 0xb7, 0xe5, 0xa2, 0x6f, 0x25, 0xd0, 0x1b, 0x50, 0xee, 0xf5, 0x9f, - 0x89, 0x21, 0x27, 0x6f, 0x63, 0xa7, 0x0d, 0xc4, 0x91, 0x48, 0x4c, 0x9f, 0x9e, 0x9d, 0xc9, 0x3f, - 0xb3, 0xeb, 0xf4, 0x07, 0xc7, 0x9c, 0x69, 0x09, 0xa3, 0xb2, 0xda, 0x3b, 0x69, 0x14, 0xe7, 0x46, - 0x26, 0xe2, 0xcc, 0x2c, 0x93, 0x2e, 0xd4, 0xbd, 0x70, 0x34, 0x96, 0x1d, 0xf1, 0xb5, 0x1f, 0xfa, - 0xd2, 0x8f, 0xc2, 0xa4, 0x51, 0x56, 0x26, 0x17, 0xf3, 0x5b, 0x4f, 0x69, 0xb0, 0x39, 0x13, 0xfa, - 0x9d, 0x03, 0x67, 0x67, 0xc0, 0x13, 0x78, 0x15, 0x8e, 0xe7, 0xf5, 0x5e, 0x3a, 0xf3, 0x5d, 0xa5, - 0xd8, 0x5c, 0xca, 0x66, 0xfa, 0x0a, 0xf8, 0xc5, 0x81, 0xf3, 0x8b, 0x14, 0x16, 0xb2, 0x69, 0x02, - 0x3c, 0x8a, 0xfd, 0x21, 0x8f, 0x27, 0x9f, 0x88, 0x89, 0xb9, 0xfe, 0x72, 0x08, 0xf9, 0x1c, 0x36, - 0x66, 0x7c, 0x7d, 0xd4, 0xd7, 0x21, 0xd2, 0xa4, 0x2e, 0x2f, 0x25, 0xa5, 0xf5, 0xd8, 0x12, 0x73, - 0xfa, 0x97, 0x03, 0x17, 0x16, 0x2e, 0x65, 0x35, 0xe9, 0xe4, 0x6b, 0xf2, 0x1a, 0xd4, 0x9f, 0xe2, - 0x64, 0xeb, 0x88, 0x44, 0xfa, 0x21, 0x47, 0x4d, 0x53, 0xb4, 0x73, 0x38, 0xf1, 0x60, 0x55, 0x61, - 0xbb, 0x7c, 0x64, 0x68, 0xbe, 0x73, 0x02, 0xcd, 0x6d, 0xab, 0x6f, 0x06, 0xbf, 0x15, 0x91, 0x8c, - 0xba, 0x88, 0xec, 0xad, 0xa6, 0x04, 0x1c, 0xe9, 0x53, 0x06, 0xaf, 0x35, 0x96, 0x23, 0xd8, 0xb4, - 0xa3, 0x70, 0x8a, 0xc9, 0xf1, 0x9d, 0x7a, 0x13, 0x20, 0x53, 0x35, 0x13, 0xe0, 0x98, 0xfa, 0xcc, - 0x29, 0xd3, 0xfb, 0xb0, 0x69, 0xe7, 0xf4, 0x6b, 0x6c, 0x68, 0xab, 0xa5, 0x90, 0x55, 0x0b, 0xed, - 0x82, 0xfb, 0x84, 0x79, 0x78, 0x57, 0xab, 0x6e, 0xb5, 0x29, 0x32, 0x12, 0x9a, 0xdc, 0x8f, 0x12, - 0x69, 0x4d, 0xf0, 0x1b, 0xb1, 0x47, 0x51, 0x2c, 0x15, 0xe3, 0x1a, 0x53, 0xdf, 0xf4, 0x4b, 0x28, - 0x3e, 0x8c, 0x06, 0x82, 0xac, 0x41, 0xc1, 0xeb, 0x18, 0x1f, 0x05, 0xaf, 0x43, 0x2e, 0x2b, 0xf7, - 0x66, 0x86, 0xd4, 0xb2, 0xc3, 0x3d, 0x61, 0x1e, 0x53, 0x1b, 0x5f, 0x81, 0x9a, 0x97, 0xb4, 0xa3, - 0x28, 0x1e, 0x60, 0xaa, 0xa3, 0xd8, 0xdc, 0x49, 0xd3, 0x20, 0xbd, 0x03, 0x75, 0x74, 0xdf, 0x93, - 0x5c, 0xa6, 0x93, 0x7a, 0x03, 0xca, 0x88, 0xa5, 0xdb, 0x19, 0x49, 0xdd, 0x7b, 0xa8, 0x67, 0x07, - 0xa0, 0x12, 0xe8, 0x03, 0xed, 0xa1, 0x7b, 0x28, 0x42, 0x99, 0x8b, 0x92, 0x92, 0x95, 0x83, 0x1a, - 0xd3, 0x02, 0xa1, 0xfa, 0x28, 0x86, 0xf3, 0x5a, 0xc6, 0x19, 0x51, 0xa6, 0xd6, 0xe8, 0x8f, 0x0e, - 0x80, 0x25, 0x34, 0x4e, 0x52, 0x13, 0x67, 0xb9, 0x09, 0x79, 0x37, 0xf7, 0x76, 0x99, 0x9f, 0xa9, - 0xe9, 0x12, 0xcb, 0xbd, 0x70, 0xb6, 0xec, 0x08, 0x35, 0xc5, 0x51, 0xcf, 0xf4, 0x35, 0x6e, 0xd2, - 0x84, 0xd7, 0x66, 0xad, 0x1d, 0x8c, 0x13, 0x29, 0x62, 0xc3, 0x08, 0xdf, 0x58, 0x1a, 0x48, 0xe3, - 0x93, 0x01, 0x8b, 0x43, 0x44, 0xae, 0x40, 0x09, 0x99, 0xda, 0x39, 0x30, 0x7b, 0x0c, 0xbd, 0x48, - 0x7b, 0xe6, 0x26, 0x59, 0x38, 0x7b, 0x08, 0x14, 0xd5, 0x8b, 0xda, 0x94, 0x8b, 0x7a, 0x4c, 0xd7, - 0xc1, 0xdd, 0xf5, 0x75, 0x7d, 0xbb, 0x0c, 0x3f, 0x15, 0xc2, 0x8f, 0x54, 0xff, 0x21, 0xc2, 0xf1, - 0x2d, 0xb1, 0xae, 0x1b, 0x08, 0xef, 0x8e, 0xd3, 0xdc, 0x6f, 0xf6, 0x51, 0xea, 0xe6, 0x1e, 0xa5, - 0x3d, 0x58, 0xd7, 0x4d, 0xf2, 0x5f, 0x3a, 0xfd, 0xb9, 0x00, 0xeb, 0x4c, 0x24, 0xfe, 0x0b, 0xe1, - 0x85, 0x89, 0x8c, 0xc7, 0xe9, 0x80, 0xfb, 0x38, 0xda, 0x33, 0xa1, 0x76, 0x99, 0x16, 0x5e, 0xa5, - 0x92, 0xc8, 0x75, 0xfc, 0x05, 0x9a, 0xae, 0xfe, 0x79, 0xd5, 0xbc, 0x0a, 0xb9, 0x0e, 0x2b, 0xbd, - 0x68, 0x1c, 0xf7, 0xd3, 0x6b, 0x70, 0x23, 0xd3, 0xd6, 0xcc, 0xf4, 0x32, 0xb3, 0x6a, 0xb9, 0x3a, - 0x2a, 0x1d, 0x5f, 0x47, 0xe4, 0xf6, 0x4c, 0x1d, 0xa9, 0x3f, 0x97, 0xea, 0xce, 0xff, 0x33, 0x83, - 0xa9, 0x65, 0x36, 0xad, 0x4d, 0xbf, 0x77, 0xe0, 0x4c, 0x9e, 0xc2, 0x2b, 0x35, 0x46, 0x9a, 0x91, - 0xc2, 0xc2, 0x8c, 0xb8, 0x8b, 0x32, 0x52, 0xcc, 0x32, 0x92, 0xbd, 0x73, 0x4b, 0xb9, 0x77, 0x2e, - 0x3d, 0x80, 0x8b, 0x73, 0x69, 0x6a, 0x47, 0xc3, 0x11, 0xd6, 0xc3, 0xbf, 0x48, 0x17, 0x8e, 0x8c, - 0x38, 0x36, 0x89, 0xaa, 0x30, 0x2d, 0xd0, 0x9b, 0x70, 0xa1, 0x27, 0x64, 0x2e, 0x49, 0xb6, 0xda, - 0x5a, 0xe0, 0x3e, 0x14, 0xcf, 0x97, 0x1c, 0x1f, 0x97, 0xe8, 0x87, 0xd0, 0x78, 0x32, 0x1a, 0x70, - 0x29, 0x4e, 0x65, 0x7d, 0x17, 0x56, 0x1f, 0x47, 0xa3, 0x28, 0x88, 0xf6, 0x27, 0x27, 0xb4, 0x7c, - 0x03, 0x56, 0xf4, 0x7c, 0xd4, 0x8f, 0x94, 0x0a, 0xb3, 0x22, 0x3d, 0x87, 0x05, 0xdd, 0xe7, 0x41, - 0x7f, 0x1c, 0x20, 0x0d, 0xfc, 0xf7, 0x4a, 0xee, 0xd6, 0x7f, 0x7d, 0xd9, 0x74, 0x7e, 0x7b, 0xd9, - 0x74, 0xfe, 0x78, 0xd9, 0x74, 0x7e, 0xfa, 0xb3, 0xf9, 0xbf, 0xbd, 0xb2, 0xfa, 0x93, 0xbf, 0xf1, - 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x13, 0x1d, 0x50, 0xda, 0x0f, 0x00, 0x00, + // 1308 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15, + 0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3, + 0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03, + 0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08, + 0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66, + 0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a, + 0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5, + 0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e, + 0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b, + 0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65, + 0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff, + 0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1, + 0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22, + 0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66, + 0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d, + 0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd, + 0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13, + 0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8, + 0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73, + 0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2, + 0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14, + 0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8, + 0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a, + 0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13, + 0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8, + 0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63, + 0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee, + 0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76, + 0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25, + 0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78, + 0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7, + 0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70, + 0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a, + 0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66, + 0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b, + 0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c, + 0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29, + 0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99, + 0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6, + 0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5, + 0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde, + 0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c, + 0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e, + 0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6, + 0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa, + 0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66, + 0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed, + 0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78, + 0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab, + 0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29, + 0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0, + 0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4, + 0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e, + 0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3, + 0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a, + 0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9, + 0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80, + 0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3, + 0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80, + 0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9, + 0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4, + 0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f, + 0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69, + 0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e, + 0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a, + 0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2, + 0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0, + 0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7, + 0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34, + 0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a, + 0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10, + 0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6, + 0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e, + 0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e, + 0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16, + 0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5, + 0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0, + 0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b, + 0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d, + 0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06, + 0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index ca987ec51..7126811ac 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package internal; message IndexMeta { - string ColumnLabel = 1; string TimeQuantum = 2; } diff --git a/pilosa.go b/pilosa.go index 71b63fcaa..a3e11b826 100644 --- a/pilosa.go +++ b/pilosa.go @@ -40,7 +40,6 @@ var ( ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey") - ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") ErrInputDefinitionNameRequired = errors.New("input-definition name required") ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrInputDefinitionValueMap = errors.New("valueMap required for map") diff --git a/server.go b/server.go index 7f6eeb6cb..cb82de12d 100644 --- a/server.go +++ b/server.go @@ -378,7 +378,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { } case *internal.CreateIndexMessage: opt := IndexOptions{ - ColumnLabel: obj.Meta.ColumnLabel, TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), } _, err := s.Holder.CreateIndex(obj.Index, opt) diff --git a/server/cluster_test.go b/server/cluster_test.go index de623bbd4..828aacdf7 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -153,8 +153,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", columnID=1) - SetBit(row=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { "cacheType": "ranked", "timeQuantum": "YMD" }}], - "fields": [{"name": "columnID", + "fields": [{"name": "col", "primaryKey": true }]} `); err != nil { @@ -351,8 +351,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", columnID=1) - SetBit(row=1, frame="f", columnID=1300000) + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=1300000) `); err != nil { t.Fatal(err) } @@ -406,8 +406,8 @@ func TestClusterResize_AddNode(t *testing.T) { // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", columnID=1) - SetBit(row=1, frame="f", columnID=2400000) + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -560,7 +560,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setBits := "" for i := 0; i < 20; i++ { - setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth) + setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setBits); err != nil { diff --git a/server/server_test.go b/server/server_test.go index 28155bab4..b91cb4317 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -56,7 +56,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -131,13 +131,13 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set bits on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil { t.Fatal(err) } @@ -204,14 +204,14 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { t.Fatal(err) } @@ -234,40 +234,6 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } -// Ensure program can set column attributes with columnLabel option. -func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { - m := test.MustRunMain() - defer m.Close() - - // Create frames. - client := m.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - // Set bits on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { - t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { - t.Fatal(err) - } - - // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { - t.Fatal(err) - } - - // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { - t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { - t.Fatalf("unexpected result: %s", res) - } - -} - // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.MustRunMainWithCluster(t, 2) @@ -285,13 +251,13 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m10.Query("i", "", ` - SetBit(row=1, frame="f", columnID=100) - SetBit(row=1, frame="f", columnID=1000) - SetBit(row=1, frame="f", columnID=100000) - SetBit(row=1, frame="f", columnID=200000) - SetBit(row=1, frame="f", columnID=400000) - SetBit(row=1, frame="f", columnID=600000) - SetBit(row=1, frame="f", columnID=800000) + SetBit(row=1, frame="f", col=100) + SetBit(row=1, frame="f", col=1000) + SetBit(row=1, frame="f", col=100000) + SetBit(row=1, frame="f", col=200000) + SetBit(row=1, frame="f", col=400000) + SetBit(row=1, frame="f", col=600000) + SetBit(row=1, frame="f", col=800000) `); err != nil { t.Fatal("setting bits:", err) } @@ -401,7 +367,7 @@ func TestMain_RecalculateHashes(t *testing.T) { data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", columnID=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { diff --git a/stats_test.go b/stats_test.go index a03407a3d..3e85532a3 100644 --- a/stats_test.go +++ b/stats_test.go @@ -206,7 +206,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { From 9a2aeac6f83e151b5de1dae3c1d1a9c9a9628323 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 20 Mar 2018 16:34:30 -0500 Subject: [PATCH 04/21] Clean up logger; make it honor --log-path flag. Add functional options to NewGossipMemberSet. --- cluster.go | 96 ++++++++++-------------- cmd/server.go | 2 +- diagnostics.go | 21 +----- diagnostics_internal_test.go | 3 - fragment.go | 25 +++---- frame.go | 7 +- gossip/gossip.go | 141 +++++++++++++++++------------------ handler.go | 121 ++++++++++++++---------------- handler_test.go | 6 +- holder.go | 36 ++++----- holder_test.go | 13 +++- index.go | 7 +- logger.go | 69 +++++++++++++++++ server.go | 49 ++++++------ server/cluster_test.go | 4 +- server/server.go | 34 +++++---- stats.go | 9 +-- stats_test.go | 5 +- statsd/statsd.go | 42 +++++------ statsd/statsd_test.go | 2 - test/handler.go | 1 - test/holder.go | 7 +- test/logger.go | 48 ++++++++++++ test/pilosa.go | 2 +- view.go | 16 ++-- 25 files changed, 410 insertions(+), 356 deletions(-) create mode 100644 logger.go create mode 100644 test/logger.go diff --git a/cluster.go b/cluster.go index a5b7b654f..7e53628ac 100644 --- a/cluster.go +++ b/cluster.go @@ -20,9 +20,7 @@ import ( "errors" "fmt" "hash/fnv" - "io" "io/ioutil" - "log" "math/rand" "net/http" "os" @@ -268,8 +266,7 @@ type Cluster struct { closing chan struct{} prefect SecurityManager - // The writer for any logging. - LogOutput io.Writer + Logger Logger // RemoteClient *http.Client @@ -289,16 +286,11 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), - LogOutput: os.Stderr, - prefect: &NopSecurityManager{}, + Logger: NopLogger, + prefect: &NopSecurityManager{}, } } -// logger returns a logger for the cluster. -func (c *Cluster) logger() *log.Logger { - return log.New(c.LogOutput, "", log.LstdFlags) -} - // Coordinator returns the coordinator node. func (c *Cluster) CoordinatorNode() *Node { return c.nodeByID(c.Coordinator) @@ -359,7 +351,7 @@ func (c *Cluster) UpdateCoordinator(n *Node) bool { // AddNode adds a node to the Cluster and updates and saves the // new topology. func (c *Cluster) AddNode(node *Node) error { - c.logger().Printf("add node %s to cluster on %s", node, c.Node) + c.Logger.Printf("add node %s to cluster on %s", node, c.Node) // If the node being added is the coordinator, set it for this node. if node.IsCoordinator { @@ -437,7 +429,7 @@ func (c *Cluster) setState(state string) { return } - c.logger().Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) + c.Logger.Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) var doCleanup bool @@ -471,7 +463,7 @@ func (c *Cluster) setState(state string) { // Clean holder. if err := cleaner.CleanHolder(); err != nil { - c.logger().Printf("holder clean error: err=%s", err) + c.Logger.Printf("holder clean error: err=%s", err) } } } @@ -487,7 +479,7 @@ func (c *Cluster) SetNodeState(state string) error { State: state, } - c.logger().Printf("Sending State %s (%s)", state, c.Coordinator) + c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -509,7 +501,7 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { } c.Topology.nodeStates[nodeID] = state - c.logger().Printf("received state %s (%s)", state, nodeID) + c.Logger.Printf("received state %s (%s)", state, nodeID) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { @@ -951,9 +943,9 @@ func (c *Cluster) Open() error { return fmt.Errorf("sending restart NodeJoin: %v", err) } - c.logger().Printf("wait for joining to complete") + c.Logger.Printf("wait for joining to complete") <-c.joining - c.logger().Printf("joining has completed") + c.Logger.Printf("joining has completed") } return nil @@ -968,7 +960,7 @@ func (c *Cluster) Close() error { } func (c *Cluster) markAsJoined() { - c.logger().Printf("mark node as joined (received coordinator update)") + c.Logger.Printf("mark node as joined (received coordinator update)") if !c.joined { c.joined = true close(c.joining) @@ -1001,9 +993,9 @@ func (c *Cluster) allNodesReady() bool { func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { j, err := c.generateResizeJob(nodeAction) if err != nil { - c.logger().Printf("generateResizeJob error: err=%s", err) + c.Logger.Printf("generateResizeJob error: err=%s", err) if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.logger().Printf("setStateAndBroadcast error: err=%s", err) + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) } return err } @@ -1017,7 +1009,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { }) // Wait for the ResizeJob to finish or be aborted. - c.logger().Printf("wait for jobResult") + c.Logger.Printf("wait for jobResult") jobResult := <-j.result // Make sure j.Run() didn't return an error. @@ -1025,7 +1017,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { return err } - c.logger().Printf("received jobResult: %s", jobResult) + c.Logger.Printf("received jobResult: %s", jobResult) switch jobResult { case ResizeJobStateDone: if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { @@ -1048,7 +1040,7 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { func (c *Cluster) setStateAndBroadcast(state string) error { c.SetState(state) // Broadcast cluster status changes to the cluster. - c.logger().Printf("broadcasting ClusterStatus: %s", state) + c.Logger.Printf("broadcasting ClusterStatus: %s", state) return c.Broadcaster.SendSync(c.Status()) } @@ -1081,7 +1073,7 @@ func (c *Cluster) listenForJoins() { case nodeAction := <-c.joiningLeavingNodes: err := c.handleNodeAction(nodeAction) if err != nil { - c.logger().Printf("handleNodeAction error: err=%s", err) + c.Logger.Printf("handleNodeAction error: err=%s", err) continue } setNormal = true @@ -1093,7 +1085,7 @@ func (c *Cluster) listenForJoins() { if setNormal { // Put the cluster back to state NORMAL and broadcast. if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.logger().Printf("setStateAndBroadcast error: err=%s", err) + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) } } @@ -1104,7 +1096,7 @@ func (c *Cluster) listenForJoins() { case nodeAction := <-c.joiningLeavingNodes: err := c.handleNodeAction(nodeAction) if err != nil { - c.logger().Printf("handleNodeAction error: err=%s", err) + c.Logger.Printf("handleNodeAction error: err=%s", err) continue } setNormal = true @@ -1117,7 +1109,7 @@ func (c *Cluster) listenForJoins() { // added/removed. It also saves a reference to the ResizeJob in the `jobs` map // for future lookup by JobID. func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { - c.logger().Printf("generateResizeJob: %v", nodeAction) + c.Logger.Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -1125,7 +1117,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, err } - c.logger().Printf("generated ResizeJob: %d", j.ID) + c.Logger.Printf("generated ResizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1215,14 +1207,14 @@ func (c *Cluster) CompleteCurrentJob(state string) error { // FollowResizeInstruction is run by any node that receives a ResizeInstruction. func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - c.logger().Printf("follow resize instruction on %s", c.Node.ID) + c.Logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { return err } - c.logger().Printf("MergeClusterStatus done, start goroutine") + c.Logger.Printf("MergeClusterStatus done, start goroutine") // The actual resizing runs in a goroutine because we don't want to block // the distribution of other ResizeInstructions to the rest of the cluster. @@ -1242,7 +1234,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err if err := func() error { // Sync the schema received in the resize instruction. - c.logger().Printf("Holder ApplySchema") + c.Logger.Printf("Holder ApplySchema") if err := c.Holder.ApplySchema(instr.Schema); err != nil { return err } @@ -1252,7 +1244,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.logger().Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) srcURI := decodeURI(src.Node.URI) @@ -1275,7 +1267,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } // Stream slice from remote node. - c.logger().Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found @@ -1309,7 +1301,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err } if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { - c.logger().Printf("sending resizeInstructionComplete error: err=%s", err) + c.Logger.Printf("sending resizeInstructionComplete error: err=%s", err) } }() return nil @@ -1363,13 +1355,7 @@ type ResizeJob struct { mu sync.RWMutex state string - // The writer for any logging. - LogOutput io.Writer -} - -// logger returns a logger for the resize job. -func (j *ResizeJob) logger() *log.Logger { - return log.New(j.LogOutput, "", log.LstdFlags) + Logger Logger } // NewResizeJob returns a new instance of ResizeJob. @@ -1397,11 +1383,11 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } return &ResizeJob{ - ID: rand.Int63(), - IDs: ids, - action: action, - result: make(chan string), - LogOutput: os.Stderr, + ID: rand.Int63(), + IDs: ids, + action: action, + result: make(chan string), + Logger: NopLogger, } } @@ -1425,18 +1411,18 @@ func (j *ResizeJob) setState(state string) { // Run distributes ResizeInstructions. func (j *ResizeJob) Run() error { - j.logger().Printf("run ResizeJob") + j.Logger.Printf("run ResizeJob") // Set job state to RUNNING. j.SetState(ResizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.nodesArePending() { - j.logger().Printf("ResizeJob contains no pending tasks; mark as done") + j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") j.result <- ResizeJobStateDone return nil } - j.logger().Printf("distribute tasks for ResizeJob") + j.Logger.Printf("distribute tasks for ResizeJob") err := j.distributeResizeInstructions() if err != nil { j.result <- ResizeJobStateAborted @@ -1466,7 +1452,7 @@ func (j *ResizeJob) nodesArePending() bool { } func (j *ResizeJob) distributeResizeInstructions() error { - j.logger().Printf("distributeResizeInstructions for job %d", j.ID) + j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) // Loop through the ResizeInstructions in ResizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create @@ -1475,7 +1461,7 @@ func (j *ResizeJob) distributeResizeInstructions() error { ID: instr.Node.ID, URI: decodeURI(instr.Node.URI), } - j.logger().Printf("send resize instructions: %v", instr) + j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { return err } @@ -1681,7 +1667,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { switch e.Event { case NodeJoin: - c.logger().Printf("received NodeJoin event: %v", e) + c.Logger.Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. if !c.IsCoordinator() { return nil @@ -1701,7 +1687,7 @@ func (c *Cluster) nodeJoin(node *Node) error { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsID(node.ID) { err := fmt.Sprintf("host is not in topology: %s", node.ID) - c.logger().Print(err) + c.Logger.Printf("%v", err) return errors.New(err) } @@ -1816,7 +1802,7 @@ func (c *Cluster) nodeLeave(node *Node) error { } func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { - c.logger().Printf("merge cluster status: %v", cs) + c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). if c.IsCoordinator() { return nil diff --git a/cmd/server.go b/cmd/server.go index ffa8c66b3..8089dae54 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -43,7 +43,7 @@ func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Long: `pilosa server runs Pilosa. It will load existing data from the configured -directory, and start listening client connections +directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr) diff --git a/diagnostics.go b/diagnostics.go index 3ad07ed42..5d3f6699b 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -18,9 +18,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" - "io/ioutil" - "log" "net/http" "strconv" "strings" @@ -52,7 +49,7 @@ type DiagnosticsCollector struct { client *http.Client - logOutput io.Writer + Logger Logger server *Server } @@ -66,7 +63,7 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector { start: time.Now(), client: &http.Client{Timeout: 10 * time.Second}, metrics: make(map[string]interface{}), - logOutput: ioutil.Discard, + Logger: NopLogger, } } @@ -119,7 +116,7 @@ func (d *DiagnosticsCollector) CheckVersion() error { d.lastVersion = rsp.Version if err := d.compareVersion(rsp.Version); err != nil { - d.logger().Printf("%s\n", err.Error()) + d.Logger.Printf("%s\n", err.Error()) } return nil @@ -160,20 +157,10 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) { d.metrics[name] = value } -// SetLogger Set the logger output type. -func (d *DiagnosticsCollector) SetLogger(logger io.Writer) { - d.logOutput = logger -} - -// logger returns a logger that writes to LogOutput. -func (d *DiagnosticsCollector) logger() *log.Logger { - return log.New(d.logOutput, "", log.LstdFlags) -} - // logErr logs the error and returns true if an error exists func (d *DiagnosticsCollector) logErr(err error) bool { if err != nil { - d.logOutput.Write([]byte(err.Error())) + d.Logger.Printf("%v", err) return true } return false diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index eb2498297..517dbed3d 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -16,7 +16,6 @@ package pilosa import ( "encoding/json" - "io/ioutil" "net/http" "net/http/httptest" "reflect" @@ -31,7 +30,6 @@ func TestDiagnosticsClient(t *testing.T) { // Create a new client. d := NewDiagnosticsCollector(server.URL) - d.SetLogger(ioutil.Discard) d.Set("gg", 10) d.Set("ss", "ss") @@ -146,7 +144,6 @@ func BenchmarkDiagnostics(b *testing.B) { // Create a new client. d := NewDiagnosticsCollector(server.URL) - d.SetLogger(ioutil.Discard) prev := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(prev) diff --git a/fragment.go b/fragment.go index ab397af47..674594081 100644 --- a/fragment.go +++ b/fragment.go @@ -26,7 +26,6 @@ import ( "hash" "io" "io/ioutil" - "log" "net/http" "os" "sort" @@ -103,8 +102,8 @@ type Fragment struct { // so that they can be mmapped and heap utilization can be kept low. MaxOpN int - // Writer used for out-of-band log entries. - LogOutput io.Writer + // Logger used for out-of-band log entries. + Logger Logger // Row attribute storage. // This is set by the parent frame unless overridden for testing. @@ -124,8 +123,8 @@ func NewFragment(path, index, frame, view string, slice uint64) *Fragment { CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, - LogOutput: ioutil.Discard, - MaxOpN: DefaultFragmentMaxOpN, + Logger: NopLogger, + MaxOpN: DefaultFragmentMaxOpN, stats: NopStatsClient, } @@ -273,7 +272,7 @@ func (f *Fragment) openCache() error { // Unmarshal cache data. var pb internal.Cache if err := proto.Unmarshal(buf, &pb); err != nil { - f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) + f.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) return nil } @@ -298,13 +297,13 @@ func (f *Fragment) Close() error { func (f *Fragment) close() error { // Flush cache if closing gracefully. if err := f.flushCache(); err != nil { - f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) + f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) return err } // Close underlying storage. if err := f.closeStorage(); err != nil { - f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) + f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) return err } @@ -343,9 +342,6 @@ func (f *Fragment) closeStorage() error { return nil } -// logger returns a logger instance for the fragment.nt. -func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) } - // Row returns a row by ID. func (f *Fragment) Row(rowID uint64) *Bitmap { f.mu.Lock() @@ -1385,18 +1381,17 @@ func (f *Fragment) Snapshot() error { defer f.mu.Unlock() return f.snapshot() } -func track(start time.Time, message string, stats StatsClient, logger *log.Logger) { +func track(start time.Time, message string, stats StatsClient, logger Logger) { elapsed := time.Since(start) logger.Printf("%s took %s", message, elapsed) stats.Histogram("snapshot", elapsed.Seconds(), 1.0) } func (f *Fragment) snapshot() error { - logger := f.logger() - logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) start := time.Now() - defer track(start, completeMessage, f.stats, logger) + defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt diff --git a/frame.go b/frame.go index 466d7ae96..831ac0a55 100644 --- a/frame.go +++ b/frame.go @@ -17,7 +17,6 @@ package pilosa import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -65,7 +64,7 @@ type Frame struct { rangeEnabled bool fields []*Field - LogOutput io.Writer + Logger Logger } // NewFrame returns a new instance of frame. @@ -95,7 +94,7 @@ func NewFrame(path, index, name string) (*Frame, error) { rangeEnabled: DefaultRangeEnabled, //fields - LogOutput: ioutil.Discard, + Logger: NopLogger, }, nil } @@ -624,7 +623,7 @@ func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { func (f *Frame) newView(path, name string) *View { view := NewView(path, f.index, f.name, name, f.cacheSize) view.cacheType = f.cacheType - view.LogOutput = f.LogOutput + view.Logger = f.Logger view.RowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster diff --git a/gossip/gossip.go b/gossip/gossip.go index 62c794560..4a1e0f5f1 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -16,10 +16,8 @@ package gossip import ( "fmt" - "io" "io/ioutil" "log" - "os" "strconv" "strings" "sync" @@ -51,8 +49,10 @@ type GossipMemberSet struct { statusHandler pilosa.StatusHandler config *gossipConfig - // The writer for any logging. - LogOutput io.Writer + Logger pilosa.Logger + + logger *log.Logger + transport *Transport } // Start implements the BroadcastReceiver interface and sets the BroadcastHandler. @@ -140,11 +140,6 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) { return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } -// logger returns a logger for the GossipMemberSet. -func (g *GossipMemberSet) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - //////////////////////////////////////////////////////////////// type gossipConfig struct { @@ -152,14 +147,58 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } -// NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport. -func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport *Transport, server *pilosa.Server) (*GossipMemberSet, error) { +type GossipMemberSetOption func(*GossipMemberSet) error + +func WithTransport(transport *Transport) func(*GossipMemberSet) error { + return func(g *GossipMemberSet) error { + g.transport = transport + return nil + } +} + +func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { + return func(g *GossipMemberSet) error { + g.logger = logger + return nil + } +} + +// NewGossipMemberSet returns a new instance of GossipMemberSet based on options. +func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { g := &GossipMemberSet{ - LogOutput: server.LogOutput, + Logger: server.Logger, } - port := transport.Net.GetAutoBindPort() + // options + for _, opt := range options { + if err := opt(g); err != nil { + return nil, err + } + } + + if g.transport == nil { + port, err := strconv.Atoi(cfg.Gossip.Port) + if err != nil { + return nil, fmt.Errorf("convert port: %s", err) + } + + bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) + if err != nil { + return nil, fmt.Errorf("getting uri from bind address: %s", err) + } + host := bindURI.Host() + + // Set up the transport. + transport, err := NewTransport(host, port, g.logger) + if err != nil { + return nil, fmt.Errorf("new tranport: %s", err) + } + + g.transport = transport + } + + port := g.transport.Net.GetAutoBindPort() bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) if err != nil { @@ -177,7 +216,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport // memberlist config conf := memberlist.DefaultWANConfig() - conf.Transport = transport.Net + conf.Transport = g.transport.Net conf.Name = name conf.BindAddr = host conf.BindPort = port @@ -196,6 +235,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport conf.Delegate = g conf.SecretKey = gossipKey conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) + conf.Logger = g.logger g.config = &gossipConfig{ memberlistConfig: conf, @@ -207,28 +247,6 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport return g, nil } -// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port. -func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) { - port, err := strconv.Atoi(cfg.Gossip.Port) - if err != nil { - return nil, fmt.Errorf("convert port: %s", err) - } - - bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) - if err != nil { - return nil, fmt.Errorf("getting uri from bind address: %s", err) - } - host := bindURI.Host() - - // Set up the transport. - transport, err := NewTransport(host, port) - if err != nil { - return nil, fmt.Errorf("new tranport: %s", err) - } - - return NewGossipMemberSetWithTransport(name, cfg, transport, server) -} - // SendSync implementation of the Broadcaster interface. func (g *GossipMemberSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) @@ -276,7 +294,7 @@ func (g *GossipMemberSet) SendAsync(pb proto.Message) error { func (g *GossipMemberSet) NodeMeta(limit int) []byte { buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) if err != nil { - g.logger().Printf("marshal message error: %s", err) + g.Logger.Printf("marshal message error: %s", err) return []byte{} } return buf @@ -287,11 +305,11 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte { func (g *GossipMemberSet) NotifyMsg(b []byte) { m, err := pilosa.UnmarshalMessage(b) if err != nil { - g.logger().Printf("unmarshal message error: %s", err) + g.Logger.Printf("unmarshal message error: %s", err) return } if err := g.handler.ReceiveMessage(m); err != nil { - g.logger().Printf("receive message error: %s", err) + g.Logger.Printf("receive message error: %s", err) return } } @@ -307,14 +325,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { func (g *GossipMemberSet) LocalState(join bool) []byte { pb, err := g.statusHandler.LocalStatus() if err != nil { - g.logger().Printf("error getting local state, err=%s", err) + g.Logger.Printf("error getting local state, err=%s", err) return []byte{} } // Marshal nodestate data to bytes. buf, err := proto.Marshal(pb) if err != nil { - g.logger().Printf("error marshalling nodestate data, err=%s", err) + g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} } return buf @@ -326,12 +344,12 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeStatus if err := proto.Unmarshal(buf, &pb); err != nil { - g.logger().Printf("error unmarshalling nodestate data, err=%s", err) + g.Logger.Printf("error unmarshalling nodestate data, err=%s", err) return } err := g.statusHandler.HandleRemoteStatus(&pb) if err != nil { - g.logger().Printf("merge state error: %s", err) + g.Logger.Printf("merge state error: %s", err) } } @@ -344,15 +362,14 @@ type GossipEventReceiver struct { ch chan memberlist.NodeEvent eventHandler pilosa.EventHandler - // The writer for any logging. - LogOutput io.Writer + Logger pilosa.Logger } // NewGossipEventReceiver returns a new instance of GossipEventReceiver. -func NewGossipEventReceiver(logOutput io.Writer) *GossipEventReceiver { +func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver { return &GossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - LogOutput: logOutput, + ch: make(chan memberlist.NodeEvent, 1), + Logger: logger, } } @@ -375,11 +392,6 @@ func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error { return nil } -// logger returns a logger for the GossipEventReceiver. -func (g *GossipEventReceiver) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - func (g *GossipEventReceiver) listen() { var nodeEventType pilosa.NodeEventType for { @@ -407,7 +419,7 @@ func (g *GossipEventReceiver) listen() { Node: node, } if err := g.eventHandler.ReceiveEvent(ne); err != nil { - g.logger().Printf("receive event error: %s", err) + g.Logger.Printf("receive event error: %s", err) } } } @@ -443,12 +455,13 @@ type Transport struct { // It will dynamically bind to a port if port is 0. // This is useful for test cases where specifiying a port is not reasonable. //func NewTransport(host string, port int) (*memberlist.NetTransport, error) { -func NewTransport(host string, port int) (*Transport, error) { +func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) { // memberlist config conf := memberlist.DefaultWANConfig() conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port + conf.Logger = logger net, err := newTransport(conf) if err != nil { @@ -469,24 +482,10 @@ func NewTransport(host string, port int) (*Transport, error) { // newTransport returns a NetTransport based on the memberlist configuration. // It will dynamically bind to a port if conf.BindPort is 0. func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { - if conf.LogOutput != nil && conf.Logger != nil { - return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") - } - - logDest := conf.LogOutput - if logDest == nil { - logDest = os.Stderr - } - - logger := conf.Logger - if logger == nil { - logger = log.New(logDest, "", log.LstdFlags) - } - nc := &memberlist.NetTransportConfig{ BindAddrs: []string{conf.BindAddr}, BindPort: conf.BindPort, - Logger: logger, + Logger: conf.Logger, } // See comment below for details about the retry in here. @@ -498,7 +497,7 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { return nt, nil } if strings.Contains(err.Error(), "address already in use") { - logger.Printf("[DEBUG] Got bind error: %v", err) + conf.Logger.Printf("[DEBUG] Got bind error: %v", err) continue } } diff --git a/handler.go b/handler.go index a23b2e5ac..ffd28da46 100644 --- a/handler.go +++ b/handler.go @@ -23,12 +23,10 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "net/url" // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" - "os" "runtime/debug" "strconv" "strings" @@ -67,8 +65,7 @@ type Handler struct { Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } - // The writer for any logging. - LogOutput io.Writer + Logger Logger // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec @@ -98,8 +95,7 @@ func NewHandler() *Handler { //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop //StatusHandler: NopStatusHandler, // TODO: implement the nop FileSystem: NopFileSystem, - - LogOutput: os.Stderr, + Logger: NopLogger, } BuildRouters(handler) handler.populateValidators() @@ -247,7 +243,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) stack := debug.Stack() msg := "PANIC: %s\n%s" - fmt.Fprintf(h.LogOutput, msg, err, stack) + h.Logger.Printf(msg, err, stack) fmt.Fprintf(w, msg, err, stack) } }() @@ -261,7 +257,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statsTags := make([]string, 0, 3) if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { - h.logger().Printf("%s %s %v", r.Method, r.URL.String(), dif) + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) statsTags = append(statsTags, "slow_query") } @@ -289,7 +285,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { filesystem, err := h.FileSystem.New() if err != nil { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") return } http.FileServer(filesystem).ServeHTTP(w, r) @@ -300,7 +296,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSchemaResponse{ Indexes: h.Holder.Schema(), }); err != nil { - h.logger().Printf("write schema response error: %s", err) + h.Logger.Printf("write schema response error: %s", err) } } @@ -308,7 +304,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { pb, err := h.StatusHandler.ClusterStatus() if err != nil { - h.logger().Printf("cluster status error: %s", err) + h.Logger.Printf("cluster status error: %s", err) return } @@ -317,7 +313,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { State: cs.State, Nodes: DecodeNodes(cs.Nodes), }); err != nil { - h.logger().Printf("write status response error: %s", err) + h.Logger.Printf("write status response error: %s", err) } } @@ -395,7 +391,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Write response back to client. if err := h.writeQueryResponse(w, r, resp); err != nil { - h.logger().Printf("write query response error: %s", err) + h.Logger.Printf("write query response error: %s", err) } } @@ -405,7 +401,7 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { Standard: h.Holder.MaxSlices(), Inverse: h.Holder.MaxInverseSlices(), }); err != nil { - h.logger().Printf("write slices-max response error: %s", err) + h.Logger.Printf("write slices-max response error: %s", err) } } @@ -431,7 +427,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getIndexResponse{ map[string]string{"name": index.Name()}, }); err != nil { - h.logger().Printf("write response error: %s", err) + h.Logger.Printf("write response error: %s", err) } } @@ -519,12 +515,12 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { Index: indexName, }) if err != nil { - h.logger().Printf("problem sending DeleteIndex message: %s", err) + h.Logger.Printf("problem sending DeleteIndex message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.Count("deleteIndex", 1, 1.0) @@ -564,14 +560,14 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { Meta: req.Options.Encode(), }) if err != nil { - h.logger().Printf("problem sending CreateIndex message: %s", err) + h.Logger.Printf("problem sending CreateIndex message: %s", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } // Encode response. if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.Count("createIndex", 1, 1.0) @@ -610,7 +606,7 @@ func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Req // Encode response. if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -665,7 +661,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -718,12 +714,12 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { Meta: req.Options.Encode(), }) if err != nil { - h.logger().Printf("problem sending CreateFrame message: %s", err) + h.Logger.Printf("problem sending CreateFrame message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -783,7 +779,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { index := h.Holder.Index(indexName) if index == nil { if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } return } @@ -801,12 +797,12 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { Frame: frameName, }) if err != nil { - h.logger().Printf("problem sending DeleteFrame message: %s", err) + h.Logger.Printf("problem sending DeleteFrame message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -848,7 +844,7 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req // Encode response. if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -899,12 +895,12 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { Field: encodeField(field), }) if err != nil { - h.logger().Printf("problem sending CreateField message: %s", err) + h.Logger.Printf("problem sending CreateField message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -943,12 +939,12 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) Field: fieldName, }) if err != nil { - h.logger().Printf("problem sending DeleteField message: %s", err) + h.Logger.Printf("problem sending DeleteField message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -979,7 +975,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { // Encode response. if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1012,7 +1008,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { // Encode response. if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1046,12 +1042,12 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { View: viewName, }) if err != nil { - h.logger().Printf("problem sending DeleteView message: %s", err) + h.Logger.Printf("problem sending DeleteView message: %s", err) } // Encode response. if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1107,7 +1103,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1276,10 +1272,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Find the Index. - h.logger().Println("importing:", req.Index, req.Frame, req.Slice) + h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) index := h.Holder.Index(req.Index) if index == nil { - h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) + h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1287,7 +1283,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Retrieve frame. f := index.Frame(req.Frame) if f == nil { - h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) + h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -1295,7 +1291,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Import into fragment. err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - h.logger().Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -1346,10 +1342,10 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) } // Find the Index. - h.logger().Println("importing:", req.Index, req.Frame, req.Slice) + h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) index := h.Holder.Index(req.Index) if index == nil { - h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) + h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1357,7 +1353,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) // Retrieve frame. f := index.Frame(req.Frame) if f == nil { - h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) + h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -1365,7 +1361,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) // Import into fragment. err = f.ImportValue(req.Field, req.ColumnIDs, req.Values) if err != nil { - h.logger().Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) return } @@ -1452,7 +1448,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.logger().Printf("json write error: %s", err) + h.Logger.Printf("json write error: %s", err) } } @@ -1475,7 +1471,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) // Stream fragment to response body. if _, err := f.WriteTo(w); err != nil { - h.logger().Printf("fragment backup error: %s", err) + h.Logger.Printf("fragment backup error: %s", err) } } @@ -1545,7 +1541,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Encode response. buf, err := proto.Marshal(&resp) if err != nil { - h.logger().Printf("merge block response encoding error: %s", err) + h.Logger.Printf("merge block response encoding error: %s", err) return } @@ -1579,7 +1575,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ Blocks: blocks, }); err != nil { - h.logger().Printf("block response encoding error: %s", err) + h.Logger.Printf("block response encoding error: %s", err) } } @@ -1680,7 +1676,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { - h.logger().Printf("write version response error: %s", err) + h.Logger.Printf("write version response error: %s", err) } } @@ -1696,7 +1692,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { }{ Version: version, }); err != nil { - h.logger().Printf("write version response error: %s", err) + h.Logger.Printf("write version response error: %s", err) } } @@ -1716,11 +1712,6 @@ func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "\n}\n") } -// logger returns a logger for the handler. -func (h *Handler) logger() *log.Logger { - return log.New(h.LogOutput, "", log.LstdFlags) -} - // QueryResult types. const ( QueryResultTypeNil uint32 = iota @@ -1908,11 +1899,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque Definition: def, }) if err != nil { - h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + h.Logger.Printf("problem sending CreateInputDefinition message: %s", err) } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1938,7 +1929,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques Frames: inputDef.frames, Fields: inputDef.fields, }); err != nil { - h.logger().Printf("write status response error: %s", err) + h.Logger.Printf("write status response error: %s", err) } } @@ -1967,11 +1958,11 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req Name: inputDefName, }) if err != nil { - h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) + h.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2013,7 +2004,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2061,7 +2052,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r Old: oldNode, New: newNode, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2101,7 +2092,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht if err := json.NewEncoder(w).Encode(removeNodeResponse{ Remove: removeNode, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2134,7 +2125,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ Info: msg, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -2271,7 +2262,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 551c81d66..1e30f41ad 100644 --- a/handler_test.go +++ b/handler_test.go @@ -37,13 +37,13 @@ import ( func TestHandlerPanics(t *testing.T) { h := test.NewHandler() - buf := &bytes.Buffer{} - h.Handler.LogOutput = buf + bufLogger := test.NewBufferLogger() + h.Handler.Logger = bufLogger w := httptest.NewRecorder() // will panic since Handler has no Holder set up h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) - bufbytes, err := ioutil.ReadAll(buf) + bufbytes, err := bufLogger.ReadAll() if err != nil { t.Fatalf("reading all logoutput: %v", err) } diff --git a/holder.go b/holder.go index 63bbbbc86..6d1d4ef12 100644 --- a/holder.go +++ b/holder.go @@ -18,9 +18,7 @@ import ( "context" "errors" "fmt" - "io" "io/ioutil" - "log" "net/http" "os" "path" @@ -71,7 +69,7 @@ type Holder struct { // The interval at which the cached row ids are persisted to disk. CacheFlushInterval time.Duration - LogOutput io.Writer + Logger Logger } // NewHolder returns a new instance of Holder. @@ -89,7 +87,7 @@ func NewHolder() *Holder { CacheFlushInterval: DefaultCacheFlushInterval, - LogOutput: os.Stderr, + Logger: NopLogger, } } @@ -97,7 +95,7 @@ func NewHolder() *Holder { // without actually loading any data into memory. // HasData is returned, and h.hasData is set. func (h *Holder) Peek() bool { - h.logger().Printf("peek at holder path: %s", h.Path) + h.Logger.Printf("peek at holder path: %s", h.Path) h.hasData = false // Open path to read all index directories. @@ -127,7 +125,7 @@ func (h *Holder) Peek() bool { func (h *Holder) Open() error { h.setFileLimit() - h.logger().Printf("open holder path: %s", h.Path) + h.Logger.Printf("open holder path: %s", h.Path) if err := os.MkdirAll(h.Path, 0777); err != nil { return err } @@ -149,18 +147,18 @@ func (h *Holder) Open() error { continue } - h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) + h.Logger.Printf("opening index: %s", filepath.Base(fi.Name())) index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err) + h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { return err } if err := index.Open(); err != nil { if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err) + h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue } return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) @@ -169,7 +167,7 @@ func (h *Holder) Open() error { h.indexes[index.Name()] = index h.mu.Unlock() } - h.logger().Printf("open holder: complete") + h.Logger.Printf("open holder: complete") // Periodically flush cache. h.wg.Add(1) @@ -374,7 +372,7 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { if err != nil { return nil, err } - index.LogOutput = h.LogOutput + index.Logger = h.Logger index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.Broadcaster index.NewAttrStore = h.NewAttrStore @@ -464,7 +462,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) } } } @@ -488,7 +486,7 @@ func (h *Holder) setFileLimit() { newLimit := &syscall.Rlimit{} if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { - h.logger().Printf("ERROR checking open file limit: %s", err) + h.Logger.Printf("ERROR checking open file limit: %s", err) return } // If the soft limit is lower than the FileLimit constant, we will try to change it. @@ -512,32 +510,30 @@ func (h *Holder) setFileLimit() { } // Try setting again with lowered Max (hard limit) if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil { - h.logger().Printf("ERROR setting open file limit: %s", err) + h.Logger.Printf("ERROR setting open file limit: %s", err) } // If we weren't trying to change the hard limit, let the user know something is wrong. } else { - h.logger().Printf("ERROR setting open file limit: %s", err) + h.Logger.Printf("ERROR setting open file limit: %s", err) } } // Check the limit after setting it. OS may not obey Setrlimit call. if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { - h.logger().Printf("ERROR checking open file limit: %s", err) + h.Logger.Printf("ERROR checking open file limit: %s", err) } else { if oldLimit.Cur < FileLimit { - h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) + h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) } } } } -func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } - func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, "ID") nodeID := "" - h.logger().Printf("load NodeID: %s", idPath) + h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { return "", err } diff --git a/holder_test.go b/holder_test.go index 5a06a9272..41ede1adf 100644 --- a/holder_test.go +++ b/holder_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "bytes" "context" "os" "path/filepath" @@ -30,6 +31,10 @@ import ( func TestHolder_Open(t *testing.T) { t.Run("ErrIndexName", func(t *testing.T) { h := test.MustOpenHolder() + + bufLogger := test.NewBufferLogger() + h.Holder.Logger = bufLogger + defer h.Close() if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { @@ -39,8 +44,12 @@ func TestHolder_Open(t *testing.T) { } if err := h.Reopen(); err != nil { t.Fatal(err) - } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { - t.Fatalf("expected log error:\n%s", logOutput) + } + + if bufbytes, err := bufLogger.ReadAll(); err != nil { + t.Fatal(err) + } else if !bytes.Contains(bufbytes, []byte("ERROR opening index: !")) { + t.Fatalf("expected log error:\n%s", bufbytes) } }) diff --git a/index.go b/index.go index 707b87213..5ecad4f78 100644 --- a/index.go +++ b/index.go @@ -17,7 +17,6 @@ package pilosa import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -66,7 +65,7 @@ type Index struct { broadcaster Broadcaster Stats StatsClient - LogOutput io.Writer + Logger Logger } // NewIndex returns a new instance of Index. @@ -92,7 +91,7 @@ func NewIndex(path, name string) (*Index, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - LogOutput: ioutil.Discard, + Logger: NopLogger, }, nil } @@ -528,7 +527,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { if err != nil { return nil, err } - f.LogOutput = i.LogOutput + f.Logger = i.Logger f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) diff --git a/logger.go b/logger.go new file mode 100644 index 000000000..bf516d187 --- /dev/null +++ b/logger.go @@ -0,0 +1,69 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "log" +) + +// Ensure nopLogger implements interface. +var _ Logger = &nopLogger{} + +// Logger represents an interface for a shared logger. +type Logger interface { + Printf(format string, v ...interface{}) + Debugf(format string, v ...interface{}) +} + +func init() { + NopLogger = &nopLogger{ + logger: log.New(ioutil.Discard, "", log.LstdFlags), + } +} + +// NopLogger represents a Logger that doesn't do anything. +var NopLogger Logger + +type nopLogger struct { + logger *log.Logger +} + +// Printf is a no-op implementation of the Logger Printf method. +func (n *nopLogger) Printf(format string, v ...interface{}) {} + +// Debugf is a no-op implementation of the Logger Debugf method. +func (n *nopLogger) Debugf(format string, v ...interface{}) {} + +// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. +type StandardLogger struct { + logger *log.Logger +} + +func NewStandardLogger(logger *log.Logger) *StandardLogger { + return &StandardLogger{ + logger: logger, + } +} + +// Printf is a no-op implementation of the Logger Printf method. +func (s *StandardLogger) Printf(format string, v ...interface{}) { + s.logger.Printf(format, v...) +} + +// Debugf is a no-op implementation of the Logger Debugf method. +func (s *StandardLogger) Debugf(format string, v ...interface{}) { + // TODO: implement this +} diff --git a/server.go b/server.go index 7f6eeb6cb..11b2ee2a3 100644 --- a/server.go +++ b/server.go @@ -20,7 +20,7 @@ import ( "errors" "fmt" "io" - "log" + "io/ioutil" "net" "net/http" "os" @@ -88,7 +88,7 @@ type Server struct { MaxWritesPerRequest int LogOutput io.Writer - logger *log.Logger + Logger Logger defaultClient InternalClient } @@ -115,9 +115,9 @@ func NewServer() *Server { MetricInterval: 0, DiagnosticInterval: 0, - LogOutput: os.Stderr, + LogOutput: ioutil.Discard, + Logger: NopLogger, } - s.logger = log.New(s.LogOutput, "", log.LstdFlags) s.Handler.Holder = s.Holder s.diagnostics.server = s @@ -126,7 +126,7 @@ func NewServer() *Server { // Open opens and initializes the server. func (s *Server) Open() error { - s.Logger().Printf("open server") + s.Logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { if err := s.OpenListener(); err != nil { @@ -151,7 +151,6 @@ func (s *Server) Open() error { // Peek at the holder to determine if there is data on disk. // Don't actually load the data until after the Cluster // management starts. - s.Holder.LogOutput = s.LogOutput s.Holder.Peek() // Create default HTTP client @@ -175,7 +174,6 @@ func (s *Server) Open() error { s.Handler.Node = node s.Handler.Cluster = s.Cluster s.Handler.Executor = e - s.Handler.LogOutput = s.LogOutput s.Cluster.prefect = s.Handler @@ -186,7 +184,7 @@ func (s *Server) Open() error { go func() { err := http.Serve(s.ln, s.Handler) if err != nil { - s.Logger().Printf("HTTP handler terminated with error: %s\n", err) + s.Logger.Printf("HTTP handler terminated with error: %s\n", err) } }() @@ -226,7 +224,7 @@ func (s *Server) Open() error { // OpenListener opens a listener for the Server. func (s *Server) OpenListener() error { - s.Logger().Printf("open server listener: %s", s.URI) + s.Logger.Printf("open server listener: %s", s.URI) if s.ln != nil { return fmt.Errorf("a listener already exists for server: %s", s.URI) } @@ -288,7 +286,7 @@ func (s *Server) LoadNodeID() string { } nodeID, err := s.Holder.loadNodeID() if err != nil { - s.Logger().Printf("loading NodeID: %v", err) + s.Logger.Printf("loading NodeID: %v", err) return s.NodeID } return nodeID @@ -321,14 +319,11 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } -// Logger returns a logger that writes to LogOutput -func (s *Server) Logger() *log.Logger { return s.logger } - func (s *Server) monitorAntiEntropy() { ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() - s.Logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) for { // Wait for tick or a close. @@ -339,7 +334,7 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.Logger().Printf("holder sync beginning") + s.Logger.Printf("holder sync beginning") // Initialize syncer with local holder and remote client. var syncer HolderSyncer @@ -352,12 +347,12 @@ func (s *Server) monitorAntiEntropy() { // Sync holders. if err := syncer.SyncHolder(); err != nil { - s.Logger().Printf("holder sync error: err=%s", err) + s.Logger.Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.Logger().Printf("holder sync complete") + s.Logger.Printf("holder sync complete") dif := time.Since(t) s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } @@ -482,7 +477,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { - s.Logger().Printf("SendSync to: %s", node.URI) + s.Logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.URI == node.URI { continue @@ -504,7 +499,7 @@ func (s *Server) SendAsync(pb proto.Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, pb proto.Message) error { - s.Logger().Printf("SendTo: %s", to.URI) + s.Logger.Printf("SendTo: %s", to.URI) ctx := context.WithValue(context.Background(), "uri", &to.URI) return s.defaultClient.SendMessage(ctx, pb) } @@ -554,7 +549,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) if err != nil { - s.Logger().Printf("merge remote status: %s", err) + s.Logger.Printf("merge remote status: %s", err) } }() @@ -579,7 +574,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger().Printf("Local Index not found: %s", index) + s.Logger.Printf("Local Index not found: %s", index) continue } if newMax > oldmaxslices[index] { @@ -595,7 +590,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger().Printf("Local Index not found: %s", index) + s.Logger.Printf("Local Index not found: %s", index) continue } if newMaxInverse > oldMaxInverseSlices[index] { @@ -611,13 +606,13 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { func (s *Server) monitorDiagnostics() { // Do not send more than once a minute if s.DiagnosticInterval < time.Minute { - s.Logger().Printf("diagnostics disabled") + s.Logger.Printf("diagnostics disabled") return } else { - s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) + s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) } - s.diagnostics.SetLogger(s.LogOutput) + s.diagnostics.Logger = s.Logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) @@ -639,7 +634,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.CheckVersion() err = s.diagnostics.Flush() if err != nil { - s.Logger().Printf("Diagnostics error: %s", err) + s.Logger.Printf("Diagnostics error: %s", err) } } @@ -670,7 +665,7 @@ func (s *Server) monitorRuntime() { defer s.GCNotifier.Close() - s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) + s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval) for { // Wait for tick or a close. diff --git a/server/cluster_test.go b/server/cluster_test.go index fbb784691..6bbdea4b4 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -54,7 +54,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.Coordinator = m0.Server.NodeID m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} - m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput) + m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger) gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) if err != nil { t.Fatal(err) @@ -81,7 +81,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()} m1.Server.Cluster.Coordinator = m0.Server.NodeID - m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput) + m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger) gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) if err != nil { t.Fatal(err) diff --git a/server/server.go b/server/server.go index ce1eff629..d45be21e2 100644 --- a/server/server.go +++ b/server/server.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "log" "math/rand" "os" "path/filepath" @@ -71,6 +72,8 @@ type Command struct { Started chan struct{} // Done will be closed when Command.Close() is called Done chan struct{} + + logger *log.Logger } // NewCommand returns a new instance of Main. @@ -115,7 +118,7 @@ func (m *Command) Run(args ...string) (err error) { return fmt.Errorf("server.Open: %v", err) } - m.Server.Logger().Printf("Listening as %s\n", m.Server.URI) + m.Server.Logger.Printf("Listening as %s\n", m.Server.URI) return nil } @@ -126,6 +129,18 @@ func (m *Command) SetupServer() error { return err } + // Set up logger based on configuration. + lw, err := GetLogWriter(m.Config.LogPath, m.Stderr) + if err != nil { + return err + } + m.Server.LogOutput = lw + m.logger = log.New(lw, "", log.LstdFlags) + m.Server.Logger = pilosa.NewStandardLogger(m.logger) + m.Server.Handler.Logger = m.Server.Logger + m.Server.Holder.Logger = m.Server.Logger + m.Server.Holder.Stats.SetLogger(m.Server.Logger) + uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { @@ -136,15 +151,10 @@ func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN cluster.Holder = m.Server.Holder + cluster.Logger = m.Server.Logger m.Server.Cluster = cluster - // Setup logging output. - m.Server.LogOutput, err = GetLogWriter(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } - // Configure data directory (for Cluster .topology) m.Server.Cluster.Path = m.Config.DataDir @@ -152,7 +162,7 @@ func (m *Command) SetupServer() error { m.Server.Holder.NewAttrStore = boltdb.NewAttrStore // Configure holder. - m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) + m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir) m.Server.Holder.Path = m.Config.DataDir m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) if m.Config.Metric.Diagnostics { @@ -165,8 +175,6 @@ func (m *Command) SetupServer() error { return err } - m.Server.Holder.Stats.SetLogger(m.Server.LogOutput) - // Copy configuration flags. m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest @@ -246,7 +254,7 @@ func (m *Command) SetupNetworking() error { if m.GossipTransport != nil { transport = m.GossipTransport } else { - transport, err = gossip.NewTransport(gossipHost, gossipPort) + transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger) if err != nil { return err } @@ -257,8 +265,8 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Coordinator = m.Server.NodeID } - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput) - gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server) + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger) + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) if err != nil { return err } diff --git a/stats.go b/stats.go index 998b84511..313708fbc 100644 --- a/stats.go +++ b/stats.go @@ -16,7 +16,6 @@ package pilosa import ( "expvar" - "io" "sort" "strings" "sync" @@ -57,7 +56,7 @@ type StatsClient interface { Timing(name string, value time.Duration, rate float64) // SetLogger Set the logger output type - SetLogger(logger io.Writer) + SetLogger(logger Logger) // Starts the service Open() @@ -79,7 +78,7 @@ func (c *nopStatsClient) Gauge(name string, value float64, rate float64) func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {} func (c *nopStatsClient) Set(name string, value string, rate float64) {} func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} -func (c *nopStatsClient) SetLogger(logger io.Writer) {} +func (c *nopStatsClient) SetLogger(logger Logger) {} func (c *nopStatsClient) Open() {} func (c *nopStatsClient) Close() error { return nil } @@ -154,7 +153,7 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 } // SetLogger has no logger. -func (c *ExpvarStatsClient) SetLogger(logger io.Writer) { +func (c *ExpvarStatsClient) SetLogger(logger Logger) { } // Open no-op. @@ -226,7 +225,7 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) } // SetLogger Sets the StatsD logger output type. -func (a MultiStatsClient) SetLogger(logger io.Writer) { +func (a MultiStatsClient) SetLogger(logger Logger) { for _, c := range a { c.SetLogger(logger) } diff --git a/stats_test.go b/stats_test.go index 93d7d9bb1..60deae993 100644 --- a/stats_test.go +++ b/stats_test.go @@ -16,8 +16,6 @@ package pilosa_test import ( "context" - "io" - "io/ioutil" "net/http" "strings" "testing" @@ -38,7 +36,6 @@ func TestMultiStatClient_Expvar(t *testing.T) { ms[0] = c hldr.Stats = ms - hldr.Stats.SetLogger(ioutil.Discard) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -357,6 +354,6 @@ func (c *MockStats) Gauge(name string, value float64, rate float64) {} func (c *MockStats) Histogram(name string, value float64, rate float64) {} func (c *MockStats) Set(name string, value string, rate float64) {} func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} -func (c *MockStats) SetLogger(logger io.Writer) {} +func (c *MockStats) SetLogger(logger pilosa.Logger) {} func (c *MockStats) Open() {} func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go index c7c072ae3..0cec63718 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,9 +15,6 @@ package statsd import ( - "io" - "io/ioutil" - "log" "time" "github.com/DataDog/datadog-go/statsd" @@ -40,9 +37,9 @@ var _ pilosa.StatsClient = &StatsClient{} // StatsClient represents a StatsD implementation of pilosa.StatsClient. type StatsClient struct { - client *statsd.Client - tags []string - logOutput io.Writer + client *statsd.Client + tags []string + logger pilosa.Logger } // NewStatsClient returns a new instance of StatsClient. @@ -53,8 +50,8 @@ func NewStatsClient(host string) (*StatsClient, error) { } return &StatsClient{ - client: c, - logOutput: ioutil.Discard, + client: c, + logger: pilosa.NopLogger, }, nil } @@ -74,16 +71,16 @@ func (c *StatsClient) Tags() []string { // WithTags returns a new client with additional tags appended. func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { return &StatsClient{ - client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), - logOutput: c.logOutput, + client: c.client, + tags: pilosa.UnionStringSlice(c.tags, tags), + logger: c.logger, } } // Count tracks the number of times something occurs per second. func (c *StatsClient) Count(name string, value int64, rate float64) { if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Count error: %s", err) + c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } @@ -91,44 +88,39 @@ func (c *StatsClient) Count(name string, value int64, rate float64) { func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { tags := append(c.tags, t...) if err := c.client.Count(Prefix+name, value, tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Count error: %s", err) + c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // Gauge sets the value of a metric. func (c *StatsClient) Gauge(name string, value float64, rate float64) { if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Gauge error: %s", err) + c.logger.Printf("statsd.StatsClient.Gauge error: %s", err) } } // Histogram tracks statistical distribution of a metric. func (c *StatsClient) Histogram(name string, value float64, rate float64) { if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Histogram error: %s", err) + c.logger.Printf("statsd.StatsClient.Histogram error: %s", err) } } // Set tracks number of unique elements. func (c *StatsClient) Set(name string, value string, rate float64) { if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Set error: %s", err) + c.logger.Printf("statsd.StatsClient.Set error: %s", err) } } // Timing tracks timing information for a metric. func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Timing error: %s", err) + c.logger.Printf("statsd.StatsClient.Timing error: %s", err) } } -// SetLogger has no logger -func (c *StatsClient) SetLogger(logger io.Writer) { - c.logOutput = logger -} - -// logger returns a logger that writes to LogOutput -func (c *StatsClient) logger() *log.Logger { - return log.New(c.logOutput, "", log.LstdFlags) +// SetLogger sets the logger for client. +func (c *StatsClient) SetLogger(logger pilosa.Logger) { + c.logger = logger } diff --git a/statsd/statsd_test.go b/statsd/statsd_test.go index 4d5c1cc98..38228349b 100644 --- a/statsd/statsd_test.go +++ b/statsd/statsd_test.go @@ -15,7 +15,6 @@ package statsd_test import ( - "io/ioutil" "reflect" "testing" "time" @@ -31,7 +30,6 @@ func TestStatsClient_WithTags(t *testing.T) { t.Fatal(err) } defer c.Close() - c.SetLogger(ioutil.Discard) // Create a new client with additional tags. c1 := c.WithTags("foo", "bar") diff --git a/test/handler.go b/test/handler.go index ea44bef9a..0f8c1cefd 100644 --- a/test/handler.go +++ b/test/handler.go @@ -41,7 +41,6 @@ func NewHandler() *Handler { Handler: pilosa.NewHandler(), } h.Handler.Executor = &h.Executor - h.Handler.LogOutput = ioutil.Discard // Handler test messages can no-op. h.Broadcaster = pilosa.NopBroadcaster diff --git a/test/holder.go b/test/holder.go index 59402cea0..97b399bea 100644 --- a/test/holder.go +++ b/test/holder.go @@ -15,7 +15,6 @@ package test import ( - "bytes" "io/ioutil" "os" @@ -26,7 +25,6 @@ import ( // Holder is a test wrapper for pilosa.Holder. type Holder struct { *pilosa.Holder - LogOutput bytes.Buffer } // NewHolder returns a new instance of Holder with a temporary path. @@ -38,7 +36,6 @@ func NewHolder() *Holder { h := &Holder{Holder: pilosa.NewHolder()} h.Path = path - h.Holder.LogOutput = &h.LogOutput h.Holder.NewAttrStore = boltdb.NewAttrStore return h @@ -62,10 +59,10 @@ func (h *Holder) Close() error { // Reopen instantiates and opens a new holder. // Note that the holder must be Closed first. func (h *Holder) Reopen() error { - path, logOutput := h.Path, h.Holder.LogOutput + path, logger := h.Path, h.Holder.Logger h.Holder = pilosa.NewHolder() h.Holder.Path = path - h.Holder.LogOutput = logOutput + h.Holder.Logger = logger h.Holder.NewAttrStore = boltdb.NewAttrStore if err := h.Holder.Open(); err != nil { return err diff --git a/test/logger.go b/test/logger.go new file mode 100644 index 000000000..b4a0079b1 --- /dev/null +++ b/test/logger.go @@ -0,0 +1,48 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "fmt" + "io/ioutil" +) + +// BufferLogger represents a test Logger that holds log messages +// in a buffer for review. +type BufferLogger struct { + buf *bytes.Buffer +} + +// NewBufferLogger returns a new instance of BufferLogger. +func NewBufferLogger() *BufferLogger { + return &BufferLogger{ + buf: &bytes.Buffer{}, + } +} + +func (b *BufferLogger) Printf(format string, v ...interface{}) { + s := fmt.Sprintf(format, v...) + _, err := b.buf.WriteString(s) + if err != nil { + panic(err) + } +} + +func (b *BufferLogger) Debugf(format string, v ...interface{}) {} + +func (b *BufferLogger) ReadAll() ([]byte, error) { + return ioutil.ReadAll(b.buf) +} diff --git a/test/pilosa.go b/test/pilosa.go index 86623c239..b00c9841c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -181,7 +181,7 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( } // Open gossip transport to use in SetupServer. - transport, err := gossip.NewTransport(host, bindPort) + transport, err := gossip.NewTransport(host, bindPort, nil) if err != nil { return seed, err } diff --git a/view.go b/view.go index dfedebbe3..f775f1b0e 100644 --- a/view.go +++ b/view.go @@ -16,9 +16,6 @@ package pilosa import ( "fmt" - "io" - "io/ioutil" - "log" "os" "path/filepath" "strconv" @@ -64,7 +61,7 @@ type View struct { stats StatsClient RowAttrStore AttrStore - LogOutput io.Writer + Logger Logger } // NewView returns a new instance of View. @@ -81,7 +78,7 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View { broadcaster: NopBroadcaster, stats: NopStatsClient, - LogOutput: ioutil.Discard, + Logger: NopLogger, } } @@ -126,9 +123,6 @@ func (v *View) Open() error { return nil } -// logger returns a logger instance for the view. -func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) } - // openFragments opens and initializes the fragments inside the view. func (v *View) openFragments() error { file, err := os.Open(filepath.Join(v.path, "fragments")) @@ -275,7 +269,7 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { frag := NewFragment(path, v.index, v.frame, v.name, slice) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize - frag.LogOutput = v.LogOutput + frag.Logger = v.Logger frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) return frag } @@ -288,7 +282,7 @@ func (v *View) DeleteFragment(slice uint64) error { return ErrFragmentNotFound } - v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) // Close data files before deletion. if err := fragment.Close(); err != nil { @@ -302,7 +296,7 @@ func (v *View) DeleteFragment(slice uint64) error { // Delete fragment cache file. if err := os.Remove(fragment.CachePath()); err != nil { - v.logger().Printf("no cache file to delete for slice %d", slice) + v.Logger.Printf("no cache file to delete for slice %d", slice) } delete(v.fragments, slice) From d113ebe147e33552e4ace9d9f1bcf366ac291b7a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 15:34:15 -0500 Subject: [PATCH 05/21] Support --verbose logging --- config.go | 2 ++ ctl/server.go | 1 + gossip/gossip.go | 3 +++ logger.go | 24 ++++++++++++++++++++---- server/server.go | 6 +++++- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/config.go b/config.go index f2dd41793..84f199dc3 100644 --- a/config.go +++ b/config.go @@ -133,6 +133,7 @@ type Config struct { MaxWritesPerRequest int `toml:"max-writes-per-request"` LogPath string `toml:"log-path"` + Verbose bool `toml:"verbose"` // TLS TLS TLSConfig @@ -178,6 +179,7 @@ func NewConfig() *Config { Bind: ":" + DefaultPort, MaxWritesPerRequest: DefaultMaxWritesPerRequest, // LogPath: "", + // Verbose: false, TLS: TLSConfig{}, } diff --git a/ctl/server.go b/ctl/server.go index 87d6a8e15..d21b8e889 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -28,6 +28,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") + flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Verbose logging") // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) diff --git a/gossip/gossip.go b/gossip/gossip.go index 4a1e0f5f1..a27da24c8 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -147,8 +147,10 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } +// GossipMemberSetOption describes a functional option for GossipMemberSet. type GossipMemberSetOption func(*GossipMemberSet) error +// WithTransport is a functional option for providing a transport to NewGossipMemberSet. func WithTransport(transport *Transport) func(*GossipMemberSet) error { return func(g *GossipMemberSet) error { g.transport = transport @@ -156,6 +158,7 @@ func WithTransport(transport *Transport) func(*GossipMemberSet) error { } } +// WithLogger is a functional option for providing a logger to NewGossipMemberSet. func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { return func(g *GossipMemberSet) error { g.logger = logger diff --git a/logger.go b/logger.go index bf516d187..ab26b0883 100644 --- a/logger.go +++ b/logger.go @@ -58,12 +58,28 @@ func NewStandardLogger(logger *log.Logger) *StandardLogger { } } -// Printf is a no-op implementation of the Logger Printf method. func (s *StandardLogger) Printf(format string, v ...interface{}) { s.logger.Printf(format, v...) } -// Debugf is a no-op implementation of the Logger Debugf method. -func (s *StandardLogger) Debugf(format string, v ...interface{}) { - // TODO: implement this +func (s *StandardLogger) Debugf(format string, v ...interface{}) {} + +// VerboseLogger is an implementation of pilosa.Logger which includes debug messages. +type VerboseLogger struct { + logger *log.Logger +} + +func NewVerboseLogger(logger *log.Logger) *VerboseLogger { + return &VerboseLogger{ + logger: logger, + } +} + +func (vb *VerboseLogger) Printf(format string, v ...interface{}) { + vb.logger.Printf(format, v...) +} + +func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { + vb.logger.Printf("VERBOSE...") + vb.logger.Printf(format, v...) } diff --git a/server/server.go b/server/server.go index d45be21e2..8fe508fbd 100644 --- a/server/server.go +++ b/server/server.go @@ -136,7 +136,11 @@ func (m *Command) SetupServer() error { } m.Server.LogOutput = lw m.logger = log.New(lw, "", log.LstdFlags) - m.Server.Logger = pilosa.NewStandardLogger(m.logger) + if m.Config.Verbose { + m.Server.Logger = pilosa.NewVerboseLogger(m.logger) + } else { + m.Server.Logger = pilosa.NewStandardLogger(m.logger) + } m.Server.Handler.Logger = m.Server.Logger m.Server.Holder.Logger = m.Server.Logger m.Server.Holder.Stats.SetLogger(m.Server.Logger) From 0dc8aa0a5cd9fe406caa13e5c3f3a11aca42cd2b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 15:40:28 -0500 Subject: [PATCH 06/21] add --verbose flag to docs --- ctl/server.go | 2 +- docs/configuration.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ctl/server.go b/ctl/server.go index d21b8e889..1c60a7c9c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -28,7 +28,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") - flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Verbose logging") + flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging") // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) diff --git a/docs/configuration.md b/docs/configuration.md index 633e83f3c..4d71dee8e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,6 +81,17 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h log-path = "/path/to/logfile" ``` +#### Verbose + +* Description: Enable verbose logging. +* Flag: `--verbose` +* Env: `PILOSA_VERBOSE` +* Config: + + ```toml + verbose = true + ``` + #### Max Writes Per Request * Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue. From 633d99b21791a1891bba7a2ad7f3f004fa3b1612 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 15:47:51 -0500 Subject: [PATCH 07/21] remove leftover `logger` from NopLogger implementation --- logger.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/logger.go b/logger.go index ab26b0883..2d3df732b 100644 --- a/logger.go +++ b/logger.go @@ -14,10 +14,7 @@ package pilosa -import ( - "io/ioutil" - "log" -) +import "log" // Ensure nopLogger implements interface. var _ Logger = &nopLogger{} @@ -29,17 +26,13 @@ type Logger interface { } func init() { - NopLogger = &nopLogger{ - logger: log.New(ioutil.Discard, "", log.LstdFlags), - } + NopLogger = &nopLogger{} } // NopLogger represents a Logger that doesn't do anything. var NopLogger Logger -type nopLogger struct { - logger *log.Logger -} +type nopLogger struct{} // Printf is a no-op implementation of the Logger Printf method. func (n *nopLogger) Printf(format string, v ...interface{}) {} From fd8040e69920ecdad5d47e88bbf3ccfaf5792b91 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 22 Mar 2018 16:24:15 -0500 Subject: [PATCH 08/21] remove LogOutput and instead close any logger that implements io.Closer --- server.go | 8 ++------ server/server.go | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/server.go b/server.go index 11b2ee2a3..08923f13a 100644 --- a/server.go +++ b/server.go @@ -19,8 +19,6 @@ import ( "crypto/tls" "errors" "fmt" - "io" - "io/ioutil" "net" "net/http" "os" @@ -87,8 +85,7 @@ type Server struct { // Misc options. MaxWritesPerRequest int - LogOutput io.Writer - Logger Logger + Logger Logger defaultClient InternalClient } @@ -115,8 +112,7 @@ func NewServer() *Server { MetricInterval: 0, DiagnosticInterval: 0, - LogOutput: ioutil.Discard, - Logger: NopLogger, + Logger: NopLogger, } s.Handler.Holder = s.Holder diff --git a/server/server.go b/server/server.go index 8fe508fbd..2dcfd12f4 100644 --- a/server/server.go +++ b/server/server.go @@ -134,7 +134,6 @@ func (m *Command) SetupServer() error { if err != nil { return err } - m.Server.LogOutput = lw m.logger = log.New(lw, "", log.LstdFlags) if m.Config.Verbose { m.Server.Logger = pilosa.NewVerboseLogger(m.logger) @@ -299,8 +298,8 @@ func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) { func (m *Command) Close() error { var logErr error serveErr := m.Server.Close() - logOutput := m.Server.LogOutput - if closer, ok := logOutput.(io.Closer); ok { + logger := m.Server.Logger + if closer, ok := logger.(io.Closer); ok { logErr = closer.Close() } close(m.Done) From 36ce12da599a18209a23c8956d31564a2f7741e1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 23 Mar 2018 12:43:48 -0500 Subject: [PATCH 09/21] This commit adds a `Close()` method to the `pilosa.Logger` interface, and it moves the file handling (open/close) out of the main Command and into the interface implementation. The Logger implementations both have a `Logger()` method which returns their internal logger (`*log.Logger`). This is because the gossip setup (memberlist) needs a `*log.Logger` for its configuration. --- cmd/server.go | 9 +++--- logger.go | 77 +++++++++++++++++++++++++++++++++++++++++++----- server/server.go | 53 +++++++++++++++------------------ test/logger.go | 2 ++ 4 files changed, 98 insertions(+), 43 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 8089dae54..81633ea01 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -17,7 +17,6 @@ package cmd import ( "fmt" "io" - "log" "os" "os/signal" "runtime/pprof" @@ -46,11 +45,11 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr) - if err != nil { - return err + // Set up the logger. + if err := Server.SetupLogger(); err != nil { + return fmt.Errorf("error setting up the logger: %v", err) } - logger := log.New(logOutput, "", log.LstdFlags) + logger := Server.Server.Logger logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) // Start CPU profiling. diff --git a/logger.go b/logger.go index 2d3df732b..445bc5212 100644 --- a/logger.go +++ b/logger.go @@ -14,7 +14,11 @@ package pilosa -import "log" +import ( + "io" + "log" + "os" +) // Ensure nopLogger implements interface. var _ Logger = &nopLogger{} @@ -23,6 +27,7 @@ var _ Logger = &nopLogger{} type Logger interface { Printf(format string, v ...interface{}) Debugf(format string, v ...interface{}) + Close() error } func init() { @@ -40,15 +45,34 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {} // Debugf is a no-op implementation of the Logger Debugf method. func (n *nopLogger) Debugf(format string, v ...interface{}) {} +// Close is a no-op implementation of the Logger Close method. +func (n *nopLogger) Close() error { return nil } + // StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. type StandardLogger struct { logger *log.Logger + f *os.File } -func NewStandardLogger(logger *log.Logger) *StandardLogger { - return &StandardLogger{ - logger: logger, +func NewStandardLogger(path string, defaultWriter io.Writer) (*StandardLogger, error) { + var lw io.Writer + var err error + var f *os.File + + if path == "" { + lw = defaultWriter + } else { + f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + lw = f } + + return &StandardLogger{ + logger: log.New(lw, "", log.LstdFlags), + f: f, + }, nil } func (s *StandardLogger) Printf(format string, v ...interface{}) { @@ -57,15 +81,42 @@ func (s *StandardLogger) Printf(format string, v ...interface{}) { func (s *StandardLogger) Debugf(format string, v ...interface{}) {} +func (s *StandardLogger) Close() error { + if s.f == nil { + return nil + } + return s.f.Close() +} + +func (s *StandardLogger) Logger() *log.Logger { + return s.logger +} + // VerboseLogger is an implementation of pilosa.Logger which includes debug messages. type VerboseLogger struct { logger *log.Logger + f *os.File } -func NewVerboseLogger(logger *log.Logger) *VerboseLogger { - return &VerboseLogger{ - logger: logger, +func NewVerboseLogger(path string, defaultWriter io.Writer) (*VerboseLogger, error) { + var lw io.Writer + var err error + var f *os.File + + if path == "" { + lw = defaultWriter + } else { + f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + lw = f } + + return &VerboseLogger{ + logger: log.New(lw, "", log.LstdFlags), + f: f, + }, nil } func (vb *VerboseLogger) Printf(format string, v ...interface{}) { @@ -73,6 +124,16 @@ func (vb *VerboseLogger) Printf(format string, v ...interface{}) { } func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { - vb.logger.Printf("VERBOSE...") vb.logger.Printf(format, v...) } + +func (vb *VerboseLogger) Close() error { + if vb.f == nil { + return nil + } + return vb.f.Close() +} + +func (vb *VerboseLogger) Logger() *log.Logger { + return vb.logger +} diff --git a/server/server.go b/server/server.go index 2dcfd12f4..4141ae3aa 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,7 @@ type Command struct { // Done will be closed when Command.Close() is called Done chan struct{} + // Passed to the Gossip implementation. logger *log.Logger } @@ -122,6 +123,26 @@ func (m *Command) Run(args ...string) (err error) { return nil } +// SetupLogger sets up the logger based on the configuration. +func (m *Command) SetupLogger() error { + if m.Config.Verbose { + vbl, err := pilosa.NewVerboseLogger(m.Config.LogPath, m.Stderr) + if err != nil { + return err + } + m.logger = vbl.Logger() + m.Server.Logger = vbl + } else { + sl, err := pilosa.NewStandardLogger(m.Config.LogPath, m.Stderr) + if err != nil { + return err + } + m.logger = sl.Logger() + m.Server.Logger = sl + } + return nil +} + // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { err := m.Config.Validate() @@ -129,17 +150,6 @@ func (m *Command) SetupServer() error { return err } - // Set up logger based on configuration. - lw, err := GetLogWriter(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } - m.logger = log.New(lw, "", log.LstdFlags) - if m.Config.Verbose { - m.Server.Logger = pilosa.NewVerboseLogger(m.logger) - } else { - m.Server.Logger = pilosa.NewStandardLogger(m.logger) - } m.Server.Handler.Logger = m.Server.Logger m.Server.Holder.Logger = m.Server.Logger m.Server.Holder.Stats.SetLogger(m.Server.Logger) @@ -280,28 +290,11 @@ func (m *Command) SetupNetworking() error { return nil } -// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path. -func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) { - // This is split out so it can be used in NewServeCmd as well as SetupServer - if path == "" { - return defaultWriter, nil - } else { - logFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - return logFile, nil - } -} - // Close shuts down the server. func (m *Command) Close() error { - var logErr error serveErr := m.Server.Close() - logger := m.Server.Logger - if closer, ok := logger.(io.Closer); ok { - logErr = closer.Close() - } + logErr := m.Server.Logger.Close() + close(m.Done) if serveErr != nil && logErr != nil { return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr) diff --git a/test/logger.go b/test/logger.go index b4a0079b1..ae5984bb7 100644 --- a/test/logger.go +++ b/test/logger.go @@ -43,6 +43,8 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) { func (b *BufferLogger) Debugf(format string, v ...interface{}) {} +func (b *BufferLogger) Close() error { return nil } + func (b *BufferLogger) ReadAll() ([]byte, error) { return ioutil.ReadAll(b.buf) } From 5c52e48b1264de0ba776b8d874889e5954eb2c19 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 27 Mar 2018 10:23:08 -0500 Subject: [PATCH 10/21] change NewStandardLogger() to only take an io.Writer --- logger.go | 63 +++++------------------------------------------- server/server.go | 25 ++++++++++++------- test/logger.go | 2 -- 3 files changed, 22 insertions(+), 68 deletions(-) diff --git a/logger.go b/logger.go index 445bc5212..93b12c3f8 100644 --- a/logger.go +++ b/logger.go @@ -17,7 +17,6 @@ package pilosa import ( "io" "log" - "os" ) // Ensure nopLogger implements interface. @@ -27,7 +26,6 @@ var _ Logger = &nopLogger{} type Logger interface { Printf(format string, v ...interface{}) Debugf(format string, v ...interface{}) - Close() error } func init() { @@ -45,34 +43,15 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {} // Debugf is a no-op implementation of the Logger Debugf method. func (n *nopLogger) Debugf(format string, v ...interface{}) {} -// Close is a no-op implementation of the Logger Close method. -func (n *nopLogger) Close() error { return nil } - // StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. type StandardLogger struct { logger *log.Logger - f *os.File } -func NewStandardLogger(path string, defaultWriter io.Writer) (*StandardLogger, error) { - var lw io.Writer - var err error - var f *os.File - - if path == "" { - lw = defaultWriter - } else { - f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - lw = f - } - +func NewStandardLogger(w io.Writer) *StandardLogger { return &StandardLogger{ - logger: log.New(lw, "", log.LstdFlags), - f: f, - }, nil + logger: log.New(w, "", log.LstdFlags), + } } func (s *StandardLogger) Printf(format string, v ...interface{}) { @@ -81,13 +60,6 @@ func (s *StandardLogger) Printf(format string, v ...interface{}) { func (s *StandardLogger) Debugf(format string, v ...interface{}) {} -func (s *StandardLogger) Close() error { - if s.f == nil { - return nil - } - return s.f.Close() -} - func (s *StandardLogger) Logger() *log.Logger { return s.logger } @@ -95,28 +67,12 @@ func (s *StandardLogger) Logger() *log.Logger { // VerboseLogger is an implementation of pilosa.Logger which includes debug messages. type VerboseLogger struct { logger *log.Logger - f *os.File } -func NewVerboseLogger(path string, defaultWriter io.Writer) (*VerboseLogger, error) { - var lw io.Writer - var err error - var f *os.File - - if path == "" { - lw = defaultWriter - } else { - f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - lw = f - } - +func NewVerboseLogger(w io.Writer) *VerboseLogger { return &VerboseLogger{ - logger: log.New(lw, "", log.LstdFlags), - f: f, - }, nil + logger: log.New(w, "", log.LstdFlags), + } } func (vb *VerboseLogger) Printf(format string, v ...interface{}) { @@ -127,13 +83,6 @@ func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { vb.logger.Printf(format, v...) } -func (vb *VerboseLogger) Close() error { - if vb.f == nil { - return nil - } - return vb.f.Close() -} - func (vb *VerboseLogger) Logger() *log.Logger { return vb.logger } diff --git a/server/server.go b/server/server.go index 4141ae3aa..66725f4ee 100644 --- a/server/server.go +++ b/server/server.go @@ -74,7 +74,8 @@ type Command struct { Done chan struct{} // Passed to the Gossip implementation. - logger *log.Logger + logOutput io.Writer + logger *log.Logger } // NewCommand returns a new instance of Main. @@ -125,18 +126,22 @@ func (m *Command) Run(args ...string) (err error) { // SetupLogger sets up the logger based on the configuration. func (m *Command) SetupLogger() error { - if m.Config.Verbose { - vbl, err := pilosa.NewVerboseLogger(m.Config.LogPath, m.Stderr) + var err error + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { return err } + } + + if m.Config.Verbose { + vbl := pilosa.NewVerboseLogger(m.logOutput) m.logger = vbl.Logger() m.Server.Logger = vbl } else { - sl, err := pilosa.NewStandardLogger(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } + sl := pilosa.NewStandardLogger(m.logOutput) m.logger = sl.Logger() m.Server.Logger = sl } @@ -292,9 +297,11 @@ func (m *Command) SetupNetworking() error { // Close shuts down the server. func (m *Command) Close() error { + var logErr error serveErr := m.Server.Close() - logErr := m.Server.Logger.Close() - + if closer, ok := m.logOutput.(io.Closer); ok { + logErr = closer.Close() + } close(m.Done) if serveErr != nil && logErr != nil { return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr) diff --git a/test/logger.go b/test/logger.go index ae5984bb7..b4a0079b1 100644 --- a/test/logger.go +++ b/test/logger.go @@ -43,8 +43,6 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) { func (b *BufferLogger) Debugf(format string, v ...interface{}) {} -func (b *BufferLogger) Close() error { return nil } - func (b *BufferLogger) ReadAll() ([]byte, error) { return ioutil.ReadAll(b.buf) } From 96e2ef534ff4e88e929562b9ed33560af46128c0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Mar 2018 14:08:01 -0500 Subject: [PATCH 11/21] Clean up Makefile Turn off the automatic installation of build-time dependencies, which was confusing and inconsistent. Update PHONY list. Simplify git status check. Remove PKGS as Go 1.9+ automatically ignores `vendor` when running tests. Consolidate release/prerelease build code. Add comments. --- Makefile | 160 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 9eb2df7e6..ddd6cb1a5 100644 --- a/Makefile +++ b/Makefile @@ -1,113 +1,133 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test -DEP := $(shell command -v dep 2>/dev/null) -STATIK := $(shell command -v statik 2>/dev/null) -PROTOC := $(shell command -v protoc 2>/dev/null) -VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -STATUS := $(shell git status --porcelain) -IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa -PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) -BUILD_TIME=`date -u +%FT%T%z` +VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) +BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) +BUILD_TIME := $(shell date -u +%FT%T%z) LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" -DOCKER_GOLANG_IMAGE=golang:latest -GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH)) -BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH) +GO_VERSION=latest -default: test pilosa +# Run tests and compile Pilosa +default: test build +# Remove vendor and build directories clean: rm -rf vendor build -$(GOPATH)/bin: - mkdir $(GOPATH)/bin - -dep: $(GOPATH)/bin - go get -u github.com/golang/dep/cmd/dep - +# Set up vendor directory using `dep` vendor: Gopkg.toml -ifndef DEP - make dep -endif + $(MAKE) require-dep dep ensure touch vendor -Gopkg.lock: dep Gopkg.toml - dep ensure - +# Run test suite test: vendor - go test $(PKGS) $(TESTFLAGS) + go test ./... $(TESTFLAGS) +# Run test suite with coverage enabled cover: vendor - make test TESTFLAGS="-coverprofile=build/coverage.out" + $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" +# Run test suite with coverage enabled and view coverage results in browser cover-viz: cover go tool cover -html=build/coverage.out -pilosa: vendor - go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Compile Pilosa +build: vendor + go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Create a single release build under the build directory release-build: vendor -ifdef DOCKER_BUILD - make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -else - make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -endif - cp LICENSE README.md build/pilosa-$(IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ - @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" + $(MAKE) $(if DOCKER_BUILD,docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + cp LICENSE README.md build/pilosa-$(VERSION_ID) + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz -release: -ifeq ($(STATUS),"") - make release-build GOOS=darwin GOARCH=amd64 - make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 - make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 -else - @echo "Will not create release with unclean git status." -endif +# Error out if there are untracked changes in Git +check-clean: + $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) +# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. +release: check-clean + $(MAKE) release-build GOOS=darwin GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + $(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 + +# Create branch-tagged pre-release for client library CI jobs prerelease-build: vendor - make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa" - cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/ - @echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz" + $(MAKE) release-build VERSION_ID=$(BRANCH_ID) +# Create prerelease build for Linux/amd64 prerelease: - make prerelease-build GOOS=linux GOARCH=amd64 + $(MAKE) prerelease-build GOOS=linux GOARCH=amd64 +# Upload prerelease to S3 prerelease-upload: prerelease - aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read + aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read +# Install Pilosa install: vendor - go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -.protoc-gen-gofast: vendor -ifndef PROTOC - $(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases") -endif - go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast - cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast - -generate-protoc: .protoc-gen-gofast +# `go generate` protocol buffers +generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -generate-statik: statik +# `go generate` statik assets (WebUI) +generate-statik: require-statik go generate github.com/pilosa/pilosa/statik +# `go generate` all needed packages generate: generate-protoc generate-statik -statik: -ifndef STATIK - go get github.com/rakyll/statik -endif - +# Create Docker image from Dockerfile docker: docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . - @echo "Created image: pilosa:$(VERSION)" + @echo Created docker image: pilosa:$(VERSION) +# Compile Pilosa inside Docker container docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS) + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./... + +###################### +# Build dependencies # +###################### + +# Verifies that needed build dependency is installed. Errors out if not installed. +define require + $(if $(shell command -v $1 2>/dev/null), + $(info Verified build dependency "$1" is installed.), + $(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`)) +endef + +require-dep: + $(call require,dep) + +require-statik: + $(call require,statik) + +require-protoc-gen-gofast: + $(call require,protoc-gen-gofast) + +require-protoc: + $(call require,protoc) + +install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc + +install-dep: + go get -u github.com/golang/dep/cmd/dep + +install-statik: + go get -u github.com/rakyll/statik + +install-protoc-gen-gofast: + go get -u github.com/gogo/protobuf/protoc-gen-gofast + +install-protoc: + @echo This tool cannot automatically install protoc. Please download and install protoc from https://github.com/google/protobuf/releases From e601d694c0e4bd7f1508471d00581815a6472379 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Mar 2018 15:02:12 -0500 Subject: [PATCH 12/21] Document Makefile --- CONTRIBUTING.md | 66 ++++++++++++++++++++++++++++++++++++++++++++ docs/installation.md | 8 ++++++ 2 files changed, 74 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 402895f1d..0a29cb1b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,6 +66,72 @@ If you want to help but you aren't sure where to start, check out our [github la git remote add upstream git@github.com:pilosa/pilosa.git ``` +### Makefile + +Pilosa includes a Makefile that automates several tasks: + +- Install Pilosa: + + ```sh + make install + ``` + +- Create the vendor directory: + + ```sh + make vendor + ``` + +- Run the test suite: + + ```sh + make test + ``` + +- View the coverage report: + + ```sh + make cover-viz + ``` + +- Clear the `vendor/` and `build/` directories: + + ```sh + make clean + ``` + +- Create release tarballs: + + ```sh + make release + ``` + +- Generate static assets for the WebUI: + + ```sh + make generate-statik + ``` + +- Regenerate protocol buffer files in `internal/`: + + ```sh + make generate-protoc + ``` + +- Create tagged Docker image: + + ```sh + make docker + ``` + +- Run tests inside Docker container: + + ```sh + make docker-test + ``` + +Additional commands are available in the `Makefile`. + ### Submitting code changes - Before starting to work on a task, sync your branch with the upstream: diff --git a/docs/installation.md b/docs/installation.md index 9ca90ccf2..225e742ae 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -134,6 +134,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). @@ -288,6 +292,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). From 743726331fb893ffb7e23f68ba21195d91163253 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 28 Mar 2018 12:22:47 -0500 Subject: [PATCH 13/21] Add make instructions for travis-ci build process and use Makefile in Docker build --- .dockerignore | 1 - .travis.yml | 2 +- Dockerfile | 9 ++++----- Makefile | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9abb766ce..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 1a4691b8c..715f81a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" install: - - make vendor generate-statik + - make install-dep install-statik vendor generate-statik script: - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 diff --git a/Dockerfile b/Dockerfile index 50e88985c..145405bc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ FROM golang:1.10 as builder -ARG ldflags='' - -COPY . /go/src/github.com/pilosa/pilosa +COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && make vendor \ - && CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa + && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" + +RUN chmod +x /go/bin/pilosa FROM scratch diff --git a/Makefile b/Makefile index ddd6cb1a5..6094495e3 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ generate: generate-protoc generate-statik # Create Docker image from Dockerfile docker: - docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . + docker build -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) # Compile Pilosa inside Docker container From 9bf70705a3582e6b7c02af9c4dd3a77aa4c588b7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 28 Mar 2018 12:23:03 -0500 Subject: [PATCH 14/21] Add docs for "make install-build-deps" --- CONTRIBUTING.md | 6 ++++++ docs/installation.md | 2 ++ 2 files changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a29cb1b5..925f789b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,12 @@ Pilosa includes a Makefile that automates several tasks: make install ``` +- Install build dependencies (dep, statik, and protoc): + + ```sh + make install-build-deps + ``` + - Create the vendor directory: ```sh diff --git a/docs/installation.md b/docs/installation.md index 225e742ae..db0828b2f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -152,6 +152,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` @@ -310,6 +311,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` From d59d20d27bc86dff711e1a33134c005782aac9fb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 29 Mar 2018 10:34:34 -0500 Subject: [PATCH 15/21] Redundant chmod --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 145405bc4..bf076e526 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,6 @@ COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" -RUN chmod +x /go/bin/pilosa - FROM scratch LABEL maintainer "dev@pilosa.com" From 14dd54cc735c9d061d3cac2df008ac76b87b1787 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 29 Mar 2018 15:50:53 -0500 Subject: [PATCH 16/21] WebUI fixes to accommodate API changes --- webui/assets/main.js | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index 6a4415a5c..d38da454c 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -254,12 +254,12 @@ function populate_version() { xhr.onload = function() { var version = JSON.parse(xhr.responseText)['version'] - var version_major_minor = /(v\d+\.\d+)/.exec(version)[0] + var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1] var doc_link = document.getElementById('nav-documentation') doc_link.onclick = function() { - window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/') + window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/') } - node.innerHTML = version + node.innerHTML = "Pilosa v" + version } xhr.send(null) } @@ -295,12 +295,19 @@ function set_active_pane_by_name(name) { function update_cluster_status() { var xhr = new XMLHttpRequest(); xhr.open('GET', '/status') - status_node = document.getElementById('status') xhr.onload = function() { var status = JSON.parse(xhr.responseText) render_status(status) } xhr.send(null) + + var xhrSchema = new XMLHttpRequest(); + xhrSchema.open('GET', '/schema') + xhrSchema.onload = function() { + var schema = JSON.parse(xhrSchema.responseText) + render_schema(schema) + } + xhrSchema.send(null) } function render_status(status) { @@ -310,7 +317,7 @@ function render_status(status) { nodes_div.removeChild(nodes_div.firstChild); } - var nodes = status["status"]["Nodes"] + var nodes = status["nodes"] table = document.createElement("table") tbody = document.createElement("tbody") table.appendChild(tbody) @@ -320,31 +327,35 @@ function render_status(status) { var header = document.createElement('tr') markup = `Host - State` + ID + Coordinator` header.innerHTML = markup tbody.appendChild(header) for(var n=0; n${nodes[n]["Host"]} - ${nodes[n]["State"]}` + markup = `${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]} + ${nodes[n]["id"]} + ${nodes[n]["isCoordinator"]}` row.innerHTML = markup tbody.appendChild(row) } nodes_div.appendChild(table) +} +function render_schema(schema) { // render index tables var indexes_div = document.getElementById("status-indexes") while (indexes_div.firstChild) { indexes_div.removeChild(indexes_div.firstChild); } - var indexes = nodes[0]["Indexes"] // TODO currently comes from only node 0 + var indexes = schema["indexes"] // TODO currently comes from only node 0 for(var n=0; n${frames[m]["Name"]} - ${frames[m]["Meta"]["CacheType"]} - ${frames[m]["Meta"]["CacheSize"]}` + row.innerHTML = `${frames[m]["name"]} + ${frames[m]["options"]["cacheType"]} + ${frames[m]["options"]["cacheSize"]}` tbody.appendChild(row) } } @@ -583,4 +594,4 @@ function parse_options(option_str) { } } return options; -} \ No newline at end of file +} From e3cd03d902174524de1082cff72c609b9342b8f6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 2 Apr 2018 13:05:37 -0500 Subject: [PATCH 17/21] move rowLabel and columnLabel constants to executor.go --- executor.go | 65 ++++++++++++++++++++++++++++------------------------- frame.go | 1 - index.go | 1 - 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/executor.go b/executor.go index 0ed6dbaed..2154b60ff 100644 --- a/executor.go +++ b/executor.go @@ -33,6 +33,9 @@ const ( // MinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. MinThreshold = 1 + + columnLabel = "col" + rowLabel = "row" ) // Executor recursively executes calls in a PQL query across all slices. @@ -128,7 +131,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(DefaultRowLabel, DefaultColumnLabel) { + if call.IsInverse(rowLabel, columnLabel) { slices = inverseSlices } } @@ -262,7 +265,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { idx := e.Holder.Index(index) if idx != nil { - if columnID, ok, err := c.UintArg(DefaultColumnLabel); ok && err == nil { + if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err @@ -273,7 +276,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { frame, _ := c.Args["frame"].(string) if fr := idx.Frame(frame); fr != nil { - rowID, _, err := c.UintArg(DefaultRowLabel) + rowID, _, err := c.UintArg(rowLabel) if err != nil { return nil, err } @@ -529,15 +532,15 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. } // Return an error if both the row and column label are specified. - rowID, rowOK, rowErr := c.UintArg(DefaultRowLabel) - columnID, columnOK, columnErr := c.UintArg(DefaultColumnLabel) + rowID, rowOK, rowErr := c.UintArg(rowLabel) + columnID, columnOK, columnErr := c.UintArg(columnLabel) if rowErr != nil || columnErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) } if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", DefaultRowLabel, DefaultColumnLabel) + return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", DefaultRowLabel, DefaultColumnLabel) + return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel) } // Determine row or column orientation. @@ -604,11 +607,11 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Read row & column id. - columnID, columnOK, err := c.UintArg(DefaultColumnLabel) + columnID, columnOK, err := c.UintArg(columnLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) } - rowID, rowOK, err := c.UintArg(DefaultRowLabel) + rowID, rowOK, err := c.UintArg(rowLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } @@ -617,9 +620,9 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C var id uint64 var viewName string if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", DefaultColumnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel) } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", DefaultColumnLabel, DefaultRowLabel) + return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel) } else if columnOK { viewName, id = ViewInverse, columnID } else { @@ -893,18 +896,18 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal } // Read fields using labels. - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit() row field '%v' required", DefaultRowLabel) + return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) } - colID, ok, err := c.UintArg(DefaultColumnLabel) + colID, ok, err := c.UintArg(columnLabel) if err != nil { return false, fmt.Errorf("reading ClearBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit col field '%v' required", DefaultColumnLabel) + return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } // Clear bits for each view. @@ -983,18 +986,18 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, } // Read fields using labels. - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { return false, fmt.Errorf("reading SetBit() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() row field '%v' required", DefaultRowLabel) + return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) } - colID, ok, err := c.UintArg(DefaultColumnLabel) + colID, ok, err := c.UintArg(columnLabel) if err != nil { return false, fmt.Errorf("reading SetBit() column: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() column field '%v' required", DefaultColumnLabel) + return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) } var timestamp *time.Time @@ -1080,11 +1083,11 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Parse labels. - columnID, ok, err := c.UintArg(DefaultColumnLabel) + columnID, ok, err := c.UintArg(columnLabel) if err != nil { return fmt.Errorf("reading SetFieldValue() column: %v", err) } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", DefaultColumnLabel) + return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) } // Copy args and remove reserved fields. @@ -1092,7 +1095,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq delete(args, "frame") // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. - delete(args, DefaultColumnLabel) + delete(args, columnLabel) // Set values. for name, value := range args { @@ -1146,17 +1149,17 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Parse labels. - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetRowAttrs() row field '%v' required", DefaultRowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required", rowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, DefaultRowLabel) + delete(attrs, rowLabel) // Set attributes. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { @@ -1205,17 +1208,17 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal return nil, ErrFrameNotFound } - rowID, ok, err := c.UintArg(DefaultRowLabel) + rowID, ok, err := c.UintArg(rowLabel) if err != nil { - return nil, fmt.Errorf("reading SetRowAttrs() row: %v", DefaultRowLabel) + return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { - return nil, fmt.Errorf("SetRowAttrs row field '%v' required", DefaultRowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, DefaultRowLabel) + delete(attrs, rowLabel) // Create frame group, if not exists. frameMap := m[frame] @@ -1284,14 +1287,14 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return ErrIndexNotFound } - col, okCol, errCol := c.UintArg(DefaultColumnLabel) + col, okCol, errCol := c.UintArg(columnLabel) if errCol != nil || !okCol { return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, DefaultColumnLabel) + delete(attrs, columnLabel) delete(attrs, "frame") // Set attributes. diff --git a/frame.go b/frame.go index f786ac0b0..b5968a5d3 100644 --- a/frame.go +++ b/frame.go @@ -32,7 +32,6 @@ import ( // Default frame settings. const ( - DefaultRowLabel = "row" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false DefaultRangeEnabled = false diff --git a/index.go b/index.go index f17854e1e..0f0cad456 100644 --- a/index.go +++ b/index.go @@ -31,7 +31,6 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "col" InputDefinitionDir = ".input-definitions" ) From ba9c26e6895a3fe05a331dcbef47bc64a41a18ff Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 2 Apr 2018 16:39:14 -0500 Subject: [PATCH 18/21] Code review fixes: Create build directory if it doesn't exist. Use better URL for protoc installation instructions. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6094495e3..824d23cad 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ test: vendor # Run test suite with coverage enabled cover: vendor + mkdir -p build $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" # Run test suite with coverage enabled and view coverage results in browser @@ -130,4 +131,4 @@ install-protoc-gen-gofast: go get -u github.com/gogo/protobuf/protoc-gen-gofast install-protoc: - @echo This tool cannot automatically install protoc. Please download and install protoc from https://github.com/google/protobuf/releases + @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html From ce4fb3c4d347eb66bc0682ca27592e73c2708c87 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 3 Apr 2018 10:36:48 -0500 Subject: [PATCH 19/21] remove references to rowLabel and columnLabel --- docs/api-reference.md | 4 ++-- docs/query-language.md | 6 +----- handler_internal_test.go | 4 ++-- pilosa.go | 11 ----------- pilosa_test.go | 24 ------------------------ webui/assets/main.js | 4 ++-- webui/index.html | 4 ++-- 7 files changed, 9 insertions(+), 48 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 9092c92bc..446235cdc 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -215,7 +215,7 @@ The request payload is JSON, and it must contain the fields `frames` and `fields * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa. +The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa. * `name` (string): Maps the source data field to actions that process the field's corresponding value. * `actions` (array): List of actions that will process the field's value. @@ -310,7 +310,7 @@ Input definition is deprecated as of v0.9. Processes the JSON payload using the given input definition. -The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. +The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition. ``` request curl localhost:10101/index/user/input/stargazer-input \ diff --git a/docs/query-language.md b/docs/query-language.md index 0ccca3a80..6b0584320 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -21,8 +21,6 @@ This section will provide a detailed reference and examples for the Pilosa Query There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type. -The default row label is `rowID`, and the default column label is `columnID`. Changing these defaults is deprecated and this feature will be removed in a future release. - #### Conventions * Angle Brackets `<>` denote required arguments @@ -46,8 +44,6 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types * `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. -* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated. -* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated. * `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` @@ -223,7 +219,7 @@ Bitmap(, ( | =UINT)) **Description:** -`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. +`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column. **Result Type:** object with attrs and bits. diff --git a/handler_internal_test.go b/handler_internal_test.go index 395ec73b6..4603662e6 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -30,7 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"columnLabel": "test"}}`, err: "Unknown key: columnLabel:test"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, } for _, test := range tests { actual := &postIndexRequest{} @@ -65,7 +65,7 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"rowLabel": "test"}}`, err: "Unknown key: rowLabel:test"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, diff --git a/pilosa.go b/pilosa.go index a3e11b826..8f1ae287e 100644 --- a/pilosa.go +++ b/pilosa.go @@ -77,9 +77,6 @@ var ( // Regular expression to validate index and frame names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) -// Regular expression to validate row and column labels. -var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`) - // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { @@ -141,14 +138,6 @@ func ValidateName(name string) error { return nil } -// ValidateLabel ensures that the label is a valid format. -func ValidateLabel(label string) error { - if labelRegexp.Match([]byte(label)) == false { - return ErrLabel - } - return nil -} - // StringInSlice checks for substring a in the slice. func StringInSlice(a string, list []string) bool { for _, b := range list { diff --git a/pilosa_test.go b/pilosa_test.go index d82e27fa3..41b0d7098 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -46,30 +46,6 @@ func TestValidateNameInvalid(t *testing.T) { } } -func TestValidateLabel(t *testing.T) { - labels := []string{ - "a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, label := range labels { - if pilosa.ValidateLabel(label) != nil { - t.Fatalf("Should be valid label: %s", label) - } - } -} - -func TestValidateLabelInvalid(t *testing.T) { - labels := []string{ - "", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, label := range labels { - if pilosa.ValidateLabel(label) == nil { - t.Fatalf("Should be invalid label: %s", label) - } - } -} - func TestStringInSlice(t *testing.T) { list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} substr := "localhost:10101" diff --git a/webui/assets/main.js b/webui/assets/main.js index 6a4415a5c..c0a24bdf6 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -472,7 +472,7 @@ class Autocompleter { } init_dynamic_keywords() { - // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list + // hit /schema, parse indexes, frames, add to list } add_keyword() { @@ -583,4 +583,4 @@ function parse_options(option_str) { } } return options; -} \ No newline at end of file +} diff --git a/webui/index.html b/webui/index.html index dc06e310a..75eaa1d5a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -55,9 +55,9 @@
Special commands
- :create index test [columnLabel=column]
+ :create index test
:use test
- :create frame foo [rowLabel=row]
+ :create frame foo
:delete index test
:delete frame foo
From 4b0da099b58e8aaf03aa6f0a793a33171c9c14c0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 3 Apr 2018 11:09:49 -0500 Subject: [PATCH 20/21] Fix syntax on conditional check for docker-build config --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 824d23cad..9e2c6fa3f 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ build: vendor # Create a single release build under the build directory release-build: vendor - $(MAKE) $(if DOCKER_BUILD,docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" cp LICENSE README.md build/pilosa-$(VERSION_ID) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz From 0bf5f089a0e14677f4b3806f1d8f114c526ca76b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 3 Apr 2018 11:25:54 -0500 Subject: [PATCH 21/21] Run prerelease on Go 1.10 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 715f81a6e..15c511e78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ deploy: skip_cleanup: true on: all_branches: true - go: 1.9 + go: "1.10" matrix: allow_failures: - go: master