From 836b467d3d1464a0fff5d41c425242700c29a110 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 29 Mar 2019 17:39:30 -0500 Subject: [PATCH 01/73] add support to modify shard width at build time use "make SHARD_WIDTH=nn" fix tests to run and pass at different shardwidths add shardwidth22 test to circle ci --- .circleci/config.yml | 6 ++++++ Makefile | 4 +++- cluster_internal_test.go | 8 ++++---- executor_test.go | 10 +++++----- fragment.go | 5 +++-- fragment_internal_test.go | 14 +++++++------- roaring/roaring_test.go | 6 +++--- server/cluster_test.go | 6 +++--- server/handler_test.go | 12 +++++++----- shardwidth20.go | 5 +++++ shardwidth22.go | 5 +++++ 11 files changed, 51 insertions(+), 30 deletions(-) create mode 100644 shardwidth20.go create mode 100644 shardwidth22.go diff --git a/.circleci/config.yml b/.circleci/config.yml index c696d5691..aa6da2fda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,12 @@ jobs: - *fast-checkout - run: sudo apt-get install lsof - run: make test + test-golang-1.12-shard22: &base-test + <<: *defaults + steps: + - *fast-checkout + - run: sudo apt-get install lsof + - run: make test SHARD_WIDTH=22 test-golang-1.12-race: <<: *defaults steps: diff --git a/Makefile b/Makefile index 45b1df78c..456456a7b 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) +SHARD_WIDTH = 20 LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(if $(ENTERPRISE_ENABLED),1)" GO_VERSION=latest ENTERPRISE ?= 0 @@ -14,6 +15,7 @@ RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) +BUILD_TAGS += shardwidth$(SHARD_WIDTH) export GO111MODULE=on # Run tests and compile Pilosa @@ -29,7 +31,7 @@ vendor: go.mod # Run test suite test: - go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) + go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) bench: go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) diff --git a/cluster_internal_test.go b/cluster_internal_test.go index a6cf437b2..fa40c48b7 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -157,15 +157,15 @@ func TestFragSources(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 1300000, nil) + _, err = field.SetBit(1, ShardWidth+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 2600000, nil) + _, err = field.SetBit(1, ShardWidth*2+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 3900000, nil) + _, err = field.SetBit(1, ShardWidth*3+1, nil) if err != nil { t.Fatal(err) } @@ -755,7 +755,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Fatal(err) } tc.SetBit("i", "f", 1, 101, nil) - tc.SetBit("i", "f", 1, 1300000, nil) + tc.SetBit("i", "f", 1, ShardWidth+1, nil) // Before starting the resize, get the CheckSum to use for // comparison later. diff --git a/executor_test.go b/executor_test.go index a36ddcf72..2dc96223e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2226,11 +2226,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Remote SetBit", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil { - t.Fatalf("quuerying remote: %v", err) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil { + t.Fatalf("querying remote: %v", err) } - if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) { + if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{pilosa.ShardWidth + 1}) { t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns()) } }) @@ -2241,11 +2241,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil { t.Fatalf("quuerying remote: %v", err) } - if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) { + if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{pilosa.ShardWidth + 1}) { t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns()) } }) diff --git a/fragment.go b/fragment.go index 4769ce700..1fdce2ee5 100644 --- a/fragment.go +++ b/fragment.go @@ -48,8 +48,9 @@ import ( const ( // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. - shardWidthExponent = 20 - ShardWidth = 1 << shardWidthExponent + // shardWidthExponent = 20 // set in shardwidthNN.go files + + ShardWidth = 1 << shardWidthExponent // shardVsContainerExponent is the power of 2 of ShardWith minus the power // of two of roaring container width (which is 16). diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 5186d6da1..3930e98e2 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -137,14 +137,14 @@ func TestFragment_SetRow(t *testing.T) { rowID := uint64(1000) // Set bits on the fragment. - if _, err := f.setBit(rowID, 8000001); err != nil { + if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(rowID, 8065536); err != nil { + } else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil { t.Fatal(err) } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) { + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) { t.Fatalf("unexpected columns: %+v", cols) } // Verify count on row. @@ -153,7 +153,7 @@ func TestFragment_SetRow(t *testing.T) { } // Set row (overwrite existing data). - row := NewRow(8000002, 8065537, 8131074) + row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000) if changed, err := f.unprotectedSetRow(row, rowID); err != nil { t.Fatal(err) } else if !changed { @@ -161,7 +161,7 @@ func TestFragment_SetRow(t *testing.T) { } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) { + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) { t.Fatalf("unexpected columns after set row: %+v", cols) } // Verify count on row. @@ -1914,7 +1914,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(b) // Generate some intersecting data. - maxX := 1048576 / 2 + maxX := ShardWidth / 2 sz := maxX rows := make([]uint64, sz) cols := make([]uint64, sz) @@ -1950,7 +1950,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { b.StopTimer() - maxX := 1048576 * 5 * 2 + maxX := ShardWidth * 5 * 2 sz := maxX rows := make([]uint64, sz) cols := make([]uint64, sz) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 9d0b3c8f8..ebf88a212 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -299,10 +299,10 @@ func TestBitmap_Max(t *testing.T) { // Ensure CountRange is correct even if rangekey is prior to initial container. func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { - s := uint64(2009 * 1048576) - e := uint64(2010 * 1048576) + s := uint64(2009 * pilosa.ShardWidth) + e := uint64(2010 * pilosa.ShardWidth) - start := s + (39314024 % 1048576) + start := s + (39314024 % pilosa.ShardWidth) bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i++ { if (i+1)%4096 == 0 { diff --git a/server/cluster_test.go b/server/cluster_test.go index 617bf6cad..b284e49df 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -85,10 +85,10 @@ func TestMain_SendReceiveMessage(t *testing.T) { } // Write data on first node. - if _, err := m0.Query("i", "", ` + if _, err := m0.Query("i", "", fmt.Sprintf(` Set(1, f=1) - Set(2400000, f=1) - `); err != nil { + Set(%d, f=1) + `, 2*pilosa.ShardWidth+1)); err != nil { t.Fatal(err) } diff --git a/server/handler_test.go b/server/handler_test.go index 701d019ae..2a6b9b8c2 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "io" "io/ioutil" gohttp "net/http" @@ -109,8 +110,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - target := `{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]} -` + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} +`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } @@ -294,7 +295,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" { + } else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" { t.Fatalf("unexpected body: %s", body) } }) @@ -311,10 +312,11 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ColumnAttrs_JSON", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)"))) + exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n" if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) + } else if body := w.Body.String(); body != exp { + t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp) } }) diff --git a/shardwidth20.go b/shardwidth20.go new file mode 100644 index 000000000..fd99f0a84 --- /dev/null +++ b/shardwidth20.go @@ -0,0 +1,5 @@ +// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32 + +package pilosa + +const shardWidthExponent = 20 diff --git a/shardwidth22.go b/shardwidth22.go new file mode 100644 index 000000000..57727a9a5 --- /dev/null +++ b/shardwidth22.go @@ -0,0 +1,5 @@ +// +build shardwidth22 + +package pilosa + +const shardWidthExponent = 22 From 811f1b41242453facb8dc1e9f173b452f42856b5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 4 Apr 2019 14:27:52 -0500 Subject: [PATCH 02/73] move build-tagged shardwidth files to subpackage --- executor.go | 3 ++- fragment.go | 5 +++-- shardwidth/16.go | 5 +++++ shardwidth/17.go | 5 +++++ shardwidth/18.go | 5 +++++ shardwidth/19.go | 5 +++++ shardwidth20.go => shardwidth/20.go | 4 ++-- shardwidth/21.go | 5 +++++ shardwidth/22.go | 5 +++++ shardwidth/23.go | 5 +++++ shardwidth/24.go | 5 +++++ shardwidth/25.go | 5 +++++ shardwidth/26.go | 5 +++++ shardwidth/27.go | 5 +++++ shardwidth/28.go | 5 +++++ shardwidth/29.go | 5 +++++ shardwidth/30.go | 5 +++++ shardwidth/31.go | 5 +++++ shardwidth/32.go | 5 +++++ shardwidth22.go | 5 ----- 20 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 shardwidth/16.go create mode 100644 shardwidth/17.go create mode 100644 shardwidth/18.go create mode 100644 shardwidth/19.go rename shardwidth20.go => shardwidth/20.go (83%) create mode 100644 shardwidth/21.go create mode 100644 shardwidth/22.go create mode 100644 shardwidth/23.go create mode 100644 shardwidth/24.go create mode 100644 shardwidth/25.go create mode 100644 shardwidth/26.go create mode 100644 shardwidth/27.go create mode 100644 shardwidth/28.go create mode 100644 shardwidth/29.go create mode 100644 shardwidth/30.go create mode 100644 shardwidth/31.go create mode 100644 shardwidth/32.go delete mode 100644 shardwidth22.go diff --git a/executor.go b/executor.go index b345d434b..89c6d8228 100644 --- a/executor.go +++ b/executor.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/shardwidth" "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -1234,7 +1235,7 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - colShard := columnID >> shardWidthExponent + colShard := columnID >> shardwidth.Exponent if colShard != shard { return rowIDs, nil } diff --git a/fragment.go b/fragment.go index 1fdce2ee5..82288c914 100644 --- a/fragment.go +++ b/fragment.go @@ -40,6 +40,7 @@ import ( "github.com/pilosa/pilosa/logger" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" + "github.com/pilosa/pilosa/shardwidth" "github.com/pilosa/pilosa/stats" "github.com/pilosa/pilosa/syswrap" "github.com/pilosa/pilosa/tracing" @@ -50,7 +51,7 @@ const ( // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. // shardWidthExponent = 20 // set in shardwidthNN.go files - ShardWidth = 1 << shardWidthExponent + ShardWidth = 1 << shardwidth.Exponent // shardVsContainerExponent is the power of 2 of ShardWith minus the power // of two of roaring container width (which is 16). @@ -60,7 +61,7 @@ const ( // which a given container is in means dividing by the number of rows per // container which is performantly expressed as a right shift by this // exponent. - shardVsContainerExponent = shardWidthExponent - 16 + shardVsContainerExponent = shardwidth.Exponent - 16 // width of roaring containers is 2^16 containerWidth = 1 << 16 diff --git a/shardwidth/16.go b/shardwidth/16.go new file mode 100644 index 000000000..f9e90c047 --- /dev/null +++ b/shardwidth/16.go @@ -0,0 +1,5 @@ +// +build shardwidth16 + +package shardwidth + +const Exponent = 16 diff --git a/shardwidth/17.go b/shardwidth/17.go new file mode 100644 index 000000000..65653456e --- /dev/null +++ b/shardwidth/17.go @@ -0,0 +1,5 @@ +// +build shardwidth17 + +package shardwidth + +const Exponent = 17 diff --git a/shardwidth/18.go b/shardwidth/18.go new file mode 100644 index 000000000..1357a3c17 --- /dev/null +++ b/shardwidth/18.go @@ -0,0 +1,5 @@ +// +build shardwidth18 + +package shardwidth + +const Exponent = 18 diff --git a/shardwidth/19.go b/shardwidth/19.go new file mode 100644 index 000000000..a3333540c --- /dev/null +++ b/shardwidth/19.go @@ -0,0 +1,5 @@ +// +build shardwidth19 + +package shardwidth + +const Exponent = 19 diff --git a/shardwidth20.go b/shardwidth/20.go similarity index 83% rename from shardwidth20.go rename to shardwidth/20.go index fd99f0a84..e45c5a472 100644 --- a/shardwidth20.go +++ b/shardwidth/20.go @@ -1,5 +1,5 @@ // +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32 -package pilosa +package shardwidth -const shardWidthExponent = 20 +const Exponent = 20 diff --git a/shardwidth/21.go b/shardwidth/21.go new file mode 100644 index 000000000..edaf0a010 --- /dev/null +++ b/shardwidth/21.go @@ -0,0 +1,5 @@ +// +build shardwidth21 + +package shardwidth + +const Exponent = 21 diff --git a/shardwidth/22.go b/shardwidth/22.go new file mode 100644 index 000000000..ab75e51da --- /dev/null +++ b/shardwidth/22.go @@ -0,0 +1,5 @@ +// +build shardwidth22 + +package shardwidth + +const Exponent = 22 diff --git a/shardwidth/23.go b/shardwidth/23.go new file mode 100644 index 000000000..2535a2dee --- /dev/null +++ b/shardwidth/23.go @@ -0,0 +1,5 @@ +// +build shardwidth23 + +package shardwidth + +const Exponent = 23 diff --git a/shardwidth/24.go b/shardwidth/24.go new file mode 100644 index 000000000..b1275afd8 --- /dev/null +++ b/shardwidth/24.go @@ -0,0 +1,5 @@ +// +build shardwidth24 + +package shardwidth + +const Exponent = 24 diff --git a/shardwidth/25.go b/shardwidth/25.go new file mode 100644 index 000000000..9599ba5a3 --- /dev/null +++ b/shardwidth/25.go @@ -0,0 +1,5 @@ +// +build shardwidth25 + +package shardwidth + +const Exponent = 25 diff --git a/shardwidth/26.go b/shardwidth/26.go new file mode 100644 index 000000000..e7dfdd13b --- /dev/null +++ b/shardwidth/26.go @@ -0,0 +1,5 @@ +// +build shardwidth26 + +package shardwidth + +const Exponent = 26 diff --git a/shardwidth/27.go b/shardwidth/27.go new file mode 100644 index 000000000..568a798f8 --- /dev/null +++ b/shardwidth/27.go @@ -0,0 +1,5 @@ +// +build shardwidth27 + +package shardwidth + +const Exponent = 27 diff --git a/shardwidth/28.go b/shardwidth/28.go new file mode 100644 index 000000000..bb43cd5a3 --- /dev/null +++ b/shardwidth/28.go @@ -0,0 +1,5 @@ +// +build shardwidth28 + +package shardwidth + +const Exponent = 28 diff --git a/shardwidth/29.go b/shardwidth/29.go new file mode 100644 index 000000000..f797b632d --- /dev/null +++ b/shardwidth/29.go @@ -0,0 +1,5 @@ +// +build shardwidth29 + +package shardwidth + +const Exponent = 29 diff --git a/shardwidth/30.go b/shardwidth/30.go new file mode 100644 index 000000000..aac761c82 --- /dev/null +++ b/shardwidth/30.go @@ -0,0 +1,5 @@ +// +build shardwidth30 + +package shardwidth + +const Exponent = 30 diff --git a/shardwidth/31.go b/shardwidth/31.go new file mode 100644 index 000000000..12f85bfcf --- /dev/null +++ b/shardwidth/31.go @@ -0,0 +1,5 @@ +// +build shardwidth31 + +package shardwidth + +const Exponent = 31 diff --git a/shardwidth/32.go b/shardwidth/32.go new file mode 100644 index 000000000..76dde6b04 --- /dev/null +++ b/shardwidth/32.go @@ -0,0 +1,5 @@ +// +build shardwidth32 + +package shardwidth + +const Exponent = 32 diff --git a/shardwidth22.go b/shardwidth22.go deleted file mode 100644 index 57727a9a5..000000000 --- a/shardwidth22.go +++ /dev/null @@ -1,5 +0,0 @@ -// +build shardwidth22 - -package pilosa - -const shardWidthExponent = 22 From b95f739b07d2ddeca055b2919590e21dd7ed5527 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 15:07:07 +0300 Subject: [PATCH 03/73] Updated import and client libraries docs --- docs/administration.md | 9 +- docs/client-libraries.md | 282 +-------------------------------------- 2 files changed, 12 insertions(+), 279 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 4bfa5470a..021bd5535 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,6 +56,11 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` +Official Pilosa client libraries support importing data. You can find the corresponding documentation at: +* [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) +* [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) +* [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) + ##### Importing Integer Values If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`. @@ -283,13 +288,13 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **NumViews:** Number of views in the Cluster. - **OpenFiles:** Open file handle count. - **GoRoutines:** Go routine count. - + You can opt-out of the Pilosa diagnostics reporting by setting the command line configuration option `--metric.diagnostics=false`, the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. ### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. -The metrics configuration options are: +The metrics configuration options are: - [Host](../configuration/#metric-host): specify host that receives metric events - [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics diff --git a/docs/client-libraries.md b/docs/client-libraries.md index ac008def4..9492f7693 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -10,281 +10,9 @@ nav = [ ## Client Libraries -This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is best to avoid using the output of a hash or randomly distributed ids with Pilosa. +We have the following official client libraries. You can find more information in their repositories: +* [Go client repository](https://github.com/pilosa/go-pilosa) +* [Java client repository](https://github.com/pilosa/java-pilosa) +* [Python client repository](https://github.com/pilosa/python-pilosa) -### Go - -You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```go -package main - -import ( - "fmt" - - "github.com/pilosa/go-pilosa" -) - -func main() { - // We will just use the default client which assumes the server is at http://localhost:10101 - client := pilosa.DefaultClient() - - // Let's load the schema from the server. - // Note that, for this example the schema should be created beforehand - // and the stargazer data should be imported. - // See the Getting Started repository: https://github.com/pilosa/getting-started/ - schema, err := client.Schema() - if err != nil { - // Most calls will return an error value. - // You should handle them appropriately. - // We will just terminate the program in this case. - // Error handling was left out for brevity in the rest of the code. - panic(err) - } - - // We need to refer to indexes and fields before we can use them in a query. - repository := schema.Index("repository") - stargazer := repository.Field("stargazer") - language := repository.Field("language") - - var response *pilosa.QueryResponse - - // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Row(14)) - fmt.Println("User 14 starred: ", response.Result().Row().Columns) - - // What are the top 5 languages in the sample data? - response, err = client.Query(language.TopN(5)) - languageIDs := []uint64{} - for _, item := range response.Result().CountItems() { - languageIDs = append(languageIDs, item.ID) - } - fmt.Println("Top 5 languages: ", languageIDs) - - // Which repositories were starred by both user 14 and 19: - response, _ = client.Query( - repository.Intersect( - stargazer.Row(14), - stargazer.Row(19))) - fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns) - - // Which repositories were starred by user 14 or 19: - response, _ = client.Query( - repository.Union( - stargazer.Row(14), - stargazer.Row(19))) - fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns) - - // Which repositories were starred by user 14 or 19 and were written in language 1: - response, _ = client.Query( - repository.Intersect( - repository.Union( - stargazer.Row(14), - stargazer.Row(19), - ), - language.Row(1))) - fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns) - - // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.Set(99999, 77777)) -} -``` - -Running the above program should produce output like this: -``` -User 14 starred: [1 2 3 362 368 391 396 409 416 430 436 450 454 460 461 464 466 469 470 483 484 486 490 491 503 504 514] -Top 5 languages: [5 1 4 9 13] -Both user 14 and 19 starred: [2 3 362 396 416 461 464 466 470 486] -User 14 or 19 starred: [1 2 3 361 362 368 376 377 378 382 386 388 391 396 398 400 409 411 412 416 426 428 430 435 436 450 452 453 454 456 460 461 464 465 466 469 470 483 484 486 487 489 490 491 500 503 504 505 512 514] -User 14 or 19 starred, written in language 1: [1 2 362 368 382 386 416 426 435 456 461 483 500 503 504 514] -``` - -### Python - -You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```python -from __future__ import print_function -from pilosa import Index, Client, PilosaError, TimeQuantum - -# We will just use the default client which assumes the server is at http://localhost:10101 -client = Client() - -# Let's load the schema from the server. -# Note that, for this example the schema should be created beforehand -# and the stargazer data should be imported. -# See the Getting Started repository: https://github.com/pilosa/getting-started/ - -# Let's create Index and Field objects, which will contain the settings -# for the corresponding indexes and fields. -try: - schema = client.schema() -except PilosaError as e: - # Most calls will raise an exception on errors. - # You should handle them appropriately. - # We will just terminate the program in this case. - raise SystemExit(e) - -# We need to refer to indexes and fields before we can use them in a query. -repository = schema.index("repository") -stargazer = repository.field("stargazer") -language = repository.field("language") - -# Which repositories did user 8 star: -repository_ids = client.query(stargazer.row(14)).result.row.columns -print("User 8 starred: ", repository_ids) - -# What are the top 5 languages in the sample data: -top_languages = client.query(language.topn(5)).result.count_items -print("Top 5 languages: ", [item.id for item in top_languages]) - -# Which repositories were starred by both user 14 and 19: -query = repository.intersect( - stargazer.row(14), - stargazer.row(19) -) -mutually_starred = client.query(query).result.row.columns -print("Both user 14 and 19 starred:", mutually_starred) - -# Which repositories were starred by user 14 or 19: -query = repository.union( - stargazer.row(14), - stargazer.row(19) -) -either_starred = client.query(query).result.row.columns -print("User 14 or 19 starred:", either_starred) - -# Which repositories were starred by user 14 or 19 and were written in language 1: -query = repository.intersect( - repository.union( - stargazer.row(14), - stargazer.row(19) - ), - language.row(1) -) -mutually_starred = client.query(query).result.row.columns -print("User 14 or 19 starred, written in language 1:", mutually_starred) - -# Set user 99999 as a stargazer for repository 77777 -client.query(stargazer.set(99999, 77777)) -``` - -Running the above program should produce output like this: -``` -('User 8 starred: ', [1L, 2L, 3L, 362L, 368L, 391L, 396L, 409L, 416L, 430L, 436L, 450L, 454L, 460L, 461L, 464L, 466L, 469L, 470L, 483L, 484L, 486L, 490L, 491L, 503L, 504L, 514L]) -('Top 5 languages: ', [5L, 1L, 4L, 9L, 13L]) -('Both user 14 and 19 starred:', [2L, 3L, 362L, 396L, 416L, 461L, 464L, 466L, 470L, 486L]) -('User 14 or 19 starred:', [1L, 2L, 3L, 361L, 362L, 368L, 376L, 377L, 378L, 382L, 386L, 388L, 391L, 396L, 398L, 400L, 409L, 411L, 412L, 416L, 426L, 428L, 430L, 435L, 436L, 450L, 452L, 453L, 454L, 456L, 460L, 461L, 464L, 465L, 466L, 469L, 470L, 483L, 484L, 486L, 487L, 489L, 490L, 491L, 500L, 503L, 504L, 505L, 512L, 514L]) -('User 14 or 19 starred, written in language 1:', [1L, 2L, 362L, 368L, 382L, 386L, 416L, 426L, 435L, 456L, 461L, 483L, 500L, 503L, 504L, 514L]) -``` - -### Java - -You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```java -import com.pilosa.client.*; -import com.pilosa.client.orm.*; -import com.pilosa.client.exceptions.PilosaException; - -import java.util.ArrayList; -import java.util.List; - -public class StarTrace { - public static void main(String[] args) { - // We will just use the default client which assumes the server is at http://localhost:10101 - PilosaClient client = PilosaClient.defaultClient(); - - // Let's load the schema from the server. - Schema schema; - try { - schema = client.readSchema(); - } - catch (PilosaException ex) { - // Most calls will return an error value. - // You should handle them appropriately. - // We will just terminate the program in this case. - throw new RuntimeException(ex); - } - - // We need to refer to indexes and fields before we can use them in a query. - Index repository = schema.index("repository"); - Field stargazer = repository.field("stargazer"); - Field language = repository.field("language"); - - QueryResponse response; - QueryResult result; - PqlQuery query; - List repositoryIDs; - - // Which repositories did user 14 star: - response = client.query(stargazer.row(14)); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 starred: " + repositoryIDs); - - // What are the top 5 languages in the sample data: - response = client.query(language.topN(5)); - List top_languages = response.getResult().getCountItems(); - List languageIDs = new ArrayList(); - for (CountResultItem item : top_languages) { - languageIDs.add(item.getID()); - } - - System.out.println("Top Languages: " +languageIDs); - - // Which repositories were starred by both user 14 and 19: - query = repository.intersect( - stargazer.row(14), - stargazer.row(19) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("Both user 14 and 19 starred: " + repositoryIDs); - - // Which repositories were starred by user 14 or 19: - query = repository.union( - stargazer.row(14), - stargazer.row(19) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 or 19 starred: " + repositoryIDs); - - // Which repositories were starred by user 14 or 19 and were written in language 1: - query = repository.intersect( - repository.union( - stargazer.row(14), - stargazer.row(19) - ), - language.row(1) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); - - // Set user 99999 as a stargazer for repository 77777: - client.query(stargazer.set(99999, 77777)); - } -} -``` - -Running the above program should produce output like this: -``` -User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514] -Top Languages: [5, 1, 4, 9, 13] -Both user 14 and 19 starred: [2, 3, 362, 396, 416, 461, 464, 466, 470, 486] -User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514] -User 14 or 19 starred, written in language 1: [1, 2, 362, 368, 382, 386, 416, 426, 435, 456, 461, 483, 500, 503, 504, 514] -``` +Check out our [Getting Started](https://github.com/pilosa/getting-started) repository for sample code for the official clients. From 73cc49770cba74b6c81a35798176e7f2cfafcac4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:17:55 +0300 Subject: [PATCH 04/73] suggest client libraries for import --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 021bd5535..42c47e517 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,7 +56,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -Official Pilosa client libraries support importing data. You can find the corresponding documentation at: +We suggest importing the data using official Pilosa client libraries. You can find the corresponding documentation at: * [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) * [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) * [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) From 2fd62872223512650c856655cc6407cf6f2dcbf5 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:20:02 +0300 Subject: [PATCH 05/73] recommend --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 42c47e517..23929c267 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,7 +56,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -We suggest importing the data using official Pilosa client libraries. You can find the corresponding documentation at: +We recommend importing the data using official Pilosa client libraries. You can find the corresponding documentation at: * [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) * [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) * [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) From 1e7208d60ff5b1c4f8a23044f99419d66969d35e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:25:21 +0300 Subject: [PATCH 06/73] the --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 23929c267..33fa73104 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,7 +56,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -We recommend importing the data using official Pilosa client libraries. You can find the corresponding documentation at: +We recommend importing data using official Pilosa client libraries. You can find the corresponding documentation at: * [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) * [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) * [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) From 5f079163cbf2c5570f580f5d7eaa18ff69c7fde2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 10 Apr 2019 16:30:40 -0500 Subject: [PATCH 07/73] validate (and panic) on duplicate PQL arguments --- pql/ast.go | 18 +++++++++++++++- pql/pqlpeg_test.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/pql/ast.go b/pql/ast.go index d78ad2828..36d6e35fb 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -103,7 +103,9 @@ func (q *Query) endConditional() { func (q *Query) addField(field string) { elem := q.lastCallStackElem() - if elem == nil || elem.lastField != "" { + if elem == nil { + panic(fmt.Sprintf("addField called with '%s' while element is nil", field)) + } else if elem.lastField != "" { panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField)) } elem.lastField = field @@ -112,6 +114,15 @@ func (q *Query) addField(field string) { } } +// validateArgField ensures that field does not already +// exist as a key in the Args map before adding the new +// key/value. +func (q *Query) validateArgField(elem *callStackElem) { + if _, exists := elem.call.Args[elem.lastField]; exists { + panic(fmt.Sprintf("multiple instances of argument '%s' provided", elem.lastField)) + } +} + func (q *Query) addVal(val interface{}) { elem := q.lastCallStackElem() if elem == nil || elem.lastField == "" { @@ -123,11 +134,13 @@ func (q *Query) addVal(val interface{}) { return } if elem.lastCond != ILLEGAL { + q.validateArgField(elem) // case 1 elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, Value: val, } } else { + q.validateArgField(elem) // case 2 elem.call.Args[elem.lastField] = val } elem.lastField = "" @@ -162,11 +175,13 @@ func (q *Query) addNumVal(val string) { } return } else if elem.lastCond != ILLEGAL { + q.validateArgField(elem) // case 3 elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, Value: ival, } } else { + q.validateArgField(elem) // case 4 elem.call.Args[elem.lastField] = ival } elem.lastField = "" @@ -175,6 +190,7 @@ func (q *Query) addNumVal(val string) { func (q *Query) startList() { elem := q.lastCallStackElem() + q.validateArgField(elem) // case 5 if elem.lastCond != ILLEGAL { elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 8492d3531..f8a6597b5 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -676,3 +676,55 @@ func TestPQLDeepEquality(t *testing.T) { }) } } + +func TestPQLPanic(t *testing.T) { + tests := []struct { + name string + call string + }{ + // case 1 + { + name: "StringConditional", + call: "Row(a==foo, a==bar)", + }, + // case 2 + { + name: "StringValue", + call: "Row(a=foo, a=bar)", + }, + // case 3 + { + name: "IntConditional", + call: "Row(a>5, a>6)", + }, + // case 4 + { + name: "IntValue", + call: "Row(a=7, a=8)", + }, + // case 5 + { + name: "List", + call: "Row(a=[7], a=[7,8])", + }, + } + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + var v interface{} + func() { + defer func() { v = recover() }() + + q, err := ParseString(test.call) + if err != nil { + t.Fatalf("parsing query '%s': %v", test.call, err) + } + _ = q + }() + + if !reflect.DeepEqual(v, "multiple instances of argument 'a' provided") { + t.Fatalf("unexpected panic value: %#v", v) + } + + }) + } +} From bdc4f3b07eb7bb4b1eee3a5c148e58fc640e1509 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 10 Apr 2019 23:48:17 -0500 Subject: [PATCH 08/73] recover the duplicate arg panic from parser, treat as error --- pql/ast.go | 2 +- pql/parser.go | 20 +++++++++++++++++++- pql/pqlpeg_test.go | 37 ++++++++++++++++++++++--------------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 36d6e35fb..20b757946 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -119,7 +119,7 @@ func (q *Query) addField(field string) { // key/value. func (q *Query) validateArgField(elem *callStackElem) { if _, exists := elem.call.Args[elem.lastField]; exists { - panic(fmt.Sprintf("multiple instances of argument '%s' provided", elem.lastField)) + panic(fmt.Sprintf("%s: %s", duplicateArgErrorMessage, elem.lastField)) } } diff --git a/pql/parser.go b/pql/parser.go index 611294971..6a28f560e 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,6 +15,7 @@ package pql import ( + "fmt" "io" "io/ioutil" "strings" @@ -25,6 +26,9 @@ import ( // timeFormat is the go-style time format used to parse string dates. const timeFormat = "2006-01-02T15:04" +// duplicateArgErrorMessage is used as an error string in the parser. +const duplicateArgErrorMessage = "duplicate argument provided" + // parser represents a parser for the PQL language. type parser struct { r io.Reader @@ -59,6 +63,20 @@ func (p *parser) Parse() (*Query, error) { if err != nil { return nil, errors.Wrap(err, "parsing") } - p.Execute() + + // Handle specific panics from the parser and return them as errors. + var v interface{} + func() { + defer func() { v = recover() }() + p.Execute() + }() + if v != nil { + if strings.HasPrefix(v.(string), duplicateArgErrorMessage) { + return nil, fmt.Errorf("%s", v) + } else { + panic(v) + } + } + return &p.Query, nil } diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index f8a6597b5..a90d58a5d 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -1,6 +1,21 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import ( + "fmt" "reflect" "strconv" "testing" @@ -677,7 +692,7 @@ func TestPQLDeepEquality(t *testing.T) { } } -func TestPQLPanic(t *testing.T) { +func TestDuplicateArgError(t *testing.T) { tests := []struct { name string call string @@ -710,21 +725,13 @@ func TestPQLPanic(t *testing.T) { } for i, test := range tests { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { - var v interface{} - func() { - defer func() { v = recover() }() - - q, err := ParseString(test.call) - if err != nil { - t.Fatalf("parsing query '%s': %v", test.call, err) - } - _ = q - }() - - if !reflect.DeepEqual(v, "multiple instances of argument 'a' provided") { - t.Fatalf("unexpected panic value: %#v", v) + _, err := ParseString(test.call) + expErr := fmt.Sprintf("%s: a", duplicateArgErrorMessage) + if err == nil { + t.Fatalf("expected error for duplicate argument: %s", test.call) + } else if err.Error() != expErr { + t.Fatalf("expected error: %s, but got: %v", expErr, err.Error()) } - }) } } From fdbfc68f7cbffad933c7cb7ace67d434cf8a926e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 12 Apr 2019 11:29:02 -0500 Subject: [PATCH 09/73] Add license headers to files missing them and CI check to verify they are present. Fixes #1633 --- .circleci/config.yml | 11 +++++++++++ Makefile | 7 +++++++ client.go | 14 ++++++++++++++ enterprise/enterprise.go | 14 ++++++++++++++ executor_internal_test.go | 14 ++++++++++++++ handler.go | 14 ++++++++++++++ http/translator.go | 14 ++++++++++++++ http/translator_test.go | 14 ++++++++++++++ inmem/translator.go | 14 ++++++++++++++ inmem/translator_test.go | 14 ++++++++++++++ internal/clustertests/cluster_test.go | 14 ++++++++++++++ mock/mock.go | 14 ++++++++++++++ mock/translator.go | 14 ++++++++++++++ roaring/container_stash.go | 14 ++++++++++++++ roaring/containers.go | 2 +- roaring/containers_btree.go | 2 +- roaring/containers_test.go | 2 +- roaring/inst.go | 14 ++++++++++++++ roaring/nop_inst.go | 14 ++++++++++++++ roaring/roaring_nop_paranoia.go | 14 ++++++++++++++ roaring/roaring_nop_stats.go | 14 ++++++++++++++ roaring/roaring_paranoia.go | 14 ++++++++++++++ roaring/roaring_stats.go | 14 ++++++++++++++ server/enterprise.go | 2 +- server/setup_logger.go | 14 ++++++++++++++ server/setup_logger_arm64.go | 14 ++++++++++++++ syswrap/mmap.go | 14 ++++++++++++++ syswrap/os.go | 14 ++++++++++++++ toml/toml.go | 14 ++++++++++++++ tracing/opentracing/opentracing.go | 14 ++++++++++++++ tracing/tracing.go | 14 ++++++++++++++ translate.go | 14 ++++++++++++++ translate_test.go | 14 ++++++++++++++ utils_internal_test.go | 2 +- 34 files changed, 401 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aa6da2fda..677f1a57c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,11 @@ jobs: - *fast-checkout - run: make install-gometalinter - run: make gometalinter + check-license-headers: + <<: *defaults + steps: + - *fast-checkout + - run: make check-license-headers test-build-arm: <<: *defaults steps: @@ -123,6 +128,9 @@ workflows: - linter: requires: - setup + - check-license-headers: + requires: + - setup - test-build-arm: requires: - setup @@ -144,10 +152,12 @@ workflows: - prerelease: requires: - linter + - check-license-headers - test-golang-1.12 - release: requires: - linter + - check-license-headers - test-golang-1.12 filters: tags: @@ -160,4 +170,5 @@ workflows: - dockerhub-upload: requires: - linter + - check-license-headers - test-golang-1.12 diff --git a/Makefile b/Makefile index 456456a7b..f3f1ab608 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) +LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ") export GO111MODULE=on # Run tests and compile Pilosa @@ -153,6 +154,12 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... +# Verify that all Go files have license header +check-license-headers: + @! find . -name '*.go' | grep -v '^./vendor' | while read fn;\ + do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \ + grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree + ###################### # Build dependencies # ###################### diff --git a/client.go b/client.go index 3762a27b9..3d09c2e08 100644 --- a/client.go +++ b/client.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go index a75223801..2075f7a4d 100644 --- a/enterprise/enterprise.go +++ b/enterprise/enterprise.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright (c) 2018 Pilosa Corp. All rights reserved. // // This file is part of Pilosa Enterprise Edition. diff --git a/executor_internal_test.go b/executor_internal_test.go index 4177968d5..ffb091d49 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/handler.go b/handler.go index 5a452ce96..9089fd778 100644 --- a/handler.go +++ b/handler.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/http/translator.go b/http/translator.go index 3a2acae04..251715ede 100644 --- a/http/translator.go +++ b/http/translator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package http import ( diff --git a/http/translator_test.go b/http/translator_test.go index ce96b3c2c..bec2089a5 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package http_test import ( diff --git a/inmem/translator.go b/inmem/translator.go index 4e1e96b55..c5f976ea2 100644 --- a/inmem/translator.go +++ b/inmem/translator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package inmem import ( diff --git a/inmem/translator_test.go b/inmem/translator_test.go index d4d232566..0a7c6eb34 100644 --- a/inmem/translator_test.go +++ b/inmem/translator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package inmem_test import ( diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index f01835c10..612189c1b 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clustertest import ( diff --git a/mock/mock.go b/mock/mock.go index 97ebf8641..60fdfd200 100644 --- a/mock/mock.go +++ b/mock/mock.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package mock import "sync" diff --git a/mock/translator.go b/mock/translator.go index 7a63d8cf0..ca5ef1e8e 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package mock import ( diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 9d3f82302..728730d64 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring import ( diff --git a/roaring/containers.go b/roaring/containers.go index cc9a09ec5..dc5d4118d 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index fb29a6d08..1d08f598c 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 9c8bedd22..ad95f2f79 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/roaring/inst.go b/roaring/inst.go index c0a5771d9..f9eaf2831 100644 --- a/roaring/inst.go +++ b/roaring/inst.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build btreeInstrumentation package roaring diff --git a/roaring/nop_inst.go b/roaring/nop_inst.go index 7a945cb6d..b5cd94458 100644 --- a/roaring/nop_inst.go +++ b/roaring/nop_inst.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !btreeInstrumentation package roaring diff --git a/roaring/roaring_nop_paranoia.go b/roaring/roaring_nop_paranoia.go index ce9af8d54..6f2fa354b 100644 --- a/roaring/roaring_nop_paranoia.go +++ b/roaring/roaring_nop_paranoia.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !roaringparanoia package roaring diff --git a/roaring/roaring_nop_stats.go b/roaring/roaring_nop_stats.go index c9e029ab7..2b1506953 100644 --- a/roaring/roaring_nop_stats.go +++ b/roaring/roaring_nop_stats.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !roaringstats package roaring diff --git a/roaring/roaring_paranoia.go b/roaring/roaring_paranoia.go index 910e1bd40..6f7d0c57e 100644 --- a/roaring/roaring_paranoia.go +++ b/roaring/roaring_paranoia.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build roaringparanoia package roaring diff --git a/roaring/roaring_stats.go b/roaring/roaring_stats.go index fd8ade91f..046c0f087 100644 --- a/roaring/roaring_stats.go +++ b/roaring/roaring_stats.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build roaringstats package roaring diff --git a/server/enterprise.go b/server/enterprise.go index 6dd3e5540..021c00e7c 100644 --- a/server/enterprise.go +++ b/server/enterprise.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/server/setup_logger.go b/server/setup_logger.go index 06d61eced..3473faa43 100644 --- a/server/setup_logger.go +++ b/server/setup_logger.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !arm64 package server diff --git a/server/setup_logger_arm64.go b/server/setup_logger_arm64.go index dd6718741..090ac1395 100644 --- a/server/setup_logger_arm64.go +++ b/server/setup_logger_arm64.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server import ( diff --git a/syswrap/mmap.go b/syswrap/mmap.go index a13dff8af..95e819999 100644 --- a/syswrap/mmap.go +++ b/syswrap/mmap.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package syswrap wraps syscalls (just mmap right now) in order to impose a // global in-process limit on the maximum number of active mmaps. package syswrap diff --git a/syswrap/os.go b/syswrap/os.go index 8916d593b..1704b0759 100644 --- a/syswrap/os.go +++ b/syswrap/os.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package syswrap import ( diff --git a/toml/toml.go b/toml/toml.go index 5193ad787..acfb93079 100644 --- a/toml/toml.go +++ b/toml/toml.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package toml import "time" diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index aacc42d6e..47ec439fc 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package opentracing import ( diff --git a/tracing/tracing.go b/tracing/tracing.go index 5792dd7ca..3625166b0 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tracing import ( diff --git a/translate.go b/translate.go index db129cd8f..6a49e4a7a 100644 --- a/translate.go +++ b/translate.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/translate_test.go b/translate_test.go index 2b5d35a49..a2f0b0268 100644 --- a/translate_test.go +++ b/translate_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/utils_internal_test.go b/utils_internal_test.go index bf98f08a8..321f4846d 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 3c1d3e3145580836ac9ee2d51d7d35fe6f1a079f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 12 Apr 2019 11:35:34 -0500 Subject: [PATCH 10/73] Use bash for check-license-headers target --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f3f1ab608..9ced6c510 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,7 @@ gometalinter: require-gometalinter vendor ./... # Verify that all Go files have license header +check-license-headers: SHELL:=/bin/bash check-license-headers: @! find . -name '*.go' | grep -v '^./vendor' | while read fn;\ do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \ From b8ab44eb625bf3188521fff132684f6b4d6d5073 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 12 Apr 2019 11:57:28 -0500 Subject: [PATCH 11/73] Fix enterprise license header and add shardwidth files to license header check --- Makefile | 2 +- enterprise/enterprise.go | 14 -------------- shardwidth/16.go | 14 ++++++++++++++ shardwidth/17.go | 14 ++++++++++++++ shardwidth/18.go | 14 ++++++++++++++ shardwidth/19.go | 14 ++++++++++++++ shardwidth/20.go | 14 ++++++++++++++ shardwidth/21.go | 14 ++++++++++++++ shardwidth/22.go | 14 ++++++++++++++ shardwidth/23.go | 14 ++++++++++++++ shardwidth/24.go | 14 ++++++++++++++ shardwidth/25.go | 14 ++++++++++++++ shardwidth/26.go | 14 ++++++++++++++ shardwidth/27.go | 14 ++++++++++++++ shardwidth/28.go | 14 ++++++++++++++ shardwidth/29.go | 14 ++++++++++++++ shardwidth/30.go | 14 ++++++++++++++ shardwidth/31.go | 14 ++++++++++++++ shardwidth/32.go | 14 ++++++++++++++ 19 files changed, 239 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9ced6c510..3fa415ccb 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ check-license-headers: SHELL:=/bin/bash check-license-headers: @! find . -name '*.go' | grep -v '^./vendor' | while read fn;\ do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \ - grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree + grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise ###################### # Build dependencies # diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go index 2075f7a4d..a75223801 100644 --- a/enterprise/enterprise.go +++ b/enterprise/enterprise.go @@ -1,17 +1,3 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // Copyright (c) 2018 Pilosa Corp. All rights reserved. // // This file is part of Pilosa Enterprise Edition. diff --git a/shardwidth/16.go b/shardwidth/16.go index f9e90c047..6334bf254 100644 --- a/shardwidth/16.go +++ b/shardwidth/16.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth16 package shardwidth diff --git a/shardwidth/17.go b/shardwidth/17.go index 65653456e..b6ba8d907 100644 --- a/shardwidth/17.go +++ b/shardwidth/17.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth17 package shardwidth diff --git a/shardwidth/18.go b/shardwidth/18.go index 1357a3c17..f69647681 100644 --- a/shardwidth/18.go +++ b/shardwidth/18.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth18 package shardwidth diff --git a/shardwidth/19.go b/shardwidth/19.go index a3333540c..e9920326b 100644 --- a/shardwidth/19.go +++ b/shardwidth/19.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth19 package shardwidth diff --git a/shardwidth/20.go b/shardwidth/20.go index e45c5a472..5505656dd 100644 --- a/shardwidth/20.go +++ b/shardwidth/20.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32 package shardwidth diff --git a/shardwidth/21.go b/shardwidth/21.go index edaf0a010..0ab1be104 100644 --- a/shardwidth/21.go +++ b/shardwidth/21.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth21 package shardwidth diff --git a/shardwidth/22.go b/shardwidth/22.go index ab75e51da..9b07fff50 100644 --- a/shardwidth/22.go +++ b/shardwidth/22.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth22 package shardwidth diff --git a/shardwidth/23.go b/shardwidth/23.go index 2535a2dee..03c5cd91c 100644 --- a/shardwidth/23.go +++ b/shardwidth/23.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth23 package shardwidth diff --git a/shardwidth/24.go b/shardwidth/24.go index b1275afd8..7e175b1bc 100644 --- a/shardwidth/24.go +++ b/shardwidth/24.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth24 package shardwidth diff --git a/shardwidth/25.go b/shardwidth/25.go index 9599ba5a3..776655580 100644 --- a/shardwidth/25.go +++ b/shardwidth/25.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth25 package shardwidth diff --git a/shardwidth/26.go b/shardwidth/26.go index e7dfdd13b..1459686bc 100644 --- a/shardwidth/26.go +++ b/shardwidth/26.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth26 package shardwidth diff --git a/shardwidth/27.go b/shardwidth/27.go index 568a798f8..9eb7ff830 100644 --- a/shardwidth/27.go +++ b/shardwidth/27.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth27 package shardwidth diff --git a/shardwidth/28.go b/shardwidth/28.go index bb43cd5a3..b3c38c944 100644 --- a/shardwidth/28.go +++ b/shardwidth/28.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth28 package shardwidth diff --git a/shardwidth/29.go b/shardwidth/29.go index f797b632d..e2b29faf1 100644 --- a/shardwidth/29.go +++ b/shardwidth/29.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth29 package shardwidth diff --git a/shardwidth/30.go b/shardwidth/30.go index aac761c82..a8684c236 100644 --- a/shardwidth/30.go +++ b/shardwidth/30.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth30 package shardwidth diff --git a/shardwidth/31.go b/shardwidth/31.go index 12f85bfcf..1dcd40ebb 100644 --- a/shardwidth/31.go +++ b/shardwidth/31.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth31 package shardwidth diff --git a/shardwidth/32.go b/shardwidth/32.go index 76dde6b04..d9408285b 100644 --- a/shardwidth/32.go +++ b/shardwidth/32.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build shardwidth32 package shardwidth From 357f0bbf8ccd5a88d43df7ae62c1fbfc8665b319 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 12 Apr 2019 13:47:40 -0500 Subject: [PATCH 12/73] Add changelog steps to PR template --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 081a20073..ceb76f05f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,6 +12,8 @@ Fixes # - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. - [ ] All new and existing tests pass. +- [ ] Make sure PR title conforms to convention in CHANGELOG.md. +- [ ] Add appropriate changelog label to PR (if applicable). ## Code review checklist This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for. @@ -22,3 +24,5 @@ This is the checklist that the reviewer will follow while reviewing your pull re - [ ] Check that tests have been written and that they cover the new functionality. - [ ] Run tests and ensure they pass. - [ ] Build and run the code, performing any applicable integration testing. +- [ ] Make sure PR title conforms to convention in CHANGELOG.md. +- [ ] Make sure PR is tagged with appropriate changelog label. From f8a8a5d0965a55ea73c9e3212d72c4edf067fdcb Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 12 Apr 2019 21:27:06 -0500 Subject: [PATCH 13/73] return orig error instead of cause in handler also include the invalid name when erroring that a name is invalid. --- http/handler.go | 2 +- index.go | 2 +- pilosa.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/http/handler.go b/http/handler.go index 3226eee50..5b79deef6 100644 --- a/http/handler.go +++ b/http/handler.go @@ -347,7 +347,7 @@ func (r *successResponse) check(err error) (statusCode int) { } r.Success = false - r.Error = &Error{Message: cause.Error()} + r.Error = &Error{Message: err.Error()} return statusCode } diff --git a/index.go b/index.go index e94692288..da749ff2d 100644 --- a/index.go +++ b/index.go @@ -155,7 +155,7 @@ func (i *Index) openFields() error { fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { - return ErrName + return errors.Wrapf(ErrName, "'%s'", fi.Name()) } if err := fld.Open(); err != nil { return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) diff --git a/pilosa.go b/pilosa.go index f8989e9b7..28eb4545f 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,7 +16,7 @@ package pilosa import ( "encoding/json" - "errors" + "github.com/pkg/errors" "regexp" ) @@ -152,7 +152,7 @@ const TimeFormat = "2006-01-02T15:04" // validateName ensures that the name is a valid format. func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { - return ErrName + return errors.Wrapf(ErrName, "'%s'", name) } return nil } From 27ff1b69c6d6f304a1f4e9ac4236dae1f5abedb9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 15 Apr 2019 08:41:13 -0500 Subject: [PATCH 14/73] fix error messages in test. Fatalf=>Errorf to see more errors. --- server/handler_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index d6c5e49ae..ff9654933 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -690,9 +690,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("POST", "/index/idx1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"index already exists"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } // create field @@ -710,9 +710,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("POST", "/index/idx1/field/fld1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"field already exists"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } // delete field @@ -730,9 +730,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("DELETE", "/index/idx1/field/fld1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"field not found"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"deleting field: field not found"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } // delete index @@ -750,9 +750,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("DELETE", "/index/idx1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"index not found"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } }) From 2414c7181263b4fb95642d35c0e149cb4a68cf70 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 15 Apr 2019 11:15:11 -0500 Subject: [PATCH 15/73] goimports --- pilosa.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pilosa.go b/pilosa.go index 28eb4545f..d410e1fd4 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,8 +16,9 @@ package pilosa import ( "encoding/json" - "github.com/pkg/errors" "regexp" + + "github.com/pkg/errors" ) // System errors. From 53aac3b17a1736656f081268f604ae9a7f315856 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 16 Apr 2019 09:56:40 -0500 Subject: [PATCH 16/73] update to latest memberlist fork with race fixes --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2cfa54cd8..8f412ab0c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pilosa/pilosa -replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 +replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d diff --git a/go.sum b/go.sum index e77168275..d33a2f661 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjf github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA= github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= +github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= From 258646a7d5d3b2edf1a110550fd6525a9380ba55 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 1 Feb 2019 20:44:49 -0600 Subject: [PATCH 17/73] Add golangci-lint to Makefile and CI config --- .circleci/config.yml | 11 +++++++++++ Makefile | 9 ++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 677f1a57c..477f7f230 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,12 @@ jobs: steps: - *fast-checkout - run: make check-license-headers + golangci-lint: + <<: *defaults + steps: + - *fast-checkout + - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.2 + - run: make golangci-lint test-build-arm: <<: *defaults steps: @@ -131,6 +137,9 @@ workflows: - check-license-headers: requires: - setup + - golangci-lint: + requires: + - setup - test-build-arm: requires: - setup @@ -158,6 +167,7 @@ workflows: requires: - linter - check-license-headers + - golangci-lint - test-golang-1.12 filters: tags: @@ -172,3 +182,4 @@ workflows: - linter - check-license-headers - test-golang-1.12 + - golangci-lint diff --git a/Makefile b/Makefile index 3fa415ccb..b3604dca9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -131,6 +131,10 @@ docker-build: docker-test: docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./... +# Run golangci-lint +golangci-lint: require-golangci-lint + golangci-lint run + # Run gometalinter with custom flags gometalinter: require-gometalinter vendor GO111MODULE=off gometalinter --vendor --disable-all \ @@ -185,6 +189,9 @@ install-protoc: install-peg: GO111MODULE=off go get github.com/pointlander/peg +install-golangci-lint: + go get -u github.com/golangci/golangci-lint/cmd/golangci-lint + install-gometalinter: GO111MODULE=off go get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install From d84fcb09e7293617f866d4bd33f5bcd9f7231569 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 1 Feb 2019 21:10:32 -0600 Subject: [PATCH 18/73] Workaround due to write permission to bin directory Pro tip: If you're gonna curl|bash, at least don't curl|sudo bash. --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 477f7f230..eb47ffcf8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,8 @@ jobs: <<: *defaults steps: - *fast-checkout - - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.2 + - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.13.2 + - run: sudo cp bin/golangci-lint /usr/local/bin/ - run: make golangci-lint test-build-arm: <<: *defaults From d49172446147a7fad663c9f02aab0c904410fbfa Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 29 Mar 2019 12:14:39 -0500 Subject: [PATCH 19/73] don't go get -u for golangci-lint golangci-lint is actually dependent on a specific not-quite most recent version of golang.org/x/tools, fixing the dependency is hard and requires changing one of the upstream packages, just omitting the `-u` lets golangci-lint grab the version it wants and use that. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3604dca9..52839b3fe 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ install-peg: GO111MODULE=off go get github.com/pointlander/peg install-golangci-lint: - go get -u github.com/golangci/golangci-lint/cmd/golangci-lint + go get github.com/golangci/golangci-lint/cmd/golangci-lint install-gometalinter: GO111MODULE=off go get -u github.com/alecthomas/gometalinter From 3070c2d4ac4b4f041df991646f8de614da610f74 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 1 Apr 2019 11:27:44 -0500 Subject: [PATCH 20/73] try suppressing modules for golangci-lint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52839b3fe..9757af157 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ install-peg: GO111MODULE=off go get github.com/pointlander/peg install-golangci-lint: - go get github.com/golangci/golangci-lint/cmd/golangci-lint + GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint install-gometalinter: GO111MODULE=off go get -u github.com/alecthomas/gometalinter From 20a8c48552a878514533a16b6f8cf420d0d441ec Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 29 Mar 2019 12:22:58 -0500 Subject: [PATCH 21/73] boltdb/attrstore.go: fix up lint about error checking There's two kinds of unchecked errors here. Writes to a hash (we don't care, hash functions usually don't error in ways we care about), and rollbacks of non-writing transactions to a database. After studying the boltdb docs, I concluded that the recommended solution is to use the `.View(...)` function instead of directly controlling the transaction, so I switched the functions to do that. --- boltdb/attrstore.go | 92 ++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 260892646..280275a4d 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -215,66 +215,64 @@ func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } // Blocks returns a list of all blocks in the store. -func (s *attrStore) Blocks() ([]pilosa.AttrBlock, error) { - tx, err := s.db.Begin(false) - if err != nil { - return nil, errors.Wrap(err, "starting transaction") - } - defer tx.Rollback() +func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) { + err = s.db.View(func(tx *bolt.Tx) error { + // Wrap cursor to segment by block. + cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) - // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) + // Iterate over each block. + for cur.nextBlock() { + block := pilosa.AttrBlock{ID: cur.blockID()} - // Iterate over each block. - var blocks []pilosa.AttrBlock - for cur.nextBlock() { - block := pilosa.AttrBlock{ID: cur.blockID()} + // Compute checksum of every key/value in block. + h := xxhash.New() + for k, v := cur.next(); k != nil; k, v = cur.next() { + // hash function writes don't usually need to be checked + _, _ = h.Write(k) + _, _ = h.Write(v) + } + block.Checksum = h.Sum(nil) - // Compute checksum of every key/value in block. - h := xxhash.New() - for k, v := cur.next(); k != nil; k, v = cur.next() { - h.Write(k) - h.Write(v) + // Append block. + blocks = append(blocks, block) } - block.Checksum = h.Sum(nil) - - // Append block. - blocks = append(blocks, block) + return nil + }) + if err != nil { + return nil, err } - return blocks, nil } // BlockData returns all data for a single block. -func (s *attrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { - m := make(map[uint64]map[string]interface{}) +func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, err error) { + m = make(map[uint64]map[string]interface{}) // Start read-only transaction. - tx, err := s.db.Begin(false) + err = s.db.View(func(tx *bolt.Tx) error { + // Move to the start of the block. + min := u64tob(i * attrBlockSize) + max := u64tob((i + 1) * attrBlockSize) + cur := tx.Bucket([]byte("attrs")).Cursor() + for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { + // Exit if we're past the end of the block. + if bytes.Compare(k, max) != -1 { + break + } + + // Decode attribute map and associate with id. + attrs, err := pilosa.DecodeAttrs(v) + if err != nil { + return errors.Wrap(err, "decoding attrs") + } + m[btou64(k)] = attrs + + } + return nil + }) if err != nil { - return nil, errors.Wrap(err, "starting transaction") + return nil, err } - defer tx.Rollback() - - // Move to the start of the block. - min := u64tob(i * attrBlockSize) - max := u64tob((i + 1) * attrBlockSize) - cur := tx.Bucket([]byte("attrs")).Cursor() - for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { - // Exit if we're past the end of the block. - if bytes.Compare(k, max) != -1 { - break - } - - // Decode attribute map and associate with id. - attrs, err := pilosa.DecodeAttrs(v) - if err != nil { - return nil, errors.Wrap(err, "decoding attrs") - } - m[btou64(k)] = attrs - - } - return m, nil } From 77d49ded6494999f207e4ed90e12f4374f9bef1b Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 29 Mar 2019 14:40:21 -0500 Subject: [PATCH 22/73] so much lint So with the switch to a new linter, we get a lot of new warnings, and the majority of them are harmless probably, but a few might be real. Variously just use _ to suppress warnings, or report errors. There's probably things here that deserve better fixes, but we can always revisit it. --- cluster.go | 11 +- cluster_internal_test.go | 68 ++++++++--- cmd/root.go | 3 +- cmd/root_test.go | 6 +- ctl/check.go | 15 ++- ctl/check_test.go | 16 ++- ctl/config_test.go | 13 +- ctl/export_test.go | 12 +- ctl/generate_config_test.go | 12 +- ctl/import_test.go | 194 +++++++++++++++++++++++------- ctl/inspect.go | 8 +- ctl/inspect_test.go | 13 +- diagnostics_internal_test.go | 21 +++- enterprise/b/btree.go | 4 +- enterprise/b/containers_btree.go | 3 +- field.go | 4 +- field_internal_test.go | 10 +- field_test.go | 15 ++- fragment.go | 4 +- fragment_internal_test.go | 89 +++++++++++--- gopsutil/systeminfo.go | 4 +- holder.go | 8 +- holder_internal_test.go | 46 +++++-- holder_test.go | 14 ++- http/client.go | 5 +- http/client_test.go | 15 ++- http/handler.go | 45 +++++-- logger/logger.go | 23 ++++ lru/lru.go | 4 +- pql/pql.peg.go | 2 +- roaring/btree.go | 10 +- roaring/containers_btree.go | 3 +- roaring/roaring.go | 13 +- roaring/roaring_internal_test.go | 55 +++++++-- roaring/roaring_test.go | 199 +++++++++++++++++++------------ server.go | 7 +- server/handler_test.go | 10 +- server/server_test.go | 7 +- utils_internal_test.go | 6 +- view.go | 2 +- 40 files changed, 733 insertions(+), 266 deletions(-) diff --git a/cluster.go b/cluster.go index 204a1016b..024c426a1 100644 --- a/cluster.go +++ b/cluster.go @@ -844,8 +844,8 @@ func (c *cluster) partition(index string, shard uint64) int { // Hash the bytes and mod by partition count. h := fnv.New64a() - h.Write([]byte(index)) - h.Write(buf[:]) + _, _ = h.Write([]byte(index)) + _, _ = h.Write(buf[:]) return int(h.Sum64() % uint64(c.partitionN)) } @@ -1892,7 +1892,12 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { for _, node := range officialNodes { if node.ID == c.Node.ID && node.State != c.Node.State { c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State) - go c.setNodeState(c.Node.State) + go func() { + err := c.setNodeState(c.Node.State) + if err != nil { + c.logger.Printf("error setting node state from %v to %v: %v", node.State, c.Node.State, err) + } + }() } if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") diff --git a/cluster_internal_test.go b/cluster_internal_test.go index fa40c48b7..84392ffd9 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -607,7 +607,9 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, in topology", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node := tc.Clusters[0] @@ -615,7 +617,9 @@ func TestCluster_ResizeStates(t *testing.T) { top := &Topology{ nodeIDs: []string{node.Node.ID}, } - tc.WriteTopology(node.Path, top) + if err := tc.WriteTopology(node.Path, top); err != nil { + t.Fatalf("writing topology: %v", err) + } // Open TestCluster. if err := tc.Open(); err != nil { @@ -635,7 +639,9 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, not in topology", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node := tc.Clusters[0] @@ -643,7 +649,9 @@ func TestCluster_ResizeStates(t *testing.T) { top := &Topology{ nodeIDs: []string{"some-other-host"}, } - tc.WriteTopology(node.Path, top) + if err := tc.WriteTopology(node.Path, top); err != nil { + t.Fatalf("writing topology: %v", err) + } // Open TestCluster. expected := "coordinator node0 is not in topology: [some-other-host]" @@ -660,14 +668,18 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, no data", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } // Open TestCluster. if err := tc.Open(); err != nil { - t.Fatal(err) + t.Fatalf("opening cluster: %v", err) } - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node0 := tc.Clusters[0] node1 := tc.Clusters[1] @@ -698,18 +710,23 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + err := tc.addNode() + if err != nil { + t.Fatalf("adding node: %v", err) + } node0 := tc.Clusters[0] // write topology to data file top := &Topology{ nodeIDs: []string{"node0", "node2"}, } - tc.WriteTopology(node0.Path, top) + if err := tc.WriteTopology(node0.Path, top); err != nil { + t.Fatalf("writing topology: %v", err) + } // Open TestCluster. if err := tc.Open(); err != nil { - t.Fatal(err) + t.Fatalf("opening cluster: %v", err) } // Ensure that node is in state STARTING before the other node joins. @@ -719,19 +736,22 @@ func TestCluster_ResizeStates(t *testing.T) { // Expect an error by adding a node not in the topology. expectedError := "host is not in topology: node1" - err := tc.addNode() + err = tc.addNode() if err == nil || err.Error() != expectedError { t.Errorf("did not receive expected error: %s", expectedError) } - tc.addNode() + err = tc.addNode() + if err != nil { + t.Fatalf("adding node: %v", err) + } node2 := tc.Clusters[2] // Ensure that node comes up in state NORMAL. if node0.State() != ClusterStateNormal { t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) } else if node2.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State()) } // Close TestCluster. @@ -742,20 +762,27 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, with data", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + err := tc.addNode() + if err != nil { + t.Fatalf("adding node: %v", err) + } node0 := tc.Clusters[0] // Open TestCluster. - if err := tc.Open(); err != nil { + if err = tc.Open(); err != nil { t.Fatal(err) } // Add Bit Data to node0. if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil { - t.Fatal(err) + t.Fatalf("creating field: %v", err) + } + if err := tc.SetBit("i", "f", 1, 101, nil); err != nil { + t.Fatalf("setting bit: %v", err) + } + if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil { + t.Fatalf("setting bit: %v", err) } - tc.SetBit("i", "f", 1, 101, nil) - tc.SetBit("i", "f", 1, ShardWidth+1, nil) // Before starting the resize, get the CheckSum to use for // comparison later. @@ -765,7 +792,10 @@ func TestCluster_ResizeStates(t *testing.T) { node0Checksum := node0Fragment.Checksum() // addNode needs to block until the resize process has completed. - tc.addNode() + err = tc.addNode() + if err != nil { + t.Fatalf("adding node: %v", err) + } node1 := tc.Clusters[1] // Ensure that nodes come up in state NORMAL. diff --git a/cmd/root.go b/cmd/root.go index 64b8b1ca9..a8f8e6ddc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -54,9 +54,8 @@ Build Time: ` + pilosa.BuildTime + "\n", if ret, err := cmd.Flags().GetBool("dry-run"); ret && err == nil { if cmd.Parent() != nil { return fmt.Errorf("dry run") - } else if err != nil { - return fmt.Errorf("problem getting dry-run flag: %v", err) } + return fmt.Errorf("problem getting dry-run flag: %v", err) } return nil diff --git a/cmd/root_test.go b/cmd/root_test.go index 28bbea20d..6807aaf3c 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -187,10 +187,12 @@ bind = "127.0.0.1:10101" "127.0.0.1:10101", "127.0.0.1:10111", ]` - file.Write([]byte(config)) + if _, err := file.Write([]byte(config)); err != nil { + t.Fatalf("writing config file: %v", err) + } file.Close() _, err = ExecNewRootCommand(t, "server", "--config", file.Name()) - if err.Error() != "invalid option in configuration file: cluster.partitions" { + if err == nil || err.Error() != "invalid option in configuration file: cluster.partitions" { t.Fatalf("Expected invalid option in configuration file, but err: '%v'", err) } } diff --git a/ctl/check.go b/ctl/check.go index b1389b6ee..0beee5bc5 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -68,7 +68,7 @@ func (cmd *CheckCommand) Run(_ context.Context) error { } // checkBitmapFile performs a consistency check on path for a roaring bitmap file. -func (cmd *CheckCommand) checkBitmapFile(path string) error { +func (cmd *CheckCommand) checkBitmapFile(path string) (err error) { // Open file handle. f, err := os.Open(path) if err != nil { @@ -86,8 +86,17 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error { if err != nil { return errors.Wrap(err, "mmapping") } - defer syscall.Munmap(data) - + defer func() { + e := syscall.Munmap(data) + if e != nil { + fmt.Fprintf(cmd.Stderr, "WARNING: munmap failed: %v", e) + } + // don't overwrite another error with this, but also indicate + // this error. + if err == nil { + err = e + } + }() // Attach the mmap file to the bitmap. bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { diff --git a/ctl/check_test.go b/ctl/check_test.go index 1b2ebd6f6..e37d99358 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -40,7 +40,9 @@ func TestCheckCommand_RunCacheFile(t *testing.T) { err := cm.Run(context.Background()) w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("copy: %v", err) + } if !strings.Contains(buf.String(), "ignoring cache file") { t.Fatalf("expect: ignoring cache file, actual: '%s'", err) @@ -59,7 +61,9 @@ func TestCheckCommand_RunSnapshot(t *testing.T) { err := cm.Run(context.Background()) w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("copy: %v", err) + } if !strings.Contains(buf.String(), "ignoring snapshot file") { t.Fatalf("expect: ignoring snapshot file, actual: '%s'", err) @@ -71,7 +75,9 @@ func TestCheckCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - file.Write([]byte("1234,1223")) + if _, err := file.Write([]byte("1234,1223")); err != nil { + t.Fatalf("writing to temp file: %v", err) + } file.Close() rder := []byte{} @@ -83,7 +89,9 @@ func TestCheckCommand_Run(t *testing.T) { err = cm.Run(context.Background()) w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("copy: %v", err) + } if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: reading roaring header:") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) diff --git a/ctl/config_test.go b/ctl/config_test.go index ca08c273b..b9a5dc0b5 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -33,13 +33,16 @@ func TestConfigCommand_Run(t *testing.T) { cm.Config = server.NewConfig() err := cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) - if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), ":10101") { + } + w.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: \n%s", buf.String()) } } diff --git a/ctl/export_test.go b/ctl/export_test.go index e3189efe3..6cb611460 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -52,8 +52,16 @@ func TestExportCommand_Run(t *testing.T) { hostport := cmd.API.Node().URI.HostPort() cm.Host = hostport - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(""))) + resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("making http request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(""))) + if err != nil { + t.Fatalf("making http request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 26b531e8f..a431cd468 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -29,12 +29,16 @@ func TestGenerateConfigCommand_Run(t *testing.T) { r, w, _ := os.Pipe() cm := NewGenerateConfigCommand(stdin, w, os.Stderr) err := cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), ":10101") { + } + w.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import_test.go b/ctl/import_test.go index e701af1b2..b20919203 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -58,7 +58,13 @@ func TestImportCommand_Basic(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") - file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } ctx := context.Background() if err != nil { t.Fatal(err) @@ -82,11 +88,14 @@ func TestImportCommand_Basic(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") - file.Write([]byte("1,2\n3,4\n5,6")) - ctx := context.Background() if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.API.Node().URI.HostPort() @@ -110,17 +119,28 @@ func TestImportCommand_RunValue(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-value.csv") - file.Write([]byte("1,2\n3,4\n5,6")) - ctx := context.Background() if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("http request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + if err != nil { + t.Fatalf("http request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -136,7 +156,13 @@ func TestImportCommand_RunValue(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-value.csv") - file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } ctx := context.Background() if err != nil { t.Fatal(err) @@ -145,8 +171,16 @@ func TestImportCommand_RunValue(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -165,17 +199,28 @@ func TestImportCommand_RunKeys(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-key.csv") - file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6")) - ctx := context.Background() if err != nil { t.Fatal(err) } + _, err = file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -192,7 +237,9 @@ func TestImportCommand_KeyReplication(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-key.csv") - + if err != nil { + t.Fatal(err) + } // create a large import file in order to test the // translateStoreBufferSize growth logic. keyBytes := []byte{} @@ -205,11 +252,11 @@ func TestImportCommand_KeyReplication(t *testing.T) { x := "fooEND,barEND" keyBytes = append(keyBytes, x...) - file.Write(keyBytes) - ctx := context.Background() + _, err = file.Write(keyBytes) if err != nil { - t.Fatal(err) + t.Fatalf("writing to tempfile: %v", err) } + ctx := context.Background() c := test.MustRunCluster(t, 2) cmd0 := c[0] @@ -220,8 +267,16 @@ func TestImportCommand_KeyReplication(t *testing.T) { cm.Host = host0 - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -255,17 +310,28 @@ func TestImportCommand_RunValueKeys(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-key.csv") - file.Write([]byte("foo1,2\nfoo3,4\nfoo5,6")) - ctx := context.Background() if err != nil { t.Fatal(err) } + _, err = file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -286,9 +352,12 @@ func TestImportCommand_InvalidFile(t *testing.T) { cm.Index = "i" cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") - file.Write([]byte("a,2\n3,5\n5,6")) if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) } cm.Paths = []string{file.Name()} err = cm.Run(context.Background()) @@ -297,9 +366,12 @@ func TestImportCommand_InvalidFile(t *testing.T) { } file, err = ioutil.TempFile("", "import1.csv") - file.Write([]byte("1,\n3,\n5,6")) if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,\n3,\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) } cm.Paths = []string{file.Name()} err = cm.Run(context.Background()) @@ -308,7 +380,10 @@ func TestImportCommand_InvalidFile(t *testing.T) { } file, err = ioutil.TempFile("", "import1.csv") - file.Write([]byte("1,2,34343\n1,3,54565,\n5,6,565")) + if err != nil { + t.Fatal(err) + } + _, err = file.Write([]byte("1,2,34343\n1,3,54565,\n5,6,565")) if err != nil { t.Fatal(err) } @@ -319,7 +394,10 @@ func TestImportCommand_InvalidFile(t *testing.T) { } file, err = ioutil.TempFile("", "import1.csv") - file.Write([]byte("1\n3\n5")) + if err != nil { + t.Fatal(err) + } + _, err = file.Write([]byte("1\n3\n5")) if err != nil { t.Fatal(err) } @@ -357,16 +435,27 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-value.csv") - file.Write([]byte("0,17\n")) - ctx := context.Background() if err != nil { t.Fatal(err) } + _, err = file.Write([]byte("0,17\n")) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -381,7 +470,10 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { if err != nil { t.Fatalf("Error creating tempfile: %s", err) } - file.Write([]byte("0,16\n")) + _, err = file.Write([]byte("0,16\n")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -393,7 +485,10 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { if err != nil { t.Fatalf("Error creating tempfile: %s", err) } - file.Write([]byte("0,19\n")) + _, err = file.Write([]byte("0,19\n")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -411,8 +506,16 @@ func TestImportCommand_RunBool(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -422,7 +525,10 @@ func TestImportCommand_RunBool(t *testing.T) { if err != nil { t.Fatal(err) } - file.Write([]byte("0,1\n1,2\n1,3")) + _, err = file.Write([]byte("0,1\n1,2\n1,3")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) @@ -437,8 +543,10 @@ func TestImportCommand_RunBool(t *testing.T) { if err != nil { t.Fatal(err) } - file.Write([]byte("0,1\n1,2\n1,3\n2,4")) - + _, err = file.Write([]byte("0,1\n1,2\n1,3\n2,4")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") { diff --git a/ctl/inspect.go b/ctl/inspect.go index c0676f8da..204ffab3b 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -64,8 +64,12 @@ func (cmd *InspectCommand) Run(_ context.Context) error { if err != nil { return errors.Wrap(err, "mmapping") } - defer syscall.Munmap(data) - + defer func() { + err := syscall.Munmap(data) + if err != nil { + fmt.Fprintf(cmd.Stderr, "inspect command: munmap failed: %v", err) + } + }() // Attach the mmap file to the bitmap. t := time.Now() fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go index c7a48d400..265d77a12 100644 --- a/ctl/inspect_test.go +++ b/ctl/inspect_test.go @@ -34,14 +34,23 @@ func TestInspectCommand_Run(t *testing.T) { if err != nil { t.Fatalf("Error creating tempfile: %s", err) } - file.Write([]byte("12358267538963")) + _, err = file.Write([]byte("12358267538963")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } file.Close() cm.Path = file.Name() err = cm.Run(context.Background()) + if err != nil { + t.Fatalf("can't run command: %v", err) + } w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + _, err = io.Copy(&buf, r) + if err != nil { + t.Fatalf("copying data: %v", err) + } if !strings.Contains(buf.String(), "unmarshaling bitmap...") { t.Fatalf("Inspect doesn't work: %s", err) } diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index 99c0a83ac..6ddc98b32 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -22,6 +22,8 @@ import ( "runtime" "strings" "testing" + + "github.com/pilosa/pilosa/logger" ) func TestDiagnosticsClient(t *testing.T) { @@ -112,19 +114,34 @@ func TestDiagnosticsVersion_Check(t *testing.T) { // Mock server. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(versionResponse{ + err := json.NewEncoder(w).Encode(versionResponse{ Version: "1.1.1", }) + if err != nil { + t.Fatalf("couldn't encode version response: %v", err) + } })) // Create a new client. d := newDiagnosticsCollector("localhost:10101") + logs := logger.NewCaptureLogger() + d.Logger = logs + version := "0.1.1" d.SetVersion(version) d.VersionURL = server.URL - d.CheckVersion() + err := d.CheckVersion() + if err != nil { + t.Fatalf("checking version: %v", err) + } + if len(logs.Prints) != 1 { + t.Fatalf("expected a version upgrade message") + } + if !strings.Contains(logs.Prints[0], "a newer version") { + t.Fatalf("expected version upgrade message, got '%s'", logs.Prints[0]) + } } func compareJSON(a, b []byte) (bool, error) { diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go index c61f5c5b6..2fa5c24e8 100644 --- a/enterprise/b/btree.go +++ b/enterprise/b/btree.go @@ -873,7 +873,7 @@ func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) { i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, true - e.next() + _ = e.next() return k, v, nil } @@ -928,7 +928,7 @@ func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) { i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, true - e.prev() + _ = e.prev() return k, v, err } diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index db5c9946d..a051d7cde 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -44,7 +44,8 @@ func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { b := &roaring.Bitmap{ Containers: newBTreeContainers(), } - b.Add(a...) + // TODO: there's no way to report an error here + _, _ = b.Add(a...) return b } diff --git a/field.go b/field.go index bd4dd1aca..d83735893 100644 --- a/field.go +++ b/field.go @@ -612,7 +612,9 @@ func (f *Field) createBSIGroup(bsig *bsiGroup) error { if err := f.addBSIGroup(bsig); err != nil { return err } - f.saveMeta() + if err := f.saveMeta(); err != nil { + return errors.Wrap(err, "saving") + } return nil } diff --git a/field_internal_test.go b/field_internal_test.go index 0a26225bb..ad8337c31 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -370,7 +370,10 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) { // bm represents remote available shards. bm := roaring.NewBitmap() for i := uint64(0); i < 1204; i += 2 { - bm.Add(i) + _, err := bm.Add(i) + if err != nil { + t.Fatalf("adding bits: %v", err) + } } if err := f.AddRemoteAvailableShards(bm); err != nil { @@ -386,7 +389,10 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) { bm1 := roaring.NewBitmap() for i := uint64(1); i < 1204; i += 2 { - bm1.Add(i) + _, err := bm1.Add(i) + if err != nil { + t.Fatalf("adding bits: %v", err) + } } if err := f.AddRemoteAvailableShards(bm1); err != nil { diff --git a/field_test.go b/field_test.go index 2a0efc0b9..5911abd6a 100644 --- a/field_test.go +++ b/field_test.go @@ -208,17 +208,20 @@ func TestField_AvailableShards(t *testing.T) { } // Set remote shards and verify. - f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)) + if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil { + t.Fatalf("adding remote shards: %v", err) + } if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" { t.Fatal(diff) } // Delete shards; only local shards should remain. - f.RemoveAvailableShard(0) - f.RemoveAvailableShard(1) - f.RemoveAvailableShard(2) - f.RemoveAvailableShard(3) - f.RemoveAvailableShard(4) + for i := uint64(0); i < 5; i++ { + err := f.RemoveAvailableShard(i) + if err != nil { + t.Fatalf("removing shard: %v", err) + } + } if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { t.Fatal(diff) } diff --git a/fragment.go b/fragment.go index 82288c914..2409a5fc5 100644 --- a/fragment.go +++ b/fragment.go @@ -1328,7 +1328,7 @@ type topOptions struct { func (f *fragment) Checksum() []byte { h := xxhash.New() for _, block := range f.Blocks() { - h.Write(block.Checksum) + _, _ = h.Write(block.Checksum) } return h.Sum(nil) } @@ -2332,7 +2332,7 @@ func (h *blockHasher) Sum() []byte { func (h *blockHasher) WriteValue(v uint64) { binary.BigEndian.PutUint64(h.buf[:], v) - h.hash.Write(h.buf[:]) + _, _ = h.hash.Write(h.buf[:]) } // fragmentSyncer syncs a local fragment to one on a remote host. diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 3930e98e2..8c3d65c66 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -654,7 +654,9 @@ func TestFragment_Range(t *testing.T) { func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { column := uint64(0) for i := 0; i < b.N; i++ { - f.setValue(column, bitDepth, uint64(i)) + // We're not checking the error because this is a benchmark. + // That does mean the result could be completely wrong... + _, _ = f.setValue(column, bitDepth, uint64(i)) column = cfunc(column) } } @@ -943,8 +945,14 @@ func TestFragment_Top_Filter(t *testing.T) { f.mustSetBits(102, 1, 2) f.RecalculateCache() // Assign attributes. - f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) - f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) + err := f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) + if err != nil { + t.Fatalf("setAttrs: %v", err) + } + err = f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) + if err != nil { + t.Fatalf("setAttrs: %v", err) + } // Retrieve top rows. if pairs, err := f.top(topOptions{ @@ -2064,7 +2072,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) - frags[j].importRoaring(data, false) + err := frags[j].importRoaring(data, false) + if err != nil { + b.Fatalf("importing roaring: %v", err) + } } eg := errgroup.Group{} b.StartTimer() @@ -2275,7 +2286,10 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { func TestGetZipfRowsSliceRoaring(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) data := getZipfRowsSliceRoaring(10, 1, 0, ShardWidth) - f.importRoaring(data, false) + err := f.importRoaring(data, false) + if err != nil { + t.Fatalf("importing roaring: %v", err) + } if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { t.Fatalf("unexpected rows: %v", f.rows(0)) } @@ -2608,7 +2622,10 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaring(buf.Bytes(), false) + err = f.importRoaring(buf.Bytes(), false) + if err != nil { + t.Fatalf("importing roaring: %v", err) + } exp := calcExpected(test[:num+1]...) for row, expCols := range exp { cols := f.row(uint64(row)).Columns() @@ -2680,7 +2697,10 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaring(buf.Bytes(), false) + err = f.importRoaring(buf.Bytes(), false) + if err != nil { + t.Fatalf("importing roaring: %v", err) + } rows, cols := toRowsCols(test.roaring) expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) pairs, err = f.top(topOptions{}) @@ -2895,19 +2915,56 @@ func TestFragmentRowIterator(t *testing.T) { func TestUnionInPlaceMapped(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) defer f.Clean(t) + // I know this doesn't actually matter in our current context, but + // strictly speaking, we do say you have to hold the lock while calling + // unprotectedWriteToFragment... + f.mu.Lock() + defer f.mu.Unlock() r0 := rand.New(rand.NewSource(2)) r1 := rand.New(rand.NewSource(1)) data0 := randPositions(1000000, r0) - setBM := roaring.NewBitmap() - setBM.OpWriter = nil - setBM.Add(data0...) - unprotectedWriteToFragment(f, setBM) - data1 := randPositions(1000000, r1) - setBM2 := roaring.NewBitmap() - setBM2.OpWriter = nil - setBM2.Add(data1...) + setBM0 := roaring.NewBitmap() + setBM0.OpWriter = nil + _, err := setBM0.Add(data0...) + if err != nil { + t.Fatalf("adding bits: %v", err) + } + count0 := setBM0.Count() - f.storage.UnionInPlace(setBM2) + data1 := randPositions(1000000, r1) + setBM1 := roaring.NewBitmap() + setBM1.OpWriter = nil + _, err = setBM1.Add(data1...) + if err != nil { + t.Fatalf("adding bits: %v", err) + } + count1 := setBM1.Count() + + // now we write setBM0 into f.storage. + err = unprotectedWriteToFragment(f, setBM0) + if err != nil { + t.Fatalf("trying to flush fragment to disk: %v", err) + } + countF := f.storage.Count() + + f.storage.UnionInPlace(setBM1) + countUnion := f.storage.Count() + + if count0 != countF { + t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF) + } + min := count0 + if count1 > min { + min = count1 + } + max := count0 + count1 + // We don't know how many bits we should have, because of overlap, + // but it should be between the size of the largest bitmap and the + // sum of the bitmaps. + if countUnion < min || countUnion > max { + t.Fatalf("union of sets with cardinality %d and %d should be between %d and %d, got %d", + count0, count1, min, max, countUnion) + } } func randPositions(n int, r *rand.Rand) []uint64 { diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index b50885241..13ada756e 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -139,9 +139,7 @@ func (s *systemInfo) collectPlatformInfo() error { // we have no way to know, let's try runtime s.cpuLogicalCores = runtime.NumCPU() } - if err != nil { - return err - } + return nil } return nil } diff --git a/holder.go b/holder.go index d751c90b2..40b21e538 100644 --- a/holder.go +++ b/holder.go @@ -594,9 +594,10 @@ func (h *Holder) loadNodeID() (string, error) { } nodeIDBytes, err := ioutil.ReadFile(idPath) - if err == nil { - nodeID = strings.TrimSpace(string(nodeIDBytes)) - } else if os.IsNotExist(err) { + // apparently it's safe to call IsNotExist on something that might + // be nil: + // https://github.com/golang/go/issues/31065 + if os.IsNotExist(err) { nodeID = uuid.NewV4().String() err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) if err != nil { @@ -605,6 +606,7 @@ func (h *Holder) loadNodeID() (string, error) { } else if err != nil { return "", errors.Wrap(err, "reading file") } + nodeID = strings.TrimSpace(string(nodeIDBytes)) return nodeID, nil } diff --git a/holder_internal_test.go b/holder_internal_test.go index b13df7809..0c96a81ea 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -112,8 +112,10 @@ func TestHolder_Optn(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) - + defer func() { + // we don't care about a failure here + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } @@ -136,7 +138,10 @@ func TestHolder_Optn(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + defer func() { + // we don't care about a failure here + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) @@ -165,8 +170,9 @@ func TestHolder_Optn(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) - + defer func() { + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0644) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } @@ -209,8 +215,14 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) // Set highest shard. - hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1)) - hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2)) + err := hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1)) + if err != nil { + t.Fatalf("adding remote shards: %v", err) + } + err = hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2)) + if err != nil { + t.Fatalf("adding remote shards: %v", err) + } // Keep replication the same and ensure we get the expected results. cluster.ReplicaN = 2 @@ -292,8 +304,20 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { func TestHolderCleaner_Reopen(t *testing.T) { h := NewHolder() h.Path = "path" - h.Open() - h.Close() - h.Open() - h.Close() + err := h.Open() + if err != nil { + t.Fatalf("couldn't open holder: %v", err) + } + err = h.Close() + if err != nil { + t.Fatalf("couldn't close holder: %v", err) + } + err = h.Open() + if err != nil { + t.Fatalf("couldn't open holder: %v", err) + } + err = h.Close() + if err != nil { + t.Fatalf("couldn't close holder: %v", err) + } } diff --git a/holder_test.go b/holder_test.go index 1bfe4615a..525ca1dc7 100644 --- a/holder_test.go +++ b/holder_test.go @@ -67,7 +67,9 @@ func TestHolder_Open(t *testing.T) { } else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(h.IndexPath("test"), 0777) + defer func() { + _ = os.Chmod(h.IndexPath("test"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) @@ -106,8 +108,9 @@ func TestHolder_Open(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0777) - + defer func() { + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } @@ -167,8 +170,9 @@ func TestHolder_Open(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0666) - + defer func() { + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0644) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } diff --git a/http/client.go b/http/client.go index 92f6269fe..f8ad7cb37 100644 --- a/http/client.go +++ b/http/client.go @@ -629,7 +629,10 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind dec := json.NewDecoder(resp.Body) rbody := &pilosa.ImportResponse{} - dec.Decode(rbody) + err = dec.Decode(rbody) + if err != nil { + return errors.Wrap(err, "decoding response body") + } if rbody.Err != "" { return errors.Wrap(errors.New(rbody.Err), "importing roaring") } diff --git a/http/client_test.go b/http/client_test.go index 2b65f82d4..0bda138ca 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -116,9 +116,18 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - c[0].RecalculateCaches() - c[1].RecalculateCaches() - c[2].RecalculateCaches() + err = c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating cache: %v", err) + } + err = c[1].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating cache: %v", err) + } + err = c[2].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating cache: %v", err) + } // Connect to each node to compare results. client := make([]*Client, 3) diff --git a/http/handler.go b/http/handler.go index 5b79deef6..bbb6e28a2 100644 --- a/http/handler.go +++ b/http/handler.go @@ -320,6 +320,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { + h *Handler Success bool `json:"success"` Error *Error `json:"error,omitempty"` } @@ -367,8 +368,18 @@ func (r *successResponse) write(w http.ResponseWriter, err error) { // Write the response. if statusCode == 0 { - w.Write(msg) - w.Write([]byte("\n")) + _, err := w.Write(msg) + if err != nil { + r.h.logger.Printf("error writing response: %v", err) + http.Error(w, string(msg), http.StatusInternalServerError) + return + } + _, err = w.Write([]byte("\n")) + if err != nil { + r.h.logger.Printf("error writing newline after response: %v", err) + http.Error(w, string(msg), http.StatusInternalServerError) + return + } } else { http.Error(w, string(msg), statusCode) } @@ -449,7 +460,10 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { req, err := h.readQueryRequest(r) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + if e != nil { + h.logger.Printf("write query response error: %v (while trying to write another error: %v)", e, err) + } return } // TODO: Remove @@ -463,7 +477,10 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { default: w.WriteHeader(http.StatusBadRequest) } - h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + if e != nil { + h.logger.Printf("write query response error: %v (while trying to write another error: %v)", e, err) + } return } @@ -612,7 +629,7 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - resp := successResponse{} + resp := successResponse{h: h} err := h.api.DeleteIndex(r.Context(), indexName) resp.write(w, err) } @@ -625,7 +642,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { } indexName := mux.Vars(r)["index"] - resp := successResponse{} + resp := successResponse{h: h} // Decode request. req := postIndexRequest{ @@ -694,7 +711,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] - resp := successResponse{} + resp := successResponse{h: h} // Decode request. var req postFieldRequest @@ -847,7 +864,7 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] - resp := successResponse{} + resp := successResponse{h: h} err := h.api.DeleteField(r.Context(), indexName, fieldName) resp.write(w, err) } @@ -863,7 +880,7 @@ func (h *Handler) handleDeleteRemoteAvailableShard(w http.ResponseWriter, r *htt fieldName := mux.Vars(r)["field"] shardID, _ := strconv.ParseUint(mux.Vars(r)["shardID"], 10, 64) - resp := successResponse{} + resp := successResponse{h: h} err := h.api.DeleteAvailableShard(r.Context(), indexName, fieldName, shardID) resp.write(w, err) } @@ -1080,7 +1097,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Write response. - w.Write(buf) + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } } // handleGetExport handles /export requests. @@ -1179,7 +1199,10 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Write response. w.Header().Set("Content-Type", "application/protobuf") w.Header().Set("Content-Length", strconv.Itoa(len(buf))) - w.Write(buf) + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing fragment/block/data response: %v", err) + } } // handleGetFragmentBlocks handles GET /internal/fragment/blocks requests. diff --git a/logger/logger.go b/logger/logger.go index ed5a2dc26..9e895d482 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -15,6 +15,7 @@ package logger import ( + "fmt" "io" "log" ) @@ -82,3 +83,25 @@ func (vb *verboseLogger) Debugf(format string, v ...interface{}) { func (vb *verboseLogger) Logger() *log.Logger { return vb.logger } + +// CaptureLogger is a logger that stores all the print and debug messages +// it sees, useful for testing. +type CaptureLogger struct { + Prints []string + Debugs []string +} + +// NewCaptureLogger yields a CaptureLogger. +func NewCaptureLogger() *CaptureLogger { + return &CaptureLogger{} +} + +// Printf formats a message and appends it to Prints. +func (cl *CaptureLogger) Printf(format string, v ...interface{}) { + cl.Prints = append(cl.Prints, fmt.Sprintf(format, v...)) +} + +// Debugf formats a message and appends it to Debugs. +func (cl *CaptureLogger) Debugf(format string, v ...interface{}) { + cl.Debugs = append(cl.Debugs, fmt.Sprintf(format, v...)) +} diff --git a/lru/lru.go b/lru/lru.go index ba0121a9a..7f2e6dc22 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -83,7 +83,7 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) { } // remove removes the provided key from the cache. -func (c *Cache) remove(key Key) { // nolint: staticcheck +func (c *Cache) remove(key Key) { // nolint: staticcheck,unused if c.cache == nil { return } @@ -121,7 +121,7 @@ func (c *Cache) Len() int { } // clear purges all stored items from the cache. -func (c *Cache) clear() { // nolint: staticcheck +func (c *Cache) clear() { // nolint: staticcheck,unused if c.OnEvicted != nil { for _, e := range c.cache { kv := e.Value.(*entry) diff --git a/pql/pql.peg.go b/pql/pql.peg.go index acec51d19..f1ae962fc 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -15,7 +15,7 @@ const endSymbol rune = 1114112 type pegRule uint8 const ( - ruleUnknown pegRule = iota + ruleUnknown pegRule = iota // nolint:varcheck,deadcode,unused ruleCalls ruleCall ruleallargs diff --git a/roaring/btree.go b/roaring/btree.go index 76427059f..192993c3b 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -870,8 +870,10 @@ func (e *enumerator) Next() (k uint64, v *Container, err error) { i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, true - e.next() - return k, v, nil + // Any error returned would be stashed in e.err, and would come up + // on the next call. + _ = e.next() + return k, v, err } func (e *enumerator) next() error { @@ -925,7 +927,9 @@ func (e *enumerator) Prev() (k uint64, v *Container, err error) { i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, true - e.prev() + // Any error returned would be stashed in e.err, and would come up + // on the next call. + _ = e.prev() return k, v, err } diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index 1d08f598c..5934a1244 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -35,7 +35,8 @@ func NewBTreeBitmap(a ...uint64) *Bitmap { b := &Bitmap{ Containers: newBTreeContainers(), } - b.Add(a...) + // TODO: We have no way to report this. + _, _ = b.Add(a...) return b } diff --git a/roaring/roaring.go b/roaring/roaring.go index a8ccd81e2..78b5f4ca8 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -137,7 +137,10 @@ func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ Containers: newSliceContainers(), } - b.AddN(a...) + // TODO: We have no way to report this. We aren't in a server context + // so we haven't got a logger, nothing is checking for nil returns + // from this... + _, _ = b.AddN(a...) return b } @@ -3695,8 +3698,8 @@ func (op *op) WriteTo(w io.Writer) (n int64, err error) { // Add checksum at the end. h := fnv.New32a() - h.Write(buf[0:9]) - h.Write(buf[13:]) + _, _ = h.Write(buf[0:9]) + _, _ = h.Write(buf[13:]) binary.LittleEndian.PutUint32(buf[9:13], h.Sum32()) // Write to writer. @@ -3719,13 +3722,13 @@ func (op *op) UnmarshalBinary(data []byte) error { // Verify checksum. h := fnv.New32a() - h.Write(data[0:9]) + _, _ = h.Write(data[0:9]) if op.typ > 1 { if len(data) < int(13+op.value*8) { return fmt.Errorf("op data truncated - expected %d, got %d", 13+op.value*8, len(data)) } - h.Write(data[13 : 13+op.value*8]) + _, _ = h.Write(data[13 : 13+op.value*8]) op.values = make([]uint64, op.value) for i := uint64(0); i < op.value; i++ { start := 13 + i*8 diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index e88356a37..8c3e7a930 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2127,10 +2127,14 @@ func TestIteratorBitmap(t *testing.T) { // but won't update to RLE until Optimize() is called b := NewFileBitmap() for i := uint64(61000); i < 71000; i++ { - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bit: %v", err) + } } for i := uint64(75000); i < 75100; i++ { - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bit: %v", err) + } } if !b.Containers.Get(0).isBitmap() { t.Fatalf("wrong container type") @@ -2393,7 +2397,9 @@ func TestRunBinSearch(t *testing.T) { } func TestBitmap_RemoveEmptyContainers(t *testing.T) { bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) - bm1.Remove(2 << 16) + if _, err := bm1.Remove(2 << 16); err != nil { + t.Fatalf("removing a bit: %v", err) + } if bm1.countEmptyContainers() != 1 { t.Fatalf("Should be 1 empty container ") } @@ -2406,13 +2412,17 @@ func TestBitmap_RemoveEmptyContainers(t *testing.T) { func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) - bm1.Remove(2 << 16) + if _, err := bm1.Remove(2 << 16); err != nil { + t.Fatalf("removing a bit: %v", err) + } var buf bytes.Buffer if _, err := bm1.WriteTo(&buf); err != nil { t.Fatalf("Failure to write to bitmap buffer. ") } bm0 := NewFileBitmap() - bm0.UnmarshalBinary(buf.Bytes()) + if err := bm0.UnmarshalBinary(buf.Bytes()); err != nil { + t.Fatalf("unmarshalling: %v", err) + } if bm0.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") } @@ -2559,7 +2569,9 @@ func TestIntersectArrayBitmap(t *testing.T) { func TestBitmapClone(t *testing.T) { b := NewFileBitmap() for i := uint64(61000); i < 71000; i++ { - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bit: %v", err) + } } c := b.Clone() if err := bitmapsEqual(b, c); err != nil { @@ -3770,24 +3782,45 @@ func TestBitmapAny(t *testing.T) { if bm.Any() { t.Error("empty bitmap should have Any()==false") } - bm.Add(1) + _, err := bm.Add(1) + if err != nil { + t.Errorf("couldn't add a bit: %v", err) + } if !bm.Any() { t.Error("bitmap with 1 bit should have Any()==true") } - bm.Add(100000) + _, err = bm.Add(100000) + if err != nil { + t.Errorf("couldn't add a bit: %v", err) + } if !bm.Any() { t.Error("bitmap with 2 bits should have Any()==true") } - bm.Remove(1) + changed, err := bm.Remove(1) + if err != nil { + t.Errorf("couldn't remove a bit: %v", err) + } + if changed != true { + t.Error("removing a set bit should have been a change") + } if !bm.Any() { t.Error("bitmap with 1 bit left after removing 1 should have Any()==true") } - bm.Add(1) + _, err = bm.Add(1) + if err != nil { + t.Errorf("couldn't remove a bit: %v", err) + } + if changed != true { + t.Error("re-addintg a previously set bit should have been a change") + } bm = bm.Difference(NewBTreeBitmap(1)) if !bm.Any() { t.Error("bitmap with 1 bit left after differencing 1 should have Any()==true") } - bm.Remove(100000) + _, err = bm.Remove(100000) + if err != nil { + t.Errorf("couldn't remove a bit: %v", err) + } if bm.Any() { t.Error("shouldn't be any left") } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index ebf88a212..b4f792629 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -164,11 +164,15 @@ func TestCheckBitmap(t *testing.T) { x := 0 for i := uint64(61000); i < 71000; i++ { x++ - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } for i := uint64(75000); i < 75100; i++ { x++ - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } err := b.Check() if err != nil { @@ -198,7 +202,7 @@ func TestCheckFullRun(t *testing.T) { if i%16384 == 0 { b.Optimize() // convert to runs } - b.Add(i) + _, _ = b.Add(i) } err := b.Check() if err != nil { @@ -238,7 +242,13 @@ func TestBitmap_Contains_Empty(t *testing.T) { // Ensure an empty bitmap does nothing when removing an element. func TestBitmap_Remove_Empty(t *testing.T) { - roaring.NewFileBitmap().Remove(1000) + changed, err := roaring.NewFileBitmap().Remove(1000) + if err != nil { + t.Fatalf("got an error removing a bit from an empty bitmap: %v", err) + } + if changed != false { + t.Fatalf("change reported removing a bit from an empty bitmap") + } } // Ensure a bitmap can return a slice of values. @@ -289,7 +299,9 @@ func TestBitmap_ForEachRange(t *testing.T) { func TestBitmap_Max(t *testing.T) { bm := roaring.NewFileBitmap() for i := uint64(1000); i <= 100000; i++ { - bm.Add(i) + if _, err := bm.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } if v := bm.Max(); v != i { t.Fatalf("max: got=%d; want=%d", v, i) @@ -310,7 +322,9 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { } else { start += 2 } - bm0.Add(start) + if _, err := bm0.Add(start); err != nil { + t.Fatalf("adding bit: %v", err) + } } a := bm0.Count() r := bm0.CountRange(s, e) @@ -323,9 +337,13 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { func TestBitmap_BitmapCountRange(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) for i := uint64(628); i < 2683301; i++ { - bm0.Add(i) + if _, err := bm0.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } + } + if _, err := bm0.Add(2683307); err != nil { + t.Fatalf("adding bits: %v", err) } - bm0.Add(2683307) if n := bm0.CountRange(1, 2683311); n != 2682674 { t.Fatalf("unexpected n: %d", n) } @@ -389,7 +407,9 @@ func TestBitmap_Intersection(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } result := bm0.Intersect(bm1) @@ -403,9 +423,13 @@ func TestBitmap_Union1(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } + } + if _, err := bm1.Add(4000000); err != nil { + t.Fatalf("adding bits: %v", err) } - bm1.Add(4000000) result := bm0.Union(bm1) if n := result.Count(); n != 2682675 { @@ -429,9 +453,13 @@ func TestBitmap_UnionInPlace1(t *testing.T) { result = roaring.NewBitmap() ) for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } + } + if _, err := bm1.Add(4000000); err != nil { + t.Fatalf("adding bits: %v", err) } - bm1.Add(4000000) result.UnionInPlace(bm0, bm1) if n := result.Count(); n != 2682675 { @@ -504,7 +532,9 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) { // size of a container to ensure we generate a maxRange container. for x := start; x < (start + 2*(0xffff+1)); x++ { set[uint64(x)] = struct{}{} - bitmap.Add(uint64(x)) + if _, err := bitmap.Add(uint64(x)); err != nil { + t.Fatalf("adding bits: %v", err) + } } } @@ -513,7 +543,9 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) { for x := 0; x < numIntsPerBatch; x++ { num := uint64(rng.Intn(maxUint64Val)) set[num] = struct{}{} - bitmap.Add(num) + if _, err := bitmap.Add(num); err != nil { + t.Fatalf("adding bits: %v", err) + } } sets = append(sets, set) @@ -588,12 +620,16 @@ func TestBitmap_IntersectArrayArray(t *testing.T) { func TestBitmap_IntersectBitmapBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i += 2 { - bm0.Add(i) + if _, err := bm0.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i += 3 { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } result := bm0.Intersect(bm1) @@ -620,7 +656,9 @@ func TestBitmap_IntersectRunRun(t *testing.T) { offset := (runLen / 2) + spaceLen for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) { for j := uint64(0); j < runLen; j++ { - bm2.Add(offset + i + j) + if _, err := bm2.Add(offset + i + j); err != nil { + t.Fatalf("adding bits: %v", err) + } } } bm2.Optimize() // convert to runs @@ -629,7 +667,9 @@ func TestBitmap_IntersectRunRun(t *testing.T) { spaceLen = uint64(1) for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) { for j := uint64(0); j < runLen; j++ { - bm3.Add(i + j) + if _, err := bm3.Add(i + j); err != nil { + t.Fatalf("adding bits: %v", err) + } } } bm3.Optimize() // convert to runs @@ -643,7 +683,7 @@ func TestBitmap_Difference(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + _, _ = bm1.Add(i) } result := bm0.Difference(bm1) if n := result.Count(); n != 1 { @@ -771,7 +811,7 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098) bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 10000; i += 2 { - bm1.Add(i) + _, _ = bm1.Add(i) } result := bm0.Xor(bm1) @@ -802,11 +842,11 @@ func TestBitmap_Xor_BitmapBitmap(t *testing.T) { bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 10000; i += 2 { - bm1.Add(i) + _, _ = bm1.Add(i) } for i := uint64(1); i < 10000; i += 2 { - bm0.Add(i) + _, _ = bm0.Add(i) } result := bm0.Xor(bm1) @@ -847,7 +887,9 @@ func TestBitmap_Flip_Bitmap(t *testing.T) { bm := roaring.NewFileBitmap() size := uint64(10000) for i := uint64(0); i < size; i += 2 { - bm.Add(i) + if _, err := bm.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } results := bm.Flip(0, size-1) if n := results.Count(); n != size/2 { @@ -921,7 +963,7 @@ func TestBitmap_IntersectionCount_RunRun(t *testing.T) { func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { bm0 := roaring.NewFileBitmap() for i := uint64(3); i <= 1000006; i += 2 { - bm0.Add(i) + _, _ = bm0.Add(i) } bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs @@ -938,7 +980,7 @@ func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098) bm1 := roaring.NewFileBitmap() for i := uint64(0); i <= 10000; i += 2 { - bm1.Add(i) + _, _ = bm1.Add(i) } if n := bm0.IntersectionCount(bm1); n != 3 { @@ -953,15 +995,15 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap() bm1 := roaring.NewFileBitmap() for i := uint64(0); i <= 10000; i += 2 { - bm0.Add(i) - bm1.Add(i + 1) + _, _ = bm0.Add(i) + _, _ = bm1.Add(i + 1) } - bm0.Add(1000) - bm1.Add(1000) + _, _ = bm0.Add(1000) + _, _ = bm1.Add(1000) - bm0.Add(2000) - bm1.Add(2000) + _, _ = bm0.Add(2000) + _, _ = bm1.Add(2000) if n := bm0.IntersectionCount(bm1); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -1005,7 +1047,7 @@ func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, ma // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { - quick.Check(func(a []uint64) bool { + err := quick.Check(func(a []uint64) bool { bm := roaring.NewFileBitmap() m := make(map[uint64]struct{}) @@ -1067,6 +1109,9 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, false, rand)) }, }) + if err != nil { + t.Fatalf("quick check failed: %v", err) + } } func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) } @@ -1091,7 +1136,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { t.Skip("short") } - quick.Check(func(a0, a1 []uint64) bool { + err := quick.Check(func(a0, a1 []uint64) bool { // Create bitmap with initial values set. bm := roaring.NewFileBitmap(a0...) @@ -1145,6 +1190,9 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { values[1] = reflect.ValueOf(GenerateUint64Slice(100, min, max, sorted, rand)) }, }) + if err != nil { + t.Fatalf("quick check failed: %v", err) + } } // Ensure iterator can iterate over all the values on the bitmap. @@ -1167,13 +1215,13 @@ func TestIterator(t *testing.T) { t.Run("run", func(t *testing.T) { bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 11; i += 1 { - bm1.Add(i) + _, _ = bm1.Add(i) } bm1.Optimize() bm2 := roaring.NewFileBitmap() for i := uint64(0); i < 12; i += 1 { - bm2.Add(i) + _, _ = bm2.Add(i) } bm2.Optimize() @@ -1203,23 +1251,24 @@ func TestIterator(t *testing.T) { // testBM creates a bitmap with 3 containers: array, bitmap, and run. func testBM() *roaring.Bitmap { - + // We should possibly be testing the adds for errors, but we + // don't have a clean way to return an error, so we don't right now. bm := roaring.NewFileBitmap() //the array for i := uint64(0); i < 1024; i += 4 { - bm.Add((1 << 16) + i) + _, _ = bm.Add((1 << 16) + i) } //the bitmap for i := uint64(0); i < 16384; i += 2 { - bm.Add((2 << 16) + i) + _, _ = bm.Add((2 << 16) + i) } //small run for i := uint64(0); i < 1024; i += 1 { - bm.Add((3 << 16) + i) + _, _ = bm.Add((3 << 16) + i) } //large run for i := uint64(0); i < 65535; i += 1 { - bm.Add((4 << 16) + i) + _, _ = bm.Add((4 << 16) + i) } bm.Optimize() //count 75007 @@ -1275,9 +1324,13 @@ func isAllType(b *roaring.Bitmap, typ string) bool { return true } +// getBenchData yields some sample data func getBenchData(tb testing.TB) *benchmarkSampleData { data := &sampleData if data.a1 == nil { + // throughout this, we ignore any errors from bitmap adds, + // because errors in those should result in the Optimize + // pass producing the wrong values, so we can just check there. const max = (1 << 24) / 64 // Build bitmap with array container. @@ -1285,28 +1338,28 @@ func getBenchData(tb testing.TB) *benchmarkSampleData { data.a2 = roaring.NewFileBitmap() // two lists of different lengths for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ { - data.a1.Add(uint64(rand.Intn(max))) - data.a2.Add(uint64(rand.Intn(max))) + _, _ = data.a1.Add(uint64(rand.Intn(max))) + _, _ = data.a2.Add(uint64(rand.Intn(max))) } for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ { - data.a1.Add(uint64(rand.Intn(max))) + _, _ = data.a1.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. data.b = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal/3; i < n; i++ { - data.b.Add(uint64(i * 3)) + _, _ = data.b.Add(uint64(i * 3)) } // build bitmap with run container data.r1 = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { - data.r1.Add(uint64(i)) + _, _ = data.r1.Add(uint64(i)) } // build bitmap with multiple runs data.r2 = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { - data.r2.Add(uint64(i)) + _, _ = data.r2.Add(uint64(i)) // break the runs up, this should produce 16 runs, which // is small enough to make RLE tempting if i&0xfff == 0xfff { @@ -1467,7 +1520,7 @@ func BenchmarkContainerLinear(b *testing.B) { bm := bmMaker() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1483,7 +1536,7 @@ func BenchmarkContainerReverse(b *testing.B) { bm := bmMaker() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1498,7 +1551,7 @@ func BenchmarkContainerColumn(b *testing.B) { bm := bmMaker() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1514,8 +1567,8 @@ func BenchmarkContainerOutsideIn(b *testing.B) { bm := bmMaker() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) - bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1532,8 +1585,8 @@ func BenchmarkContainerInsideOut(b *testing.B) { bm := bmMaker() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { - bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) - bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1545,7 +1598,7 @@ func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() for col := uint64(0); col < pilosa.ShardWidth; col++ { - bm.Add(col) + _, _ = bm.Add(col) } } } @@ -1554,9 +1607,9 @@ func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { - bm.Add(col) + _, _ = bm.Add(col) } - bm.Add(0) + _, _ = bm.Add(0) } } @@ -1565,14 +1618,14 @@ func BenchmarkSliceAscendingStriped(b *testing.B) { bm := roaring.NewFileBitmap() l := uint64(pilosa.ShardWidth / 8) for col := uint64(0); col < l; col++ { - bm.Add(l*0 + col) - bm.Add(l*1 + col) - bm.Add(l*2 + col) - bm.Add(l*3 + col) - bm.Add(l*4 + col) - bm.Add(l*5 + col) - bm.Add(l*6 + col) - bm.Add(l*7 + col) + _, _ = bm.Add(l*0 + col) + _, _ = bm.Add(l*1 + col) + _, _ = bm.Add(l*2 + col) + _, _ = bm.Add(l*3 + col) + _, _ = bm.Add(l*4 + col) + _, _ = bm.Add(l*5 + col) + _, _ = bm.Add(l*6 + col) + _, _ = bm.Add(l*7 + col) } } } @@ -1582,14 +1635,14 @@ func BenchmarkSliceDescendingStriped(b *testing.B) { bm := roaring.NewFileBitmap() l := uint64(pilosa.ShardWidth / 8) for col := uint64(l); col < l+1; col-- { - bm.Add(l*7 + col) - bm.Add(l*6 + col) - bm.Add(l*5 + col) - bm.Add(l*4 + col) - bm.Add(l*3 + col) - bm.Add(l*2 + col) - bm.Add(l*1 + col) - bm.Add(l*0 + col) + _, _ = bm.Add(l*7 + col) + _, _ = bm.Add(l*6 + col) + _, _ = bm.Add(l*5 + col) + _, _ = bm.Add(l*4 + col) + _, _ = bm.Add(l*3 + col) + _, _ = bm.Add(l*2 + col) + _, _ = bm.Add(l*1 + col) + _, _ = bm.Add(l*0 + col) } } } diff --git a/server.go b/server.go index 3e989e530..a4d05b144 100644 --- a/server.go +++ b/server.go @@ -559,7 +559,7 @@ func (s *Server) receiveMessage(m Message) error { return err } case *SetCoordinatorMessage: - s.cluster.setCoordinator(obj.New) + return s.cluster.setCoordinator(obj.New) case *UpdateCoordinatorMessage: s.cluster.updateCoordinator(obj.New) case *NodeStateMessage: @@ -705,7 +705,10 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.EnrichWithMemoryInfo() s.diagnostics.EnrichWithSchemaProperties() - s.diagnostics.CheckVersion() + err = s.diagnostics.CheckVersion() + if err != nil { + s.logger.Printf("can't check version: %v", err) + } err = s.diagnostics.Flush() if err != nil { s.logger.Printf("diagnostics error: %s", err) diff --git a/server/handler_test.go b/server/handler_test.go index ff9654933..082869081 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -846,7 +846,10 @@ func TestClusterTranslator(t *testing.T) { cluster := make(test.Cluster, 2) cluster[0] = test.NewCommandNode(true) cluster[0].Config.Gossip.Port = "0" - cluster[0].Start() + err := cluster[0].Start() + if err != nil { + t.Fatalf("starting cluster 1: %v", err) + } httpTranslateStore := http.NewTranslateStore(cluster[0].URL()) cluster[1] = test.NewCommandNode(false, server.OptCommandServerOptions( @@ -855,7 +858,10 @@ func TestClusterTranslator(t *testing.T) { ) cluster[1].Config.Gossip.Port = "0" cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()} - cluster[1].Start() + err = cluster[1].Start() + if err != nil { + t.Fatalf("starting cluster 1: %v", err) + } test.MustDo("POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}") test.MustDo("POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}") diff --git a/server/server_test.go b/server/server_test.go index 1cc3bd87f..19b79c6ea 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -900,7 +900,7 @@ func TestClusterExhaustingConnections(t *testing.T) { return nil }) } - err := eg.Wait() + err = eg.Wait() if err != nil { t.Fatalf("setting lots of shards: %v", err) } @@ -929,7 +929,10 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { bm := roaring.NewBitmap() bm.DirectAdd(0) buf := &bytes.Buffer{} - bm.WriteTo(buf) + _, err := bm.WriteTo(buf) + if err != nil { + t.Fatalf("writing to buffer: %v", err) + } data := buf.Bytes() eg := errgroup.Group{} diff --git a/utils_internal_test.go b/utils_internal_test.go index 321f4846d..3396cb4e9 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -57,15 +57,15 @@ func NewTestCluster(n int) *cluster { // NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { uri := defaultURI() - uri.setScheme(scheme) - uri.setHost(host) + _ = uri.setScheme(scheme) + _ = uri.setHost(host) uri.SetPort(port) return *uri } func NewTestURIFromHostPort(host string, port uint16) URI { uri := defaultURI() - uri.setHost(host) + _ = uri.setHost(host) uri.SetPort(port) return *uri } diff --git a/view.go b/view.go index 5780e682c..b1e5dee78 100644 --- a/view.go +++ b/view.go @@ -173,7 +173,7 @@ func (v *view) availableShards() *roaring.Bitmap { b := roaring.NewBitmap() for shard := range v.fragments { - b.Add(shard) // ignore error, no writer attached + _, _ = b.Add(shard) // ignore error, no writer attached } return b } From d5907b2a2e86aafcaf70a5be794512cc23a23777 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 29 Mar 2019 17:29:21 -0500 Subject: [PATCH 23/73] lint fixes to cluster behavior in utils test This is more lint fixes, but it's less obvious to me what the right handling for errors is, or whether disregarding them is safe, so it's a separate commit. --- utils_internal_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/utils_internal_test.go b/utils_internal_test.go index 3396cb4e9..2f21fb04b 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -235,7 +235,9 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) // add nodes if saveTopology { for _, n := range t.common.Nodes { - c.addNode(n) + if err := c.addNode(n); err != nil { + return nil, err + } } } @@ -314,7 +316,10 @@ func (b bcast) SendSync(m Message) error { // Apply the send message to all nodes (except the coordinator). for _, c := range b.t.Clusters { if c != b.c { - c.mergeClusterStatus(obj) + err := c.mergeClusterStatus(obj) + if err != nil { + return err + } } } b.t.mu.RLock() @@ -348,7 +353,9 @@ func (b bcast) SendTo(to *Node, m Message) error { } case *ResizeInstructionComplete: coord := b.t.clusterByID(to.ID) - go coord.markResizeInstructionComplete(obj) + // this used to be async, but that prevented us from checking + // its error status... + return coord.markResizeInstructionComplete(obj) case *ClusterStatus: // Apply the send message to the node. for _, c := range b.t.Clusters { From 2c6eb6689588d256407bef51cc0103023060edd8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 1 Apr 2019 15:56:14 -0500 Subject: [PATCH 24/73] check for slightly fewer errors json.Decoder.Decode() can yield io.EOF which is not actually an error. This appears to have caused a number of indirect test failures by making ImportRoaring generally report failure. --- http/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/http/client.go b/http/client.go index f8ad7cb37..4f2f1c208 100644 --- a/http/client.go +++ b/http/client.go @@ -630,7 +630,8 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind dec := json.NewDecoder(resp.Body) rbody := &pilosa.ImportResponse{} err = dec.Decode(rbody) - if err != nil { + // Decode can return EOF when no error occurred. helpful! + if err != nil && err != io.EOF { return errors.Wrap(err, "decoding response body") } if rbody.Err != "" { From 79451bd53cc3be36d4c8e788465a4cf4642197d4 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 1 Apr 2019 19:05:47 -0500 Subject: [PATCH 25/73] undo accidental change to test case contents --- ctl/import_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index b20919203..01abf372c 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -313,7 +313,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6")) + _, err = file.Write([]byte("foo1,2\nfoo3,4\nfoo5,6")) if err != nil { t.Fatalf("writing to tempfile: %v", err) } @@ -355,7 +355,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { if err != nil { t.Fatalf("creating tempfile: %v", err) } - _, err = file.Write([]byte("1,2\n3,4\n5,6")) + _, err = file.Write([]byte("a,2\n3,5\n5,6")) if err != nil { t.Fatalf("writing to tempfile: %v", err) } From c9cebe21bf2216d3e5f3163bfb039479a43b5b52 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 1 Apr 2019 19:34:34 -0500 Subject: [PATCH 26/73] unbreak holder node ID logic The attempt to fix up the logic broke returns from loadNodeID() in some cases, because it was overwriting the node ID generated in the IsNotExist case. --- holder.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/holder.go b/holder.go index 40b21e538..41ca16a9d 100644 --- a/holder.go +++ b/holder.go @@ -587,27 +587,23 @@ func (h *Holder) setFileLimit() { func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, ".id") - nodeID := "" h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { return "", errors.Wrap(err, "creating directory") } nodeIDBytes, err := ioutil.ReadFile(idPath) - // apparently it's safe to call IsNotExist on something that might - // be nil: - // https://github.com/golang/go/issues/31065 - if os.IsNotExist(err) { - nodeID = uuid.NewV4().String() - err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) - if err != nil { - return "", errors.Wrap(err, "writing file") - } - } else if err != nil { + if err == nil { + return strings.TrimSpace(string(nodeIDBytes)), nil + } + if !os.IsNotExist(err) { return "", errors.Wrap(err, "reading file") } - nodeID = strings.TrimSpace(string(nodeIDBytes)) - + nodeID := uuid.NewV4().String() + err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) + if err != nil { + return "", errors.Wrap(err, "writing file") + } return nodeID, nil } From 9fc8a5b352e5b03a6373857e264a24463e9b9b9d Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 1 Apr 2019 19:47:59 -0500 Subject: [PATCH 27/73] handle a specific error that might be an expected error I'm honestly not sure here. --- ctl/inspect_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go index 265d77a12..69efe07fa 100644 --- a/ctl/inspect_test.go +++ b/ctl/inspect_test.go @@ -41,7 +41,7 @@ func TestInspectCommand_Run(t *testing.T) { file.Close() cm.Path = file.Name() err = cm.Run(context.Background()) - if err != nil { + if err != nil && err.Error() != "unmarshalling: reading roaring header: did not find expected serialCookie in header" { t.Fatalf("can't run command: %v", err) } From f15347064f81691a824840a65f43d04c83843e0b Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 1 Apr 2019 22:15:19 -0500 Subject: [PATCH 28/73] fix race in cluster state transition The anonymous goroutine, if it gets an error, can race with other changes. Make the values we intend to call it on parameters so it will work with those even if other things are happening. --- cluster.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 024c426a1..4f865dc36 100644 --- a/cluster.go +++ b/cluster.go @@ -1892,12 +1892,12 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { for _, node := range officialNodes { if node.ID == c.Node.ID && node.State != c.Node.State { c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State) - go func() { - err := c.setNodeState(c.Node.State) + go func(fromState, toState string) { + err := c.setNodeState(toState) if err != nil { - c.logger.Printf("error setting node state from %v to %v: %v", node.State, c.Node.State, err) + c.logger.Printf("error setting node state from %v to %v: %v", fromState, toState, err) } - }() + }(node.State, c.Node.State) } if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") From e61e62a695abe739e46ea4733d6e186461247c5f Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 5 Apr 2019 15:19:53 -0500 Subject: [PATCH 29/73] don't run gometalinter on CI anymore gometalinter is slow, golangci-lint is fast and checks a lot more things, let's just use that. We leave the old targets in the Makefile for now so we can use them for sanity-checking the results. --- .circleci/config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb47ffcf8..415dd4411 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,12 +24,6 @@ jobs: - persist_to_workspace: root: . paths: "*" - linter: - <<: *defaults - steps: - - *fast-checkout - - run: make install-gometalinter - - run: make gometalinter check-license-headers: <<: *defaults steps: From babcf8c33122cd3fb5414a266a39a74ba6aea7ee Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 8 Apr 2019 11:05:02 -0500 Subject: [PATCH 30/73] continue having a linter target --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 415dd4411..d55e8704b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: steps: - *fast-checkout - run: make check-license-headers - golangci-lint: + linter: <<: *defaults steps: - *fast-checkout From 578ac7601180741f3622003dbd3235c6fafe35c8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 8 Apr 2019 11:33:21 -0500 Subject: [PATCH 31/73] don't call the golangci-lint workflow anymore If we're renaming golangci-lint to linter (since it's now our default linter), we no longer have a workflow named golangci-lint, so we shouldn't be calling it or requiring it from other workflows. --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d55e8704b..c7fe6e11b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,9 +132,6 @@ workflows: - check-license-headers: requires: - setup - - golangci-lint: - requires: - - setup - test-build-arm: requires: - setup @@ -162,7 +159,6 @@ workflows: requires: - linter - check-license-headers - - golangci-lint - test-golang-1.12 filters: tags: @@ -177,4 +173,3 @@ workflows: - linter - check-license-headers - test-golang-1.12 - - golangci-lint From ae17fcef7ee6d3cc51bb3dcd599c6fa6f9d9541a Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 9 Apr 2019 17:05:31 -0500 Subject: [PATCH 32/73] refix a lint Another test change made an `err :=` fail because it's no longer declaring a new variable, but another one needed the :. Or a patch applied incorrectly. It is a mystery. --- server/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 19b79c6ea..e911b25e6 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -900,7 +900,7 @@ func TestClusterExhaustingConnections(t *testing.T) { return nil }) } - err = eg.Wait() + err := eg.Wait() if err != nil { t.Fatalf("setting lots of shards: %v", err) } @@ -955,7 +955,7 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { return nil }) } - err := eg.Wait() + err = eg.Wait() if err != nil { t.Fatalf("setting lots of shards: %v", err) } From 302830ed6069198db2f3bfeadaf6219c6bd442cf Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 9 Apr 2019 17:10:44 -0500 Subject: [PATCH 33/73] fix lint in btree_test --- roaring/btree_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 67fd49e68..c12892f62 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -101,16 +101,16 @@ func (t *tree) dump() string { n = i + 1 } } - f.Format("%sX#%d(%p) n %d:%d {", pref, h, x, x.c, n) + _, _ = f.Format("%sX#%d(%p) n %d:%d {", pref, h, x, x.c, n) a := []interface{}{} for i, v := range x.x[:n] { a = append(a, v.ch) if i != 0 { - f.Format(" ") + _, _ = f.Format(" ") } - f.Format("(C#%d K %v)", handle(v.ch), v.k) + _, _ = f.Format("(C#%d K %v)", handle(v.ch), v.k) } - f.Format("}\n") + _, _ = f.Format("}\n") for _, p := range a { pagedump(p, pref+". ") } @@ -122,14 +122,14 @@ func (t *tree) dump() string { n = i + 1 } } - f.Format("%sD#%d(%p) P#%d N#%d n %d:%d {", pref, h, x, handle(x.p), handle(x.n), x.c, n) + _, _ = f.Format("%sD#%d(%p) P#%d N#%d n %d:%d {", pref, h, x, handle(x.p), handle(x.n), x.c, n) for i, d := range x.d[:n] { if i != 0 { - f.Format(" ") + _, _ = f.Format(" ") } - f.Format("%v:%v", d.k, d.v) + _, _ = f.Format("%v:%v", d.k, d.v) } - f.Format("}\n") + _, _ = f.Format("}\n") } } From fc5fc4151b942ad9b7df0bf8a9625885bd5ffc84 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 9 Apr 2019 17:10:54 -0500 Subject: [PATCH 34/73] add missing error check --- utils_internal_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils_internal_test.go b/utils_internal_test.go index 2f21fb04b..76e70fb9e 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -360,7 +360,10 @@ func (b bcast) SendTo(to *Node, m Message) error { // Apply the send message to the node. for _, c := range b.t.Clusters { if c.Node.ID == to.ID { - c.mergeClusterStatus(obj) + err := c.mergeClusterStatus(obj) + if err != nil { + return err + } } } b.t.mu.RLock() From 449c853850892762052ca6507a2fda90a988e583 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 16 Apr 2019 11:24:07 -0500 Subject: [PATCH 35/73] address meta-lint or half-baked lint fixes Clean up some spelling and consistency issues for the lint fixes. --- boltdb/attrstore.go | 4 ++-- cluster_internal_test.go | 17 ++++++----------- cmd/root.go | 7 +++++-- ctl/generate_config.go | 2 +- ctl/inspect.go | 2 +- ctl/inspect_test.go | 2 +- field_test.go | 2 +- 7 files changed, 17 insertions(+), 19 deletions(-) diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 280275a4d..e8c770fde 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -239,7 +239,7 @@ func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) { return nil }) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting blocks") } return blocks, nil } @@ -271,7 +271,7 @@ func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, er return nil }) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting block data") } return m, nil } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 84392ffd9..2a557694d 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -710,8 +710,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { tc := NewClusterCluster(0) - err := tc.addNode() - if err != nil { + if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } node0 := tc.Clusters[0] @@ -736,13 +735,11 @@ func TestCluster_ResizeStates(t *testing.T) { // Expect an error by adding a node not in the topology. expectedError := "host is not in topology: node1" - err = tc.addNode() - if err == nil || err.Error() != expectedError { + if err := tc.addNode(); err == nil || err.Error() != expectedError { t.Errorf("did not receive expected error: %s", expectedError) } - err = tc.addNode() - if err != nil { + if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } node2 := tc.Clusters[2] @@ -762,14 +759,13 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, with data", func(t *testing.T) { tc := NewClusterCluster(0) - err := tc.addNode() - if err != nil { + if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } node0 := tc.Clusters[0] // Open TestCluster. - if err = tc.Open(); err != nil { + if err := tc.Open(); err != nil { t.Fatal(err) } @@ -792,8 +788,7 @@ func TestCluster_ResizeStates(t *testing.T) { node0Checksum := node0Fragment.Checksum() // addNode needs to block until the resize process has completed. - err = tc.addNode() - if err != nil { + if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } node1 := tc.Clusters[1] diff --git a/cmd/root.go b/cmd/root.go index a8f8e6ddc..3ff5c22e4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,11 +51,14 @@ Build Time: ` + pilosa.BuildTime + "\n", } // return "dry run" error if "dry-run" flag is set - if ret, err := cmd.Flags().GetBool("dry-run"); ret && err == nil { + ret, err := cmd.Flags().GetBool("dry-run") + if err != nil { + return fmt.Errorf("problem getting dry-run flag: %v", err) + } + if ret { if cmd.Parent() != nil { return fmt.Errorf("dry run") } - return fmt.Errorf("problem getting dry-run flag: %v", err) } return nil diff --git a/ctl/generate_config.go b/ctl/generate_config.go index 0e59ce097..5a7e0db9f 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -42,7 +42,7 @@ func (cmd *GenerateConfigCommand) Run(_ context.Context) error { conf := server.NewConfig() ret, err := toml.Marshal(*conf) if err != nil { - return errors.Wrap(err, "unmarshaling default config") + return errors.Wrap(err, "unmarshalling default config") } fmt.Fprintf(cmd.Stdout, "%s\n", ret) return nil diff --git a/ctl/inspect.go b/ctl/inspect.go index 204ffab3b..98222d0de 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -72,7 +72,7 @@ func (cmd *InspectCommand) Run(_ context.Context) error { }() // Attach the mmap file to the bitmap. t := time.Now() - fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") + fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...") bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { return errors.Wrap(err, "unmarshalling") diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go index 69efe07fa..bb87f894d 100644 --- a/ctl/inspect_test.go +++ b/ctl/inspect_test.go @@ -51,7 +51,7 @@ func TestInspectCommand_Run(t *testing.T) { if err != nil { t.Fatalf("copying data: %v", err) } - if !strings.Contains(buf.String(), "unmarshaling bitmap...") { + if !strings.Contains(buf.String(), "unmarshalling bitmap...") { t.Fatalf("Inspect doesn't work: %s", err) } diff --git a/field_test.go b/field_test.go index 5911abd6a..88a3f3569 100644 --- a/field_test.go +++ b/field_test.go @@ -219,7 +219,7 @@ func TestField_AvailableShards(t *testing.T) { for i := uint64(0); i < 5; i++ { err := f.RemoveAvailableShard(i) if err != nil { - t.Fatalf("removing shard: %v", err) + t.Fatalf("removing shard %d: %v", i, err) } } if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { From e0a9fd72b784fb105ba1d5abcb36bffe32cf2edc Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 16 Apr 2019 14:38:33 -0500 Subject: [PATCH 36/73] Release v1.3.0 --- CHANGELOG.md | 62 ++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 2 +- docs/installation.md | 22 ++++++++-------- 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b558ac7..531116f0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,68 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.3.0] - 2019-04-16 + +This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. + +### Added +- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) +- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) +- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) +- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) +- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) +- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) +- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) +- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) +- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) +- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) +- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) +- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) +- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) +- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) +- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) +- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) + +### Fixed +- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) +- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) +- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) +- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) +- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) +- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) +- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) +- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) +- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) +- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) +- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) +- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) +- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) +- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) +- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) +- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) +- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) +- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) + +### Changed +- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) +- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) +- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) +- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) +- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) +- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) +- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) +- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) + +### Performance +- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) +- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) +- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) +- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) +- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) +- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) +- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) +- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) + ## [1.2.0] - 2018-12-20 This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. diff --git a/Dockerfile b/Dockerfile index 9914401d8..afbda01f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.11.5 as builder +FROM golang:1.12.4 as builder COPY . pilosa diff --git a/docs/installation.md b/docs/installation.md index 69de1376e..bebc5d53d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.3.0/pilosa-v1.3.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.2.0-darwin-amd64.tar.gz + tar xfz pilosa-v1.3.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.2.0-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.3.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -201,19 +201,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.3.0/pilosa-v1.3.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.2.0-linux-amd64.tar.gz + tar xfz pilosa-v1.3.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.2.0-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.3.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -230,7 +230,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -293,7 +293,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: From b46ff7b990b32b3142037d6e1f22ffdc9373ffe3 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 17 Apr 2019 18:10:05 -0500 Subject: [PATCH 37/73] fix some lint warnings raised in VS-Code --- api.go | 14 ++++++++--- client.go | 1 + cluster.go | 25 +++++++++++++++++++ diagnostics.go | 6 ++--- event.go | 1 + executor.go | 27 ++++++++++++--------- field.go | 24 +++++++++++++++++- fragment.go | 3 +-- fragment_internal_test.go | 2 +- gc.go | 2 +- handler.go | 18 ++++++++++++++ pilosa.go | 6 +++-- row.go | 1 + server.go | 51 ++++++++++++++++++++++++++++++++++++--- translate.go | 13 ++++++++++ uri.go | 4 +++ 16 files changed, 169 insertions(+), 29 deletions(-) diff --git a/api.go b/api.go index 19e41732d..db55334a3 100644 --- a/api.go +++ b/api.go @@ -96,7 +96,7 @@ func (api *API) validate(f apiMethod) error { if _, ok := validAPIMethods[state][f]; ok { return nil } - return newApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) + return newAPIMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) } // Query parses a PQL query out of the request and executes it. @@ -617,7 +617,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return nil } -// PostClusterMessage is for internal use. It decodes a protobuf message out of +// ClusterMessage is for internal use. It decodes a protobuf message out of // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { span, _ := tracing.StartSpanFromContext(ctx, "API.ClusterMessage") @@ -712,7 +712,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri return errors.Wrap(err, "sending DeleteView message") } -// IndexAttrDiff +// IndexAttrDiff determines the local column attribute data blocks which differ from those provided. func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.IndexAttrDiff") defer span.Finish() @@ -750,6 +750,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At return attrs, nil } +// FieldAttrDiff determines the local row attribute data blocks which differ from those provided. func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.FieldAttrDiff") defer span.Finish() @@ -796,6 +797,8 @@ type ImportOptions struct { // ImportOption is a functional option type for API.Import. type ImportOption func(*ImportOptions) error +// OptImportOptionsClear is a functional option on ImportOption +// used to specify whether the import is a set or clear operation. func OptImportOptionsClear(c bool) ImportOption { return func(o *ImportOptions) error { o.Clear = c @@ -803,6 +806,8 @@ func OptImportOptionsClear(c bool) ImportOption { } } +// OptImportOptionsIgnoreKeyCheck is a functional option on ImportOption +// used to specify whether key check should be ignored. func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption { return func(o *ImportOptions) error { o.IgnoreKeyCheck = b @@ -1175,7 +1180,7 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } -// Info returns information about this server instance +// Info returns information about this server instance. func (api *API) Info() serverInfo { si := api.server.systemInfo // we don't report errors on failures to get this information @@ -1192,6 +1197,7 @@ func (api *API) Info() serverInfo { } } +// TranslateKeys handles a TranslateKeyRequest. func (api *API) TranslateKeys(body io.Reader) ([]byte, error) { reqBytes, err := ioutil.ReadAll(body) if err != nil { diff --git a/client.go b/client.go index 3d09c2e08..d537705c8 100644 --- a/client.go +++ b/client.go @@ -72,6 +72,7 @@ type InternalClient interface { //=============== +// InternalQueryClient is the internal interface for querying a node. type InternalQueryClient interface { QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) } diff --git a/cluster.go b/cluster.go index 4f865dc36..674a0a267 100644 --- a/cluster.go +++ b/cluster.go @@ -1969,12 +1969,16 @@ func (c *cluster) setStatic(hosts []string) error { return nil } +// ClusterStatus describes the status of the cluster including its +// state and node topology. type ClusterStatus struct { ClusterID string State string Nodes []*Node } +// ResizeInstruction contains the instruction provided to a node +// during a cluster resize operation. type ResizeInstruction struct { JobID int64 Node *Node @@ -1984,6 +1988,8 @@ type ResizeInstruction struct { ClusterStatus *ClusterStatus } +// ResizeSource is the source of data for a node acting on a +// ResizeInstruction. type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -2023,82 +2029,101 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { return t, nil } +// CreateShardMessage is an internal message indicating shard creation. type CreateShardMessage struct { Index string Field string Shard uint64 } +// CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { Index string Meta *IndexOptions } +// DeleteIndexMessage is an internal message indicating index deletion. type DeleteIndexMessage struct { Index string } +// CreateFieldMessage is an internal message indicating field creation. type CreateFieldMessage struct { Index string Field string Meta *FieldOptions } +// DeleteFieldMessage is an internal message indicating field deletion. type DeleteFieldMessage struct { Index string Field string } +// DeleteAvailableShardMessage is an internal message indicating available shard deletion. type DeleteAvailableShardMessage struct { Index string Field string ShardID uint64 } +// CreateViewMessage is an internal message indicating view creation. type CreateViewMessage struct { Index string Field string View string } + +// DeleteViewMessage is an internal message indicating view deletion. type DeleteViewMessage struct { Index string Field string View string } +// ResizeInstructionComplete is an internal message to the coordinator indicating +// that the resize instructions performed on a single node have completed. type ResizeInstructionComplete struct { JobID int64 Node *Node Error string } +// SetCoordinatorMessage is an internal message instructing nodes to honor a new coordinator. type SetCoordinatorMessage struct { New *Node } +// UpdateCoordinatorMessage is an internal message for reassigning the coordinator. type UpdateCoordinatorMessage struct { New *Node } +// NodeStateMessage is an internal message for broadcasting a node's state. type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } +// NodeStatus is an internal message representing the contents of a node. type NodeStatus struct { Node *Node Indexes []*IndexStatus Schema *Schema } +// IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { Name string Fields []*FieldStatus } +// FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string AvailableShards *roaring.Bitmap } +// RecalculateCaches is an internal message for recalculating all caches +// within a holder. type RecalculateCaches struct{} diff --git a/diagnostics.go b/diagnostics.go index 1f9466772..c16cd9b0d 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -230,11 +230,11 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() { for _, index := range d.server.holder.Indexes() { numShards += index.AvailableShards().Count() - numIndexes += 1 + numIndexes++ for _, field := range index.Fields() { - numFields += 1 + numFields++ if field.Type() == FieldTypeInt { - bsiFieldCount += 1 + bsiFieldCount++ } if field.TimeQuantum() != "" { timeQuantumEnabled = true diff --git a/event.go b/event.go index 0d5e59e99..b27bd1bf6 100644 --- a/event.go +++ b/event.go @@ -17,6 +17,7 @@ package pilosa // NodeEventType are the types of node events. type NodeEventType int +// Constant node event types. const ( NodeJoin NodeEventType = iota NodeLeave diff --git a/executor.go b/executor.go index 89c6d8228..ce30a32c6 100644 --- a/executor.go +++ b/executor.go @@ -987,6 +987,8 @@ type FieldRow struct { RowKey string `json:"rowKey,omitempty"` } +// MarshalJSON marshals FieldRow to JSON such that +// either a Key or an ID is included. func (fr FieldRow) MarshalJSON() ([]byte, error) { if fr.RowKey != "" { return json.Marshal(struct { @@ -1006,10 +1008,12 @@ func (fr FieldRow) MarshalJSON() ([]byte, error) { }) } +// String is the FieldRow stringer. func (fr FieldRow) String() string { return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey) } +// GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` @@ -1048,6 +1052,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount { return ret } +// Compare is used in ordering two GroupCount objects. func (g GroupCount) Compare(o GroupCount) int { for i := range g.Group { if g.Group[i].RowID < o.Group[i].RowID { @@ -1158,7 +1163,7 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s // views contains the list of views to inspect (and merge) // in order to represent `Rows` for the field. - var views []string = []string{viewStandard} + var views = []string{viewStandard} // Handle `time` fields. if f.Type() == FieldTypeTime { @@ -1706,11 +1711,11 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq } // Forward call to remote node otherwise. - if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil); err != nil { + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + if err != nil { return false, err - } else { - ret = res[0].(bool) } + ret = res[0].(bool) } return ret, nil } @@ -1982,11 +1987,11 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. } // Forward call to remote node otherwise. - if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil); err != nil { + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + if err != nil { return false, err - } else { - ret = res[0].(bool) } + ret = res[0].(bool) } return ret, nil } @@ -2017,11 +2022,11 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq } // Forward call to remote node otherwise. - if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil); err != nil { + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + if err != nil { return false, err - } else { - ret = res[0].(bool) } + ret = res[0].(bool) } return ret, nil } @@ -2881,7 +2886,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde return nil, errors.Wrap(err, "getting previous") } else if hasPrev && !ignorePrev { if i == len(children)-1 { - prev += 1 + prev++ } gbi.rowIters[i].Seek(prev) } diff --git a/field.go b/field.go index d83735893..b76a5af35 100644 --- a/field.go +++ b/field.go @@ -92,6 +92,8 @@ type Field struct { // FieldOption is a functional option type for pilosa.fieldOptions. type FieldOption func(fo *FieldOptions) error +// OptFieldKeys is a functional option on FieldOptions +// used to specify whether keys are used for this field. func OptFieldKeys() FieldOption { return func(fo *FieldOptions) error { fo.Keys = true @@ -99,6 +101,8 @@ func OptFieldKeys() FieldOption { } } +// OptFieldTypeDefault is a functional option on FieldOptions +// used to set the field type and cache setting to the default values. func OptFieldTypeDefault() FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -111,6 +115,9 @@ func OptFieldTypeDefault() FieldOption { } } +// OptFieldTypeSet is a functional option on FieldOptions +// used to specify the field as being type `set` and to +// provide any respective configuration values. func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -123,6 +130,9 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { } } +// OptFieldTypeInt is a functional option on FieldOptions +// used to specify the field as being type `int` and to +// provide any respective configuration values. func OptFieldTypeInt(min, max int64) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -138,7 +148,9 @@ func OptFieldTypeInt(min, max int64) FieldOption { } } -// OptFieldTypeTime sets the field type to time. +// OptFieldTypeTime is a functional option on FieldOptions +// used to specify the field as being type `time` and to +// provide any respective configuration values. // Pass true to skip creation of the standard view. func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { return func(fo *FieldOptions) error { @@ -155,6 +167,9 @@ func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { } } +// OptFieldTypeMutex is a functional option on FieldOptions +// used to specify the field as being type `mutex` and to +// provide any respective configuration values. func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -167,6 +182,9 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { } } +// OptFieldTypeBool is a functional option on FieldOptions +// used to specify the field as being type `bool` and to +// provide any respective configuration values. func OptFieldTypeBool() FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -1036,6 +1054,7 @@ func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { return int64(vmax) + bsig.Min, int64(vcount), nil } +// Range performs a conditional operation on Field. func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) @@ -1283,6 +1302,9 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { } } +// MarshalJSON marshals FieldOptions to JSON such that +// only those attributes associated to the field type +// are included. func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { case FieldTypeSet: diff --git a/fragment.go b/fragment.go index 2409a5fc5..c08571e78 100644 --- a/fragment.go +++ b/fragment.go @@ -50,7 +50,6 @@ import ( const ( // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. // shardWidthExponent = 20 // set in shardwidthNN.go files - ShardWidth = 1 << shardwidth.Exponent // shardVsContainerExponent is the power of 2 of ShardWith minus the power @@ -2299,7 +2298,7 @@ func (ri *rowIterator) Next() (r *Row, rowID uint64, wrapped bool) { } rowID = ri.rowIDs[ri.cur] r = ri.f.row(rowID) - ri.cur += 1 + ri.cur++ return r, rowID, wrapped } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 8c3d65c66..db0159734 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2757,7 +2757,7 @@ func calcTop(rowIDs, colIDs []uint64) []Pair { func calcExpected(inputs ...[]uint64) [][]uint64 { // create map of row id to set of column values in that row. rows := make(map[uint64]map[uint64]struct{}) - var maxrow uint64 = 0 + var maxrow uint64 for _, input := range inputs { for _, val := range input { row := val / ShardWidth diff --git a/gc.go b/gc.go index 1456c22c2..e5e5d7031 100644 --- a/gc.go +++ b/gc.go @@ -32,6 +32,6 @@ type nopGCNotifier struct{} func (n *nopGCNotifier) Close() {} // AfterGC is a no-op implementation of GCNotifier AfterGC method. -func (c *nopGCNotifier) AfterGC() <-chan struct{} { +func (n *nopGCNotifier) AfterGC() <-chan struct{} { return nil } diff --git a/handler.go b/handler.go index 9089fd778..f42cf1993 100644 --- a/handler.go +++ b/handler.go @@ -74,6 +74,8 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { }) } +// Handler is the interface for the data handler, a wrapper around +// Pilosa's data store. type Handler interface { Serve() error Close() error @@ -89,8 +91,11 @@ func (n nopHandler) Close() error { return nil } +// NopHandler is a no-op implementation of the Handler interface. var NopHandler Handler = nopHandler{} +// ImportValueRequest describes the import request structure +// for a value (BSI) import. type ImportValueRequest struct { Index string Field string @@ -100,6 +105,8 @@ type ImportValueRequest struct { Values []int64 } +// ImportRequest describes the import request structure +// for an import. type ImportRequest struct { Index string Field string @@ -111,15 +118,20 @@ type ImportRequest struct { Timestamps []int64 } +// ImportRoaringRequest describes the import request structure +// for an import containing roaring-encoded data. type ImportRoaringRequest struct { Clear bool Views map[string][]byte } +// ImportResponse is the structured response of an import. type ImportResponse struct { Err string } +// BlockDataRequest describes the structure of a request +// for fragment block data. type BlockDataRequest struct { Index string Field string @@ -128,17 +140,23 @@ type BlockDataRequest struct { Block uint64 } +// BlockDataResponse is the structured response of a block +// data request. type BlockDataResponse struct { RowIDs []uint64 ColumnIDs []uint64 } +// TranslateKeysRequest describes the structure of a request +// for a batch of key translations. type TranslateKeysRequest struct { Index string Field string Keys []string } +// TranslateKeysResponse is the structured response of a key +// translation request. type TranslateKeysResponse struct { IDs []uint64 } diff --git a/pilosa.go b/pilosa.go index d410e1fd4..88510cef9 100644 --- a/pilosa.go +++ b/pilosa.go @@ -77,8 +77,8 @@ type apiMethodNotAllowedError struct { error } -// newApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. -func newApiMethodNotAllowedError(err error) apiMethodNotAllowedError { +// newAPIMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. +func newAPIMethodNotAllowedError(err error) apiMethodNotAllowedError { return apiMethodNotAllowedError{err} } @@ -128,6 +128,8 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } +// MarshalJSON marshals the ColumnAttrSet to JSON such that +// either a Key or an ID is included. func (cas ColumnAttrSet) MarshalJSON() ([]byte, error) { if cas.Key != "" { return json.Marshal(struct { diff --git a/row.go b/row.go index 0a8fc0acd..e393a9db4 100644 --- a/row.go +++ b/row.go @@ -43,6 +43,7 @@ func NewRow(columns ...uint64) *Row { return r } +// IsEmpty returns true if the row doesn't contain any set bits. func (r *Row) IsEmpty() bool { if len(r.segments) == 0 { return true diff --git a/server.go b/server.go index a4d05b144..0e32a392a 100644 --- a/server.go +++ b/server.go @@ -75,6 +75,7 @@ type Server struct { // nolint: maligned dataDir string } +// Holder returns the holder for server. // TODO: have this return an interface for Holder instead of concrete object? func (s *Server) Holder() *Holder { return s.holder @@ -83,6 +84,8 @@ func (s *Server) Holder() *Holder { // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error +// OptServerLogger is a functional option on Server +// used to set the logger. func OptServerLogger(l logger.Logger) ServerOption { return func(s *Server) error { s.logger = l @@ -90,6 +93,8 @@ func OptServerLogger(l logger.Logger) ServerOption { } } +// OptServerReplicaN is a functional option on Server +// used to set the number of replicas. func OptServerReplicaN(n int) ServerOption { return func(s *Server) error { s.cluster.ReplicaN = n @@ -97,6 +102,8 @@ func OptServerReplicaN(n int) ServerOption { } } +// OptServerDataDir is a functional option on Server +// used to set the data directory. func OptServerDataDir(dir string) ServerOption { return func(s *Server) error { s.dataDir = dir @@ -104,6 +111,9 @@ func OptServerDataDir(dir string) ServerOption { } } +// OptServerAttrStoreFunc is a functional option on Server +// used to provide the function to use to generate a new +// attribute store. func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { return func(s *Server) error { s.holder.NewAttrStore = af @@ -111,6 +121,8 @@ func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { } } +// OptServerAntiEntropyInterval is a functional option on Server +// used to set the anti-entropy interval. func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { return func(s *Server) error { s.antiEntropyInterval = interval @@ -118,6 +130,8 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { } } +// OptServerLongQueryTime is a functional option on Server +// used to set long query duration. func OptServerLongQueryTime(dur time.Duration) ServerOption { return func(s *Server) error { s.cluster.longQueryTime = dur @@ -125,6 +139,8 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } +// OptServerMaxWritesPerRequest is a functional option on Server +// used to set the maximum number of writes allowed per request. func OptServerMaxWritesPerRequest(n int) ServerOption { return func(s *Server) error { s.maxWritesPerRequest = n @@ -132,6 +148,8 @@ func OptServerMaxWritesPerRequest(n int) ServerOption { } } +// OptServerMetricInterval is a functional option on Server +// used to set the interval between metric samples. func OptServerMetricInterval(dur time.Duration) ServerOption { return func(s *Server) error { s.metricInterval = dur @@ -139,6 +157,8 @@ func OptServerMetricInterval(dur time.Duration) ServerOption { } } +// OptServerSystemInfo is a functional option on Server +// used to set the system information source. func OptServerSystemInfo(si SystemInfo) ServerOption { return func(s *Server) error { s.systemInfo = si @@ -146,6 +166,8 @@ func OptServerSystemInfo(si SystemInfo) ServerOption { } } +// OptServerGCNotifier is a functional option on Server +// used to set the garbage collection notification source. func OptServerGCNotifier(gcn GCNotifier) ServerOption { return func(s *Server) error { s.gcNotifier = gcn @@ -153,6 +175,8 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } +// OptServerInternalClient is a functional option on Server +// used to set the implementation of InternalClient. func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { s.executor = newExecutor(optExecutorInternalQueryClient(c)) @@ -162,7 +186,7 @@ func OptServerInternalClient(c InternalClient) ServerOption { } } -// DEPRECATED +// OptServerPrimaryTranslateStore has been deprecated. func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { return func(s *Server) error { s.logger.Printf("DEPRECATED: OptServerPrimaryTranslateStore") @@ -170,6 +194,9 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { } } +// OptServerPrimaryTranslateStoreFunc is a functional option on Server +// used to specify the function used to create a new primary translate +// store. func OptServerPrimaryTranslateStoreFunc(tf func(interface{}) TranslateStore) ServerOption { return func(s *Server) error { @@ -178,6 +205,8 @@ func OptServerPrimaryTranslateStoreFunc(tf func(interface{}) TranslateStore) Ser } } +// OptServerStatsClient is a functional option on Server +// used to specify the stats client. func OptServerStatsClient(sc stats.StatsClient) ServerOption { return func(s *Server) error { s.holder.Stats = sc @@ -185,6 +214,8 @@ func OptServerStatsClient(sc stats.StatsClient) ServerOption { } } +// OptServerDiagnosticsInterval is a functional option on Server +// used to specify the duration between diagnostic checks. func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { return func(s *Server) error { s.diagnosticInterval = dur @@ -192,6 +223,8 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { } } +// OptServerURI is a functional option on Server +// used to set the server URI. func OptServerURI(uri *URI) ServerOption { return func(s *Server) error { s.uri = *uri @@ -199,7 +232,7 @@ func OptServerURI(uri *URI) ServerOption { } } -// OptClusterDisabled tells the server whether to use a static cluster with the +// OptServerClusterDisabled tells the server whether to use a static cluster with the // defined hosts. Mostly used for testing. func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { return func(s *Server) error { @@ -209,6 +242,8 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +// OptServerSerializer is a functional option on Server +// used to set the serializer. func OptServerSerializer(ser Serializer) ServerOption { return func(s *Server) error { s.serializer = ser @@ -216,6 +251,8 @@ func OptServerSerializer(ser Serializer) ServerOption { } } +// OptServerIsCoordinator is a functional option on Server +// used to specify whether or not this server is the coordinator. func OptServerIsCoordinator(is bool) ServerOption { return func(s *Server) error { s.isCoordinator = is @@ -223,6 +260,8 @@ func OptServerIsCoordinator(is bool) ServerOption { } } +// OptServerNodeID is a functional option on Server +// used to set the server node ID. func OptServerNodeID(nodeID string) ServerOption { return func(s *Server) error { s.nodeID = nodeID @@ -230,6 +269,9 @@ func OptServerNodeID(nodeID string) ServerOption { } } +// OptServerClusterHasher is a functional option on Server +// used to specify the consistent hash algorithm for data +// location within the cluster. func OptServerClusterHasher(h Hasher) ServerOption { return func(s *Server) error { s.cluster.Hasher = h @@ -237,6 +279,8 @@ func OptServerClusterHasher(h Hasher) ServerOption { } } +// OptServerTranslateFileMapSize is a functional option on Server +// used to specify the size of the translate file. func OptServerTranslateFileMapSize(mapSize int) ServerOption { return func(s *Server) error { s.holder.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) @@ -681,9 +725,8 @@ func (s *Server) monitorDiagnostics() { if s.diagnosticInterval < time.Minute { s.logger.Printf("diagnostics disabled") return - } else { - s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) } + s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) diff --git a/translate.go b/translate.go index 6a49e4a7a..3e4245ae1 100644 --- a/translate.go +++ b/translate.go @@ -33,6 +33,7 @@ import ( "github.com/pkg/errors" ) +// Log entry type constants. const ( LogEntryTypeInsertColumn = 1 LogEntryTypeInsertRow = 2 @@ -42,6 +43,7 @@ const ( defaultReplicationRetryInterval = 1 * time.Second ) +// Translate store errors. var ( ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") @@ -99,12 +101,17 @@ type TranslateFile struct { // TranslateFileOption is a functional option type for pilosa.TranslateFile type TranslateFileOption func(f *TranslateFile) error +// OptTranslateFileMapSize is a functional option on TranslateFile +// used to set the map size. func OptTranslateFileMapSize(mapSize int) TranslateFileOption { return func(f *TranslateFile) error { f.mapSize = mapSize return nil } } + +// OptTranslateFileLogger is a functional option on TranslateFile +// used to set the file logger. func OptTranslateFileLogger(l logger.Logger) TranslateFileOption { return func(s *TranslateFile) error { s.logger = l @@ -151,6 +158,7 @@ func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { return f } +// Open opens the translate file. func (s *TranslateFile) Open() (err error) { // Open writer & buffered writer. if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { @@ -232,6 +240,7 @@ func (s *TranslateFile) handlePrimaryStoreEvent(ev primaryStoreEvent) error { return nil } +// Close closes the translate file. func (s *TranslateFile) Close() (err error) { s.once.Do(func() { close(s.closing) @@ -588,6 +597,7 @@ func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (str return "", nil } +// TranslateRowsToUint64 converts a slice of row keys to a slice of row IDs. func (s *TranslateFile) TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) { key := fieldKey{index, field} @@ -679,6 +689,7 @@ func (s *TranslateFile) TranslateRowsToUint64(index, field string, values []stri return ret, nil } +// TranslateRowToString translates a row ID to a string key. func (s *TranslateFile) TranslateRowToString(index, field string, id uint64) (string, error) { s.mu.RLock() if idx := s.rows[fieldKey{index, field}]; idx != nil { @@ -700,6 +711,8 @@ func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser return rc, nil } +// LogEntry is a batch of Key/ID mappings which is replicated to other nodes +// for read-only key translation. type LogEntry struct { Type uint8 Index []byte diff --git a/uri.go b/uri.go index 231457691..00216afbd 100644 --- a/uri.go +++ b/uri.go @@ -56,8 +56,11 @@ func defaultURI() *URI { } } +// URIs is a convenience type representing a slice of URI. type URIs []URI +// HostPortStrings returns a slice of host:port strings +// based on the slice of URI. func (u URIs) HostPortStrings() []string { s := make([]string, len(u)) for i, a := range u { @@ -199,6 +202,7 @@ func (u *URI) MarshalJSON() ([]byte, error) { return json.Marshal(output) } +// UnmarshalJSON unmarshals a byte slice to a URI. func (u *URI) UnmarshalJSON(b []byte) error { var input struct { Scheme string `json:"scheme,omitempty"` From 3a07abdeae4ea68a67392bdb7fd9ede1ad0367f9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 22 Apr 2019 17:36:40 -0500 Subject: [PATCH 38/73] remove shard validation stuff it seems to have a bug where there is some race on cluster startup which can cause it to think that the node doesn't own any shards. --- field.go | 9 +------ holder.go | 8 ------ index.go | 7 ++--- server.go | 3 --- server_internal_test.go | 58 ----------------------------------------- view.go | 20 +++++--------- 6 files changed, 10 insertions(+), 95 deletions(-) diff --git a/field.go b/field.go index b76a5af35..0b2cf6cf5 100644 --- a/field.go +++ b/field.go @@ -58,10 +58,6 @@ const ( FieldTypeBool = "bool" ) -func defaultShardValidator(shard uint64) bool { - return true -} - // Field represents a container for views. type Field struct { mu sync.RWMutex @@ -84,7 +80,6 @@ type Field struct { // Shards with data on any node in the cluster, according to this node. remoteAvailableShards *roaring.Bitmap - shardValidator func(uint64) bool logger logger.Logger } @@ -230,8 +225,7 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { remoteAvailableShards: roaring.NewBitmap(), - shardValidator: defaultShardValidator, - logger: logger.NopLogger, + logger: logger.NopLogger, } return f, nil } @@ -780,7 +774,6 @@ func (f *Field) newView(path, name string) *view { view.rowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster - view.shardValidator = f.shardValidator return view } diff --git a/holder.go b/holder.go index 41ca16a9d..9384d6622 100644 --- a/holder.go +++ b/holder.go @@ -77,8 +77,6 @@ type Holder struct { // The interval at which the cached row ids are persisted to disk. cacheFlushInterval time.Duration - shardValidatorFunc func(index string, shard uint64) bool - Logger logger.Logger } @@ -125,9 +123,6 @@ func NewHolder() *Holder { NewAttrStore: newNopAttrStore, cacheFlushInterval: defaultCacheFlushInterval, - shardValidatorFunc: func(index string, shard uint64) bool { - return true //default - }, Logger: logger.NopLogger, } @@ -430,9 +425,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { index.broadcaster = h.broadcaster index.newAttrStore = h.NewAttrStore index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data")) - index.shardValidator = func(shard uint64) bool { - return h.shardValidatorFunc(name, shard) - } return index, nil } diff --git a/index.go b/index.go index da749ff2d..8b67f003c 100644 --- a/index.go +++ b/index.go @@ -50,9 +50,8 @@ type Index struct { // Column attribute storage and cache. columnAttrs AttrStore - broadcaster broadcaster - Stats stats.StatsClient - shardValidator func(uint64) bool + broadcaster broadcaster + Stats stats.StatsClient logger logger.Logger } @@ -75,7 +74,6 @@ func NewIndex(path, name string) (*Index, error) { broadcaster: NopBroadcaster, Stats: stats.NopStatsClient, logger: logger.NopLogger, - shardValidator: defaultShardValidator, trackExistence: true, }, nil } @@ -405,7 +403,6 @@ func (i *Index) newField(path, name string) (*Field, error) { f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) - f.shardValidator = i.shardValidator return f, nil } diff --git a/server.go b/server.go index 0e32a392a..ad81aa1b3 100644 --- a/server.go +++ b/server.go @@ -365,9 +365,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest s.holder.broadcaster = s - s.holder.shardValidatorFunc = func(index string, shard uint64) bool { - return s.cluster.ownsShard(s.nodeID, index, shard) - } err = s.cluster.setup() if err != nil { diff --git a/server_internal_test.go b/server_internal_test.go index 86688ebbe..a8af8186c 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -16,12 +16,9 @@ package pilosa import ( "io/ioutil" - "os" "runtime" "testing" "time" - - "github.com/pilosa/pilosa/roaring" ) // Ensure the file handle count is working @@ -63,58 +60,3 @@ func TestMonitorAntiEntropyZero(t *testing.T) { t.Fatalf("monitorAntiEntropy should have returned immediately with duration 0") } } - -func TestOnlyOpenOwnedFiles(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa") - if err != nil { - t.Fatalf("getting temp dir: %v", err) - } - defer func() { - err := os.RemoveAll(path) - if err != nil { - t.Logf("cleaning up temp dir: %v", err) - } - }() - bm := roaring.NewFileBitmap(1, 2, 3) - err = os.MkdirAll(path+"/i/f/views/standard/fragments", os.ModeDir|os.ModePerm) - if err != nil { - t.Fatalf("mkdirall: %v", err) - } - one, err := os.Create(path + "/i/f/views/standard/fragments/1") - if err != nil { - t.Fatalf("creating one: %v", err) - } - two, err := os.Create(path + "/i/f/views/standard/fragments/2") - if err != nil { - t.Fatalf("creating two: %v", err) - } - _, err = bm.WriteTo(one) - if err != nil { - t.Fatalf("writing to one: %v", err) - } - _, err = bm.WriteTo(two) - if err != nil { - t.Fatalf("writing to two: %v", err) - } - - h := NewHolder() - h.Path = path - h.shardValidatorFunc = func(index string, shard uint64) bool { - return shard == 1 - } - - err = h.Open() - if err != nil { - t.Fatalf("opening holder: %v", err) - } - - view := h.Index("i").Field("f").view("standard") - - if len(view.fragments) != 1 { - t.Errorf("should have one fragment, but have: %d", len(view.fragments)) - } - - if _, ok := view.fragments[1]; !ok { - t.Errorf("should have fragment 1, but fragments: %#v", view.fragments) - } -} diff --git a/view.go b/view.go index b1e5dee78..f53cba183 100644 --- a/view.go +++ b/view.go @@ -52,11 +52,10 @@ type view struct { // Fragments by shard. fragments map[uint64]*fragment - broadcaster broadcaster - stats stats.StatsClient - rowAttrStore AttrStore - logger logger.Logger - shardValidator func(uint64) bool + broadcaster broadcaster + stats stats.StatsClient + rowAttrStore AttrStore + logger logger.Logger } // newView returns a new instance of View. @@ -73,10 +72,9 @@ func newView(path, index, field, name string, fieldOptions FieldOptions) *view { fragments: make(map[uint64]*fragment), - broadcaster: NopBroadcaster, - stats: stats.NopStatsClient, - logger: logger.NopLogger, - shardValidator: defaultShardValidator, + broadcaster: NopBroadcaster, + stats: stats.NopStatsClient, + logger: logger.NopLogger, } } @@ -134,10 +132,6 @@ func (v *view) openFragments() error { if err != nil { continue } - //skip shard if not owned - if !v.shardValidator(shard) { - continue - } frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { From 0ef3e5e1441e9a743ee76e363a880315ded76969 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 25 Apr 2019 16:03:44 -0500 Subject: [PATCH 39/73] add ability to post schema using holder.applySchema New API warning: this adds ApplySchema to pilosa.API and allows POSTing to the /schema endpoint --- api.go | 4 ++++ http/handler.go | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index db55334a3..1405b219e 100644 --- a/api.go +++ b/api.go @@ -655,6 +655,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.holder.limitedSchema() } +func (api *API) ApplySchema(ctx context.Context, s *Schema) error { + return api.holder.applySchema(s) +} + // Views returns the views in the given field. func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Views") diff --git a/http/handler.go b/http/handler.go index bbb6e28a2..8fd3b446c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -187,6 +187,7 @@ func (h *Handler) populateValidators() { h.validators["GetInfo"] = queryValidationSpecRequired() h.validators["RecalculateCaches"] = queryValidationSpecRequired() h.validators["GetSchema"] = queryValidationSpecRequired() + h.validators["PostSchema"] = queryValidationSpecRequired() h.validators["GetStatus"] = queryValidationSpecRequired() h.validators["GetVersion"] = queryValidationSpecRequired() h.validators["PostClusterMessage"] = queryValidationSpecRequired() @@ -255,6 +256,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") @@ -412,11 +414,25 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } schema := h.api.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { + if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here? h.logger.Printf("write schema response error: %s", err) } } +func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { + fmt.Println("here") + schema := &pilosa.Schema{} + if err := json.NewDecoder(r.Body).Decode(schema); err != nil { + http.Error(w, fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err), http.StatusBadRequest) + return + } + + if err := h.api.ApplySchema(r.Context(), schema); err != nil { + http.Error(w, fmt.Sprintf("apply schema to Pilosa: %v", err), http.StatusBadRequest) + return + } +} + // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { From 53fb82ea72a6edb4478f87f7126d6f5b75fdcd96 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 26 Apr 2019 15:55:10 -0500 Subject: [PATCH 40/73] remove errant debugging println Co-Authored-By: jaffee --- http/handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index 8fd3b446c..7d59c5b89 100644 --- a/http/handler.go +++ b/http/handler.go @@ -420,7 +420,6 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { - fmt.Println("here") schema := &pilosa.Schema{} if err := json.NewDecoder(r.Body).Decode(schema); err != nil { http.Error(w, fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err), http.StatusBadRequest) From a6ee14240384d022a6651cbf6119e98f18390ada Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 26 Apr 2019 18:27:10 -0500 Subject: [PATCH 41/73] update docs, add test --- docs/api-reference.md | 20 +++++++++++++++++++- http/handler.go | 1 + server/handler_test.go | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 959c850d6..68dae46b5 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -241,7 +241,7 @@ curl -XDELETE localhost:10101/index/user/field/language Returns the schema of all indexes in JSON. ``` request -curl -XGET localhost:10101/index +curl -XGET localhost:10101/schema ``` ``` response { @@ -276,6 +276,24 @@ curl -XGET localhost:10101/index } ``` +### Duplicate schema into empty Pilosa cluster + +`POST /schema` + +To duplicate one Pilosa cluster's schema to another, it's possible to +pass the output of `GET /schema` as the request body of `POST /schema` +and all the indexes and fields in the schema will be created in +Pilosa. As of this writing, the behavior of POSTing a schema to a +non-empty Pilosa cluster is undefined. These semantics will likely be +ironed out in a future version. + +``` request +# after (e.g.) curl -XGET localhost:10101/schema > schema.json +curl -XPOST localhost:10101/schema --data-binary @schema.json +``` + +Response: `204 No Content` + ### Get version `GET /version` diff --git a/http/handler.go b/http/handler.go index 7d59c5b89..c6335e3dd 100644 --- a/http/handler.go +++ b/http/handler.go @@ -430,6 +430,7 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("apply schema to Pilosa: %v", err), http.StatusBadRequest) return } + w.WriteHeader(http.StatusNoContent) } // handleGetStatus handles GET /status requests. diff --git a/server/handler_test.go b/server/handler_test.go index 082869081..1f56aa6f3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -36,13 +36,13 @@ import ( "github.com/pilosa/pilosa/test" ) -// Ensure the handler returns "not found" for invalid paths. func TestHandler_Endpoints(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} + // Ensure the handler returns "not found" for invalid paths. t.Run("Not Found", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -51,6 +51,44 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("SchemaEmpty", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + body := w.Body.String() + if body != "{\"indexes\":null}\n" { + t.Fatalf("unexpected empty schema: '%v'", body) + } + + }) + + t.Run("PostSchema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) + if w.Code != gohttp.StatusNoContent { + t.Fatalf("unexpected code: %v", w.Code) + } + idx, err := cmd.API.Index(context.Background(), "blah") + if err != nil { + t.Fatalf("getting index: %v", err) + } + if idx.Name() != "blah" { + t.Fatalf("index did not get set, got %v", idx.Name()) + } + + fld, err := cmd.API.Field(context.Background(), "blah", "f1") + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fld.Name() != "f1" { + t.Fatalf("unexpected field: %v", fld.Name()) + } + + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/blah", nil)) + }) + t.Run("Info", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) From 9e6662fb0087bdb508518b19dbc2e9b79f59aec2 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 29 Apr 2019 19:31:04 -0500 Subject: [PATCH 42/73] send POSTed schema to all nodes in cluster also fix a *bunch* of tests that weren't closing the clusters they created. Cleaned up one test to use t.Run instead of just checking everything in a loop --- api.go | 27 +++++++++++++++++- apimethod_string.go | 35 ++++++++++++++++++++++-- client.go | 5 ++++ ctl/export_test.go | 4 ++- ctl/import_test.go | 37 +++++++++++++++++++------ executor_test.go | 8 ++++-- go.mod | 7 ++++- go.sum | 18 ++++++++++++ http/client.go | 27 ++++++++++++++++++ http/client_test.go | 31 ++++++++++++++++----- http/handler.go | 11 ++++++-- http/translator_test.go | 14 ++++++---- server/cluster_test.go | 8 +++--- server/config_internal_test.go | 41 +++++++++++++++------------- server/handler_test.go | 50 ++++++++++++++++++++++++++++++++-- server/server_test.go | 1 + stats/stats_test.go | 4 ++- test/pilosa_test.go | 1 + 18 files changed, 273 insertions(+), 56 deletions(-) diff --git a/api.go b/api.go index 1405b219e..8b30053fb 100644 --- a/api.go +++ b/api.go @@ -655,7 +655,30 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.holder.limitedSchema() } -func (api *API) ApplySchema(ctx context.Context, s *Schema) error { +// ApplySchema takes the given schema and applies it across the +// cluster (if remote is false), or just to this node (if remote is +// true). This is designed for the use case of replicating a schema +// from one Pilosa cluster to another which is initially empty. It is +// not officially supported in other scenarios and may produce +// surprising results. +func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.ApplySchema") + defer span.Finish() + + if err := api.validate(apiApplySchema); err != nil { + return errors.Wrap(err, "validating api method") + } + + if !remote { + nodes := api.cluster.Nodes() + for i, node := range nodes { + err := api.server.defaultClient.PostSchema(ctx, &node.URI, s, true) + if err != nil { + return errors.Wrapf(err, "forwarding post schema to node %d of %d", i+1, len(nodes)) + } + } + } + return api.holder.applySchema(s) } @@ -1277,6 +1300,7 @@ const ( //apiStatsWithTags // not implemented //apiVersion // not implemented apiViews + apiApplySchema ) var methodsCommon = map[apiMethod]struct{}{ @@ -1310,4 +1334,5 @@ var methodsNormal = map[apiMethod]struct{}{ apiRemoveNode: {}, apiShardNodes: {}, apiViews: {}, + apiApplySchema: {}, } diff --git a/apimethod_string.go b/apimethod_string.go index cafeab5af..309648a62 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,40 @@ package pilosa import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[apiClusterMessage-0] + _ = x[apiCreateField-1] + _ = x[apiCreateIndex-2] + _ = x[apiDeleteField-3] + _ = x[apiDeleteAvailableShard-4] + _ = x[apiDeleteIndex-5] + _ = x[apiDeleteView-6] + _ = x[apiExportCSV-7] + _ = x[apiFragmentBlockData-8] + _ = x[apiFragmentBlocks-9] + _ = x[apiFragmentData-10] + _ = x[apiField-11] + _ = x[apiFieldAttrDiff-12] + _ = x[apiImport-13] + _ = x[apiImportValue-14] + _ = x[apiIndex-15] + _ = x[apiIndexAttrDiff-16] + _ = x[apiQuery-17] + _ = x[apiRecalculateCaches-18] + _ = x[apiRemoveNode-19] + _ = x[apiResizeAbort-20] + _ = x[apiSetCoordinator-21] + _ = x[apiShardNodes-22] + _ = x[apiViews-23] + _ = x[apiApplySchema-24] +} -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 166, 182, 191, 205, 213, 229, 237, 257, 270, 284, 301, 314, 322} +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema" + +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/client.go b/client.go index d537705c8..3b815a6bd 100644 --- a/client.go +++ b/client.go @@ -46,6 +46,7 @@ type FieldValue struct { type InternalClient interface { MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) + PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) Nodes(ctx context.Context) ([]*Node, error) @@ -103,6 +104,10 @@ func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, return nil, nil } func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } +func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error { + return nil +} + func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { return nil } diff --git a/ctl/export_test.go b/ctl/export_test.go index 6cb611460..e9f7db8a7 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -44,7 +44,9 @@ func TestExportCommand_Validation(t *testing.T) { } func TestExportCommand_Run(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) diff --git a/ctl/import_test.go b/ctl/import_test.go index 01abf372c..dd4d4a9e6 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -70,7 +70,9 @@ func TestImportCommand_Basic(t *testing.T) { t.Fatal(err) } - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" @@ -97,7 +99,9 @@ func TestImportCommand_Basic(t *testing.T) { } ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" @@ -128,7 +132,9 @@ func TestImportCommand_RunValue(t *testing.T) { } ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) @@ -168,7 +174,9 @@ func TestImportCommand_RunValue(t *testing.T) { t.Fatal(err) } - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) @@ -208,7 +216,9 @@ func TestImportCommand_RunKeys(t *testing.T) { } ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) @@ -259,6 +269,7 @@ func TestImportCommand_KeyReplication(t *testing.T) { ctx := context.Background() c := test.MustRunCluster(t, 2) + defer c.Close() cmd0 := c[0] cmd1 := c[1] @@ -319,7 +330,9 @@ func TestImportCommand_RunValueKeys(t *testing.T) { } ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) @@ -343,7 +356,9 @@ func TestImportCommand_RunValueKeys(t *testing.T) { } func TestImportCommand_InvalidFile(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -429,7 +444,9 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { } func TestImportCommand_BugOverwriteValue(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -503,7 +520,9 @@ func TestImportCommand_RunBool(t *testing.T) { cm := NewImportCommand(stdin, stdout, stderr) ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) diff --git a/executor_test.go b/executor_test.go index c49938637..65440c03a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -508,7 +508,9 @@ func TestExecutor_Execute_Count(t *testing.T) { // Ensure a set query can be executed. func TestExecutor_Execute_Set(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} hldr.SetBit("i", "f", 1, 0) @@ -571,7 +573,9 @@ func TestExecutor_Execute_Set(t *testing.T) { }) t.Run("RowKeyColumnKey", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) diff --git a/go.mod b/go.mod index 8f412ab0c..513390d90 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,12 @@ require ( github.com/spf13/viper v1.3.1 github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 - golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 + golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect + golang.org/x/text v0.3.2 // indirect + golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 ) diff --git a/go.sum b/go.sum index d33a2f661..9bb418b94 100644 --- a/go.sum +++ b/go.sum @@ -105,15 +105,33 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU= +golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1 h1:MSSXVSCgrxTAYytvleklMKlLdxjexiJWNffJciO1nCI= +golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/http/client.go b/http/client.go index 4f2f1c208..74706588d 100644 --- a/http/client.go +++ b/http/client.go @@ -132,6 +132,33 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error return rsp.Indexes, nil } +func (c *InternalClient) PostSchema(ctx context.Context, uri *pilosa.URI, s *pilosa.Schema, remote bool) error { + u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote)) + buf, err := json.Marshal(s) + if err != nil { + return errors.Wrap(err, "marshalling schema") + } + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + return errors.Errorf("unexpected status code: %s", resp.Status) + } + return nil +} + // CreateIndex creates a new index on the server. func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex") diff --git a/http/client_test.go b/http/client_test.go index 0bda138ca..7fa398efa 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -182,7 +182,10 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can export data. func TestClient_Export(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] + host := cmd.URL() cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true}) @@ -345,7 +348,9 @@ func TestClient_Export(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -514,7 +519,9 @@ func TestClient_ImportRoaring(t *testing.T) { // Ensure client can bulk import data. func TestClient_ImportKeys(t *testing.T) { t.Run("SingleNode", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true}) @@ -611,6 +618,7 @@ func TestClient_ImportKeys(t *testing.T) { t.Run("MultiNode", func(t *testing.T) { cluster := test.MustRunCluster(t, 2) + defer cluster.Close() cmd0 := cluster[0] cmd1 := cluster[1] host0 := cmd0.URL() @@ -686,7 +694,9 @@ func TestClient_ImportKeys(t *testing.T) { }) t.Run("IntegerFieldSingleNode", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -769,7 +779,9 @@ func TestClient_ImportKeys(t *testing.T) { // Ensure client can bulk import value data. func TestClient_ImportValue(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -875,7 +887,9 @@ func TestClient_ImportValue(t *testing.T) { // Ensure client can bulk import data while tracking existence. func TestClient_ImportExistence(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -952,7 +966,10 @@ func TestClient_ImportExistence(t *testing.T) { // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] + holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/http/handler.go b/http/handler.go index c6335e3dd..b52ee52a3 100644 --- a/http/handler.go +++ b/http/handler.go @@ -187,7 +187,7 @@ func (h *Handler) populateValidators() { h.validators["GetInfo"] = queryValidationSpecRequired() h.validators["RecalculateCaches"] = queryValidationSpecRequired() h.validators["GetSchema"] = queryValidationSpecRequired() - h.validators["PostSchema"] = queryValidationSpecRequired() + h.validators["PostSchema"] = queryValidationSpecRequired().Optional("remote") h.validators["GetStatus"] = queryValidationSpecRequired() h.validators["GetVersion"] = queryValidationSpecRequired() h.validators["PostClusterMessage"] = queryValidationSpecRequired() @@ -420,13 +420,20 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + remoteStr := q.Get("remote") + var remote bool + if remoteStr == "true" { + remote = true + } + schema := &pilosa.Schema{} if err := json.NewDecoder(r.Body).Decode(schema); err != nil { http.Error(w, fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err), http.StatusBadRequest) return } - if err := h.api.ApplySchema(r.Context(), schema); err != nil { + if err := h.api.ApplySchema(r.Context(), schema, remote); err != nil { http.Error(w, fmt.Sprintf("apply schema to Pilosa: %v", err), http.StatusBadRequest) return } diff --git a/http/translator_test.go b/http/translator_test.go index bec2089a5..87bb4366d 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -37,7 +37,9 @@ func TestTranslateStore_Reader(t *testing.T) { // "translator_test.go:65: unexpected EOF" t.Skip() - primary := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + primary := cluster[0] hldr := test.Holder{Holder: primary.Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) @@ -103,9 +105,10 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + cluster := test.MustRunCluster(t, 1, []server.CommandOption{opts}) + defer cluster.Close() + primary := cluster[0] - defer primary.Close() defer close(done) // Connect to server and begin streaming. @@ -135,8 +138,9 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] - defer primary.Close() + cluster := test.MustRunCluster(t, 1, []server.CommandOption{opts}) + defer cluster.Close() + primary := cluster[0] ts := http.NewTranslateStore(primary.URL()) _, err := ts.Reader(context.Background(), 0) diff --git a/server/cluster_test.go b/server/cluster_test.go index 04ff8ef68..b6f6cf6bf 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -33,8 +33,7 @@ import ( func TestMain_SendReceiveMessage(t *testing.T) { ms := test.MustRunCluster(t, 2) m0, m1 := ms[0], ms[1] - defer m0.Close() - defer m1.Close() + defer ms.Close() // Expected indexes and Fields expected := map[string][]string{ @@ -127,8 +126,7 @@ func TestClusterResize_EmptyNode(t *testing.T) { // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { clus := test.MustRunCluster(t, 2) - defer clus[0].Close() - defer clus[1].Close() + defer clus.Close() if clus[0].API.State() != pilosa.ClusterStateNormal { t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) @@ -141,6 +139,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { clus := test.MustRunCluster(t, 2) + defer clus.Close() if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) @@ -552,6 +551,7 @@ func TestCluster_GossipMembership(t *testing.T) { func TestClusterResize_RemoveNode(t *testing.T) { cluster := test.MustRunCluster(t, 3) + defer cluster.Close() m0 := cluster[0] m1 := cluster[1] diff --git a/server/config_internal_test.go b/server/config_internal_test.go index 2b29ea51e..ca7548895 100644 --- a/server/config_internal_test.go +++ b/server/config_internal_test.go @@ -16,6 +16,7 @@ package server import ( "context" + "fmt" "net" "os" "strings" @@ -125,29 +126,31 @@ func TestConfig_validateAddrs(t *testing.T) { } for i, test := range tests { - c := NewConfig() + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + c := NewConfig() - c.Bind = test.in.bind - c.Advertise = test.in.advertise + c.Bind = test.in.bind + c.Advertise = test.in.advertise - err := c.validateAddrs(context.Background()) + err := c.validateAddrs(context.Background()) - if err != nil && test.expErr == "" { - t.Fatal(err) - } else if err == nil && test.expErr != "" { - t.Fatalf("test %d: expected error string to contain %s, but got no error", i, test.expErr) - } else if err != nil && test.expErr != "" { - if strings.Contains(err.Error(), test.expErr) { - continue - } else { - t.Fatalf("test %d: expected error string to contain %s, but got %s", i, test.expErr, err.Error()) + if err != nil && test.expErr == "" { + t.Fatal(err) + } else if err == nil && test.expErr != "" { + t.Fatalf("expected error string to contain %s, but got no error", test.expErr) + } else if err != nil && test.expErr != "" { + if strings.Contains(err.Error(), test.expErr) { + return + } else { + t.Fatalf("expected error string to contain %s, but got %s", test.expErr, err.Error()) + } } - } - if c.Bind != test.exp.bind { - t.Fatalf("test %d: bind address: expected %s, but got %s", i, test.exp.bind, c.Bind) - } else if c.Advertise != test.exp.advertise { - t.Fatalf("test %d: advertise address: expected %s, but got %s", i, test.exp.advertise, c.Advertise) - } + if c.Bind != test.exp.bind { + t.Fatalf("bind address: expected %s, but got %s", test.exp.bind, c.Bind) + } else if c.Advertise != test.exp.advertise { + t.Fatalf("advertise address: expected %s, but got %s", test.exp.advertise, c.Advertise) + } + }) } } diff --git a/server/handler_test.go b/server/handler_test.go index 1f56aa6f3..fc832269a 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -36,8 +36,49 @@ import ( "github.com/pilosa/pilosa/test" ) +func TestHandler_PostSchemaCluster(t *testing.T) { + cluster := test.MustRunCluster(t, 3) + defer cluster.Close() + cmd := cluster[0] + h := cmd.Handler.(*http.Handler).Handler + + t.Run("PostSchema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) + if w.Code != gohttp.StatusNoContent { + bod, err := ioutil.ReadAll(w.Result().Body) + if err != nil { + t.Errorf("reading body: %v", err) + } + t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod) + } + for i := 0; i < len(cluster); i++ { + cmd = cluster[i] + idx, err := cmd.API.Index(context.Background(), "blah") + if err != nil { + t.Fatalf("getting index: %v", err) + } + if idx.Name() != "blah" { + t.Fatalf("index did not get set, got %v", idx.Name()) + } + + fld, err := cmd.API.Field(context.Background(), "blah", "f1") + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fld.Name() != "f1" { + t.Fatalf("unexpected field: %v", fld.Name()) + } + } + + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/blah", nil)) + }) +} + func TestHandler_Endpoints(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -68,7 +109,11 @@ func TestHandler_Endpoints(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) if w.Code != gohttp.StatusNoContent { - t.Fatalf("unexpected code: %v", w.Code) + bod, err := ioutil.ReadAll(w.Result().Body) + if err != nil { + t.Errorf("reading body: %v", err) + } + t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod) } idx, err := cmd.API.Index(context.Background(), "blah") if err != nil { @@ -698,6 +743,7 @@ func TestHandler_Endpoints(t *testing.T) { } clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) + defer clus.Close() w = httptest.NewRecorder() h := clus[0].Handler.(*http.Handler).Handler h.ServeHTTP(w, req) diff --git a/server/server_test.go b/server/server_test.go index e911b25e6..11a026dea 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -321,6 +321,7 @@ func TestConfig_Parse_DataDir(t *testing.T) { func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 cluster := test.MustRunCluster(t, clusterSize) + defer cluster.Close() // Create the schema. client0 := cluster[0].Client() diff --git a/stats/stats_test.go b/stats/stats_test.go index 3da83ce70..f09cce79e 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -211,7 +211,9 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } func TestStatsCount_APICalls(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 1f00ff68b..1d0feb237 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -28,6 +28,7 @@ import ( func TestNewCluster(t *testing.T) { numNodes := 3 cluster := test.MustRunCluster(t, numNodes) + defer cluster.Close() coordinator := getCoordinator(cluster[0]) for i := 1; i < numNodes; i++ { From 82f5f632ad032ce7550efcd76c624e41f2af1ae1 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 30 Apr 2019 11:48:57 -0500 Subject: [PATCH 43/73] Fix typos --- docs/data-model.md | 2 +- docs/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 1a0781ccc..c12227a29 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -186,7 +186,7 @@ Set(3, A=8, 2017-05-19T00:00) ``` ![time quantum field diagram](/img/docs/field-time-quantum.png) -*Time quantum fueld diagram* +*Time quantum field diagram* #### Mutex diff --git a/docs/getting-started.md b/docs/getting-started.md index 257e77628..45473d7ec 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -74,7 +74,7 @@ curl localhost:10101/index/repository/field/stargazer \ {"success":true} ``` -Since our data contains time stamps whcih represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. +Since our data contains time stamps which represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` field, which will contain IDs for programming languages: ``` request From 875c95b2c3157a26f476bfba1d831c9ee9a075aa Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Sat, 20 Apr 2019 17:43:50 -0500 Subject: [PATCH 44/73] add more Debugf() statements to the holder open process --- field.go | 9 +++++++++ fragment.go | 3 +++ index.go | 5 +++++ view.go | 6 ++++++ 4 files changed, 23 insertions(+) diff --git a/field.go b/field.go index 0b2cf6cf5..6ef362728 100644 --- a/field.go +++ b/field.go @@ -379,27 +379,33 @@ func (f *Field) Options() FieldOptions { func (f *Field) Open() error { if err := func() error { // Ensure the field's path exists. + f.logger.Debugf("ensure field path exists: %s", f.path) if err := os.MkdirAll(f.path, 0777); err != nil { return errors.Wrap(err, "creating field dir") } + f.logger.Debugf("load meta file for index/field: %s/%s", f.index, f.name) if err := f.loadMeta(); err != nil { return errors.Wrap(err, "loading meta") } + f.logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name) if err := f.loadAvailableShards(); err != nil { return errors.Wrap(err, "loading available shards") } // Apply the field options loaded from meta. + f.logger.Debugf("apply options for index/field: %s/%s", f.index, f.name) if err := f.applyOptions(f.options); err != nil { return errors.Wrap(err, "applying options") } + f.logger.Debugf("open views for index/field: %s/%s", f.index, f.name) if err := f.openViews(); err != nil { return errors.Wrap(err, "opening views") } + f.logger.Debugf("open row attribute store for index/field: %s/%s", f.index, f.name) if err := f.rowAttrStore.Open(); err != nil { return errors.Wrap(err, "opening attrstore") } @@ -410,6 +416,7 @@ func (f *Field) Open() error { return err } + f.logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name) return nil } @@ -434,11 +441,13 @@ func (f *Field) openViews() error { } name := filepath.Base(fi.Name()) + f.logger.Debugf("open index/field/view: %s/%s/%s", f.index, f.name, fi.Name()) view := f.newView(f.viewPath(name), name) if err := view.open(); err != nil { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } view.rowAttrStore = f.rowAttrStore + f.logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name) f.viewMap[view.name] = view } diff --git a/fragment.go b/fragment.go index c08571e78..7c24a5d66 100644 --- a/fragment.go +++ b/fragment.go @@ -163,11 +163,13 @@ func (f *fragment) Open() error { if err := func() error { // Initialize storage in a function so we can close if anything goes wrong. + f.Logger.Debugf("open storage for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) if err := f.openStorage(); err != nil { return errors.Wrap(err, "opening storage") } // Fill cache with rows persisted to disk. + f.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) if err := f.openCache(); err != nil { return errors.Wrap(err, "opening cache") } @@ -185,6 +187,7 @@ func (f *fragment) Open() error { return err } + f.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) return nil } diff --git a/index.go b/index.go index 8b67f003c..dbb9c052a 100644 --- a/index.go +++ b/index.go @@ -107,15 +107,18 @@ func (i *Index) options() IndexOptions { // Open opens and initializes the index. func (i *Index) Open() error { // Ensure the path exists. + i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { return errors.Wrap(err, "creating directory") } // Read meta file. + i.logger.Debugf("load meta file for index: %s", i.name) if err := i.loadMeta(); err != nil { return errors.Wrap(err, "loading meta file") } + i.logger.Debugf("open fields for index: %s", i.name) if err := i.openFields(); err != nil { return errors.Wrap(err, "opening fields") } @@ -151,6 +154,7 @@ func (i *Index) openFields() error { continue } + i.logger.Debugf("open field: %s", fi.Name()) fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { return errors.Wrapf(ErrName, "'%s'", fi.Name()) @@ -158,6 +162,7 @@ func (i *Index) openFields() error { if err := fld.Open(); err != nil { return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } + i.logger.Debugf("add field to index.fields: %s", fi.Name()) i.fields[fld.Name()] = fld } return nil diff --git a/view.go b/view.go index f53cba183..e8894db7d 100644 --- a/view.go +++ b/view.go @@ -88,12 +88,14 @@ func (v *view) open() error { if err := func() error { // Ensure the view's path exists. + v.logger.Debugf("ensure view path exists: %s", v.path) if err := os.MkdirAll(v.path, 0777); err != nil { return errors.Wrap(err, "creating view directory") } else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil { return errors.Wrap(err, "creating fragments directory") } + v.logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name) if err := v.openFragments(); err != nil { return errors.Wrap(err, "opening fragments") } @@ -104,6 +106,7 @@ func (v *view) open() error { return err } + v.logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name) return nil } @@ -130,14 +133,17 @@ func (v *view) openFragments() error { // Parse filename into integer. shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) if err != nil { + v.logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name()) continue } + v.logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard) frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } frag.RowAttrStore = v.rowAttrStore + v.logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard) v.fragments[frag.shard] = frag } From 61bf3d929de4849dac7001894835b7247bb7164c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 10 Apr 2019 13:12:21 -0500 Subject: [PATCH 45/73] Add more tracing and metdata to importRoaring --- api.go | 5 +++-- field.go | 10 ++++++++-- fragment.go | 19 +++++++++++++++++-- fragment_internal_test.go | 40 ++++++++++++++++++++++----------------- go.mod | 8 ++++++-- go.sum | 10 ++++++++++ http/handler.go | 13 +++++++++++-- 7 files changed, 78 insertions(+), 27 deletions(-) diff --git a/api.go b/api.go index 8b30053fb..237593083 100644 --- a/api.go +++ b/api.go @@ -290,6 +290,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) { // bitmap. func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring") + span.LogKV("index", indexName, "field", fieldName) defer span.Finish() if err = api.validate(apiField); err != nil { @@ -325,7 +326,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, } fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err = field.importRoaring(viewData, shard, viewName, req.Clear) + err = field.importRoaring(ctx, viewData, shard, viewName, req.Clear) if err != nil { return errors.Wrap(err, "importing pilosa roaring") } @@ -335,7 +336,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, // field.importRoaring changes the standard roaring run format to pilosa roaring data := make([]byte, len(viewData)) copy(data, viewData) - err = field.importRoaring(data, shard, viewName, req.Clear) + err = field.importRoaring(ctx, data, shard, viewName, req.Clear) if err != nil { return errors.Wrap(err, "importing standard roaring") } diff --git a/field.go b/field.go index 6ef362728..3b0b63feb 100644 --- a/field.go +++ b/field.go @@ -16,6 +16,7 @@ package pilosa import ( "bufio" + "context" "encoding/json" "fmt" "io/ioutil" @@ -32,6 +33,7 @@ import ( "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/stats" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -1218,10 +1220,14 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return nil } -func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear bool) error { +func (f *Field) importRoaring(ctx context.Context, data []byte, shard uint64, viewName string, clear bool) error { + span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaring") + defer span.Finish() + if viewName == "" { viewName = viewStandard } + span.LogKV("view", viewName, "bytes", len(data), "shard", shard) view, err := f.createViewIfNotExists(viewName) if err != nil { return errors.Wrap(err, "creating view") @@ -1232,7 +1238,7 @@ func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaring(data, clear); err != nil { + if err := frag.importRoaring(ctx, data, clear); err != nil { return err } diff --git a/fragment.go b/fragment.go index 7c24a5d66..6a666e026 100644 --- a/fragment.go +++ b/fragment.go @@ -1811,11 +1811,15 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear // importRoaring imports from the official roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version // of the roaring format. The cache is updated to reflect the new data. -func (f *fragment) importRoaring(data []byte, clear bool) error { +func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) error { + span, ctx := tracing.StartSpanFromContext(ctx, "fragment.importRoaring") + defer span.Finish() f.mu.Lock() defer f.mu.Unlock() bm := roaring.NewBTreeBitmap() + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.UnmarshalBinary") err := bm.UnmarshalBinary(data) + span.Finish() if err != nil { return err } @@ -1851,16 +1855,25 @@ func (f *fragment) importRoaring(data []byte, clear bool) error { if clear { toSet, toClear = toClear, toSet } - return f.importPositions(toSet, toClear, rowSet) + span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.ImportPositions") + err := f.importPositions(toSet, toClear, rowSet) + span.Finish() + return err } if clear { + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaringDifference") bm = f.storage.Difference(bm) + span.Finish() } else if f.storage.Containers.Size() >= bm.Containers.Size() { + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaringStorageUIP") f.storage.UnionInPlace(bm) bm = f.storage + span.Finish() } else { + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaringBitmapUIP") bm.UnionInPlace(f.storage) + span.Finish() } for rowID := range rowSet { @@ -1869,7 +1882,9 @@ func (f *fragment) importRoaring(data []byte, clear bool) error { } f.cache.Recalculate() + span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.WriteToFragment") err = unprotectedWriteToFragment(f, bm) + span.Finish() return err } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index db0159734..144499ca3 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -16,6 +16,7 @@ package pilosa import ( "bytes" + "context" "flag" "fmt" "io" @@ -745,7 +746,7 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = opN defer f.Clean(b) - err := f.importRoaring(getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) + err := f.importRoaringT(getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) } @@ -781,14 +782,14 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = opN defer f.Clean(b) - err := f.importRoaring(getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) + err := f.importRoaringT(getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) } for i := 0; i < numUpdates; i++ { data := getUpdataRoaring(numRows, bitsPerUpdate, int64(i)) b.StartTimer() - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) b.StopTimer() if err != nil { b.Fatalf("doing small roaring import: %v", err) @@ -1024,7 +1025,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { if err != nil { t.Fatalf("writing to bytes: %v", err) } - err = f.importRoaring(b.Bytes(), false) + err = f.importRoaringT(b.Bytes(), false) if err != nil { t.Fatalf("importing data: %v", err) } @@ -2008,7 +2009,7 @@ func BenchmarkImportRoaring(b *testing.B) { for i := 0; i < b.N; i++ { f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) b.StartTimer() - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { f.Clean(b) b.Fatalf("import error: %v", err) @@ -2041,7 +2042,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - return frags[j].importRoaring(data, false) + return frags[j].importRoaringT(data, false) }) } err := eg.Wait() @@ -2072,7 +2073,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) - err := frags[j].importRoaring(data, false) + err := frags[j].importRoaringT(data, false) if err != nil { b.Fatalf("importing roaring: %v", err) } @@ -2082,7 +2083,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - return frags[j].importRoaring(updata, false) + return frags[j].importRoaringT(updata, false) }) } err := eg.Wait() @@ -2138,12 +2139,12 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { b.Errorf("import error: %v", err) } b.StartTimer() - err = f.importRoaring(updata, false) + err = f.importRoaringT(updata, false) if err != nil { f.Clean(b) b.Errorf("import error: %v", err) @@ -2174,12 +2175,12 @@ func BenchmarkUpdatePathological(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) - err := f.importRoaring(exists, false) + err := f.importRoaringT(exists, false) if err != nil { b.Fatalf("importing roaring: %v", err) } b.StartTimer() - err = f.importRoaring(inc, false) + err = f.importRoaringT(inc, false) if err != nil { b.Fatalf("importing second: %v", err) } @@ -2196,7 +2197,7 @@ func initBigFrag() { for i := int64(0); i < 10; i++ { // 10 million rows, 1 bit per column, random seeded by i data := getZipfRowsSliceRoaring(10000000, i, 0, ShardWidth) - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { panic(fmt.Sprintf("setting up fragment data: %v", err)) } @@ -2273,7 +2274,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { b.Fatalf("opening fragment: %v", err) } b.StartTimer() - err = nf.importRoaring(updata, false) + err = nf.importRoaringT(updata, false) b.StopTimer() if err != nil { b.Fatalf("bulkImport: %v", err) @@ -2286,7 +2287,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { func TestGetZipfRowsSliceRoaring(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) data := getZipfRowsSliceRoaring(10, 1, 0, ShardWidth) - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { t.Fatalf("importing roaring: %v", err) } @@ -2431,6 +2432,11 @@ func (f *fragment) Clean(t testing.TB) { } } +// importRoaringT calls importRoaring with context.Background() for convenience +func (f *fragment) importRoaringT(data []byte, clear bool) error { + return f.importRoaring(context.Background(), data, clear) +} + // CleanKeep is just like Clean(), but it doesn't remove the // fragment file (note that it DOES remove the cache file). func (f *fragment) CleanKeep(t testing.TB) { @@ -2622,7 +2628,7 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - err = f.importRoaring(buf.Bytes(), false) + err = f.importRoaringT(buf.Bytes(), false) if err != nil { t.Fatalf("importing roaring: %v", err) } @@ -2697,7 +2703,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - err = f.importRoaring(buf.Bytes(), false) + err = f.importRoaringT(buf.Bytes(), false) if err != nil { t.Fatalf("importing roaring: %v", err) } diff --git a/go.mod b/go.mod index 513390d90..053cee36a 100644 --- a/go.mod +++ b/go.mod @@ -5,27 +5,31 @@ replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.2.0 github.com/google/go-cmp v0.2.0 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/opentracing/opentracing-go v1.0.2 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.8.1 github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.1 - github.com/uber/jaeger-client-go v2.15.0+incompatible - github.com/uber/jaeger-lib v1.5.0 + github.com/uber/jaeger-client-go v2.16.0+incompatible + github.com/uber/jaeger-lib v2.0.0+incompatible // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 diff --git a/go.sum b/go.sum index 9bb418b94..1e00ccec0 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2M github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -19,6 +21,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -50,6 +54,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= @@ -80,6 +86,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -96,6 +104,8 @@ github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= +github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= diff --git a/http/handler.go b/http/handler.go index b52ee52a3..78b5f48a9 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1617,15 +1617,23 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request remote = true } + ctx := r.Context() + // Read entire body. + span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") body, err := ioutil.ReadAll(r.Body) + span.LogKV("bodySize", len(body)) + span.Finish() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } req := &pilosa.ImportRoaringRequest{} - if err := h.api.Serializer.Unmarshal(body, req); err != nil { + span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") + err = h.api.Serializer.Unmarshal(body, req) + span.Finish() + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -1639,7 +1647,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp := &pilosa.ImportResponse{} // TODO give meaningful stats for import - err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, remote, req) + err = h.api.ImportRoaring(ctx, indexName, fieldName, shard, remote, req) if err != nil { resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { @@ -1648,6 +1656,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request w.WriteHeader(http.StatusInternalServerError) } } + // Marshal response object. buf, err := h.api.Serializer.Marshal(resp) if err != nil { From 00911d024bfdae21adb19d82a877ade456028fbe Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 30 Apr 2019 16:55:46 -0500 Subject: [PATCH 46/73] add span around fragment lock, bytes written metadata --- fragment.go | 26 +++++++++++++++----------- fragment_internal_test.go | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/fragment.go b/fragment.go index 6a666e026..8e8fa0ce5 100644 --- a/fragment.go +++ b/fragment.go @@ -1814,8 +1814,10 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) error { span, ctx := tracing.StartSpanFromContext(ctx, "fragment.importRoaring") defer span.Finish() + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.AcquireFragmentLock") f.mu.Lock() defer f.mu.Unlock() + span.Finish() bm := roaring.NewBTreeBitmap() span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.UnmarshalBinary") err := bm.UnmarshalBinary(data) @@ -1883,7 +1885,8 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e f.cache.Recalculate() span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.WriteToFragment") - err = unprotectedWriteToFragment(f, bm) + n, err := unprotectedWriteToFragment(f, bm) + span.LogKV("bytesWritten", n) span.Finish() return err } @@ -1915,12 +1918,13 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg } func (f *fragment) snapshot() error { - return unprotectedWriteToFragment(f, f.storage) + _, err := unprotectedWriteToFragment(f, f.storage) + return err } // unprotectedWriteToFragment writes the fragment f with bm as the data. It is unprotected, and // f.mu must be locked when calling it. -func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer +func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err error) { // nolint: interfacer completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() @@ -1930,39 +1934,39 @@ func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // noli snapshotPath := f.path + snapshotExt file, err := os.Create(snapshotPath) if err != nil { - return fmt.Errorf("create snapshot file: %s", err) + return n, fmt.Errorf("create snapshot file: %s", err) } defer file.Close() // Write storage to snapshot. bw := bufio.NewWriter(file) - if _, err := bm.WriteTo(bw); err != nil { - return fmt.Errorf("snapshot write to: %s", err) + if n, err = bm.WriteTo(bw); err != nil { + return n, fmt.Errorf("snapshot write to: %s", err) } if err := bw.Flush(); err != nil { - return fmt.Errorf("flush: %s", err) + return n, fmt.Errorf("flush: %s", err) } // Close current storage. if err := f.closeStorage(); err != nil { - return fmt.Errorf("close storage: %s", err) + return n, fmt.Errorf("close storage: %s", err) } // Move snapshot to data file location. if err := os.Rename(snapshotPath, f.path); err != nil { - return fmt.Errorf("rename snapshot: %s", err) + return n, fmt.Errorf("rename snapshot: %s", err) } // Reopen storage. if err := f.openStorage(); err != nil { - return fmt.Errorf("open storage: %s", err) + return n, fmt.Errorf("open storage: %s", err) } // Reset operation count. f.opN = 0 - return nil + return n, nil } // RecalculateCache rebuilds the cache regardless of invalidate time delay. diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 144499ca3..43b11357c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2947,7 +2947,7 @@ func TestUnionInPlaceMapped(t *testing.T) { count1 := setBM1.Count() // now we write setBM0 into f.storage. - err = unprotectedWriteToFragment(f, setBM0) + _, err = unprotectedWriteToFragment(f, setBM0) if err != nil { t.Fatalf("trying to flush fragment to disk: %v", err) } From 27fab06e7869346238b89de9d574b5e355b38f8e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 1 May 2019 17:35:16 -0500 Subject: [PATCH 47/73] simplify contributing instructions by removing weird upstream thing we can probably remove GOPATH too, but I'll save that for another day. For now, we make it so that the obvious thing (cloning the official repo) works as a normal part of the contribution process. --- CONTRIBUTING.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93e4854f5..8e4b45e4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,11 +28,17 @@ If you want to help but you aren't sure where to start, check out our [github la - Fork the [Pilosa repository][2] to your own account. -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo: +- It will be easier to follow these instructions if you: + + ```sh + export GH_USERNAME= + ``` + +- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: ```sh mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone git@github.com:${USER}/pilosa.git + git clone https://github.com/pilosa/pilosa.git ``` - `cd` to your pilosa directory: @@ -46,20 +52,14 @@ If you want to help but you aren't sure where to start, check out our [github la ```sh make install ``` - - or - - ``` - go install github.com/pilosa/pilosa/cmd/... - ``` Running `pilosa` should now run a Pilosa instance. -- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo: +- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username ```sh cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add upstream git@github.com:pilosa/pilosa.git + git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git ``` ### Makefile @@ -133,9 +133,8 @@ Additional commands are available in the `Makefile`. - Before starting to work on a task, sync your branch with the upstream: ```sh - git fetch upstream git checkout master - git merge upstream/master + git pull ``` - Create a local feature branch: @@ -155,15 +154,16 @@ Additional commands are available in the `Makefile`. - Verify that your pull request is applied to the latest version of code on github: ```sh - git remote add upstream git@github.com:pilosa/pilosa.git - git fetch upstream - git rebase -i upstream/master + git checkout master + git pull + git checkout something-amazing + git rebase master ``` - Push to your fork: ```sh - git push -u something-amazing + git push -u $GH_USERNAME something-amazing:something-amazing ``` - Submit a [pull request][3] From 51ac675e8279f49750cef3193e65ec78df149eb4 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Wed, 24 Apr 2019 20:13:11 +0200 Subject: [PATCH 48/73] TranslateFile - reopen the same instance Signed-off-by: kuba-- --- go.mod | 9 ++- go.sum | 33 ++++++---- translate.go | 1 + translate_test.go | 156 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 185 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 8f412ab0c..a5fff553c 100644 --- a/go.mod +++ b/go.mod @@ -3,29 +3,36 @@ module github.com/pilosa/pilosa replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 require ( + github.com/BurntSushi/toml v0.3.1 // indirect github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.2.0 github.com/google/go-cmp v0.2.0 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/opentracing/opentracing-go v1.0.2 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.8.1 github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.1 + github.com/uber-go/atomic v1.3.2 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible - github.com/uber/jaeger-lib v1.5.0 + github.com/uber/jaeger-lib v1.5.0 // indirect + go.uber.org/atomic v1.3.2 // indirect golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 diff --git a/go.sum b/go.sum index d33a2f661..71f6ea860 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,13 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -19,6 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -29,8 +36,6 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= -github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -43,13 +48,14 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= @@ -58,19 +64,15 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pilosa/memberlist v0.1.3 h1:6am86S+mnY3zKPmH5yHtTqdNpqH/KjxF6WSHk95Msyo= -github.com/pilosa/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjfFAqnhX3T/4X3ItZK1H+r9neBnK+nV1ec= -github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA= -github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -80,6 +82,9 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -93,15 +98,18 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= -github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -114,6 +122,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/translate.go b/translate.go index 3e4245ae1..94eb57186 100644 --- a/translate.go +++ b/translate.go @@ -167,6 +167,7 @@ func (s *TranslateFile) Open() (err error) { return errors.Wrapf(err, "open file %s", s.Path) } s.w = bufio.NewWriter(s.file) + s.n = 0 // Memory map data file. if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { diff --git a/translate_test.go b/translate_test.go index a2f0b0268..4fa977e62 100644 --- a/translate_test.go +++ b/translate_test.go @@ -758,6 +758,156 @@ func TestTranslateFile_ReassignPrimaryTranslateStore(t *testing.T) { }) } +func TestTranslateFile_ReopenTheSameInstance(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + if value, err := s.TranslateColumnToString("IDX0", 1); err != nil { + t.Fatal(err) + } else if value != "foo" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + s.MustOpen() + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + s.MustOpen() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FIELD0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + // Reopen the store. + s.MustOpen() + + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } +} + func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { const batchSize = 1000 @@ -836,10 +986,14 @@ func (t *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser func MustOpenTranslateFile() *TranslateFile { s := NewTranslateFile() + s.MustOpen() + return s +} + +func (s *TranslateFile) MustOpen() { if err := s.Open(); err != nil { panic(err) } - return s } func (s *TranslateFile) Close() error { From de61d0417231732574603115352363e95056b347 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 10 May 2019 13:55:21 -0500 Subject: [PATCH 49/73] failing test for group by with filter using string keys also, apparently our API code was assuming that imports with keys always had timestamps which seemed wrong, so I fixed that. --- api.go | 13 ++++++++----- executor_test.go | 37 +++++++++++++++++++++++++++++++++++++ go.sum | 2 ++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index 237593083..2b96d023c 100644 --- a/api.go +++ b/api.go @@ -897,11 +897,14 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if _, ok := m[shard]; !ok { m[shard] = make([]Bit, 0) } - m[shard] = append(m[shard], Bit{ - RowID: req.RowIDs[i], - ColumnID: colID, - Timestamp: req.Timestamps[i], - }) + bit := Bit{ + RowID: req.RowIDs[i], + ColumnID: colID, + } + if len(req.Timestamps) > 0 { + bit.Timestamp = req.Timestamps[i] + } + m[shard] = append(m[shard], bit) } // Signal to the receiving nodes to ignore checking for key translation. diff --git a/executor_test.go b/executor_test.go index 65440c03a..ce84d04c4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3243,7 +3243,44 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { } }) } +} +func TestExecutor_GroupByStrings(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "generals", pilosa.OptFieldKeys()) + + req := &pilosa.ImportRequest{ + Index: "istring", + Field: "generals", + Shard: 0, + RowKeys: []string{"r1", "r2", "r1", "r2", "r1", "r2", "r1", "r2", "r1", "r2"}, + ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"}, + } + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatalf("importing: %v", err) + } + + tests := []struct { + query string + }{ + { + query: "GroupBy(Rows(generals), filter=Row(generals=r2))", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "istring", + Query: test.query, + }) + if err != nil { + t.Fatalf("got an error %v", err) + } + fmt.Println(r) + }) + } } func TestExecutor_Execute_Rows_Keys(t *testing.T) { diff --git a/go.sum b/go.sum index a1541510b..f12f88d07 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,8 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQG github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= +github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= From e185a01e67ba12fbfb175ed1c63ed37cc36556a5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 10 May 2019 14:15:06 -0500 Subject: [PATCH 50/73] add translation for groupby filter arg, improve test --- executor.go | 10 ++++++++++ executor_test.go | 20 ++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index ce30a32c6..41077168e 100644 --- a/executor.go +++ b/executor.go @@ -2536,6 +2536,16 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e } } + if filter, ok, err := c.CallArg("filter"); ok { + if err != nil { + return errors.Wrap(err, "getting filter call") + } + err = e.translateCall(index, idx, filter) + if err != nil { + return errors.Wrap(err, "translating filter call") + } + } + prev, ok := c.Args["previous"] if !ok { return nil // nothing else to be translated diff --git a/executor_test.go b/executor_test.go index ce84d04c4..fd3090401 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3262,23 +3262,35 @@ func TestExecutor_GroupByStrings(t *testing.T) { } tests := []struct { - query string + query string + expected []pilosa.GroupCount }{ + { + query: "GroupBy(Rows(generals))", + expected: []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5}, + }, + }, { query: "GroupBy(Rows(generals), filter=Row(generals=r2))", + expected: []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5}, + }, }, } - for i, test := range tests { + for i, tst := range tests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ Index: "istring", - Query: test.query, + Query: tst.query, }) if err != nil { t.Fatalf("got an error %v", err) } - fmt.Println(r) + results := r.Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, tst.expected, results) }) } } From fb93f90f3162ac71e10d83f66aba15e1fb40b9f9 Mon Sep 17 00:00:00 2001 From: Shaquille Wyan Que Date: Mon, 13 May 2019 11:20:50 -0500 Subject: [PATCH 51/73] fixed error message returned by regex on field and index names --- docs/getting-started.md | 1 + docs/query-language.md | 2 +- go.sum | 1 + pilosa.go | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 45473d7ec..c7dd54a8c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -63,6 +63,7 @@ curl localhost:10101/index/repository -X POST ``` response {"success":true} ``` +The index name must be 64 characters or less, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. Let's create the `stargazer` field which has user IDs of stargazers as its rows: ``` request diff --git a/docs/query-language.md b/docs/query-language.md index 408a967bc..0064401cc 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types -* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04) * `UINT` An unsigned integer (e.g. 42839) * `BOOL` A boolean value, `true` or `false` diff --git a/go.sum b/go.sum index f12f88d07..674e2adea 100644 --- a/go.sum +++ b/go.sum @@ -145,6 +145,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1 h1:MSSXVSCgrxTAYytvleklMKlLdxjexiJWNffJciO1nCI= golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pilosa.go b/pilosa.go index 88510cef9..42ab3d3c1 100644 --- a/pilosa.go +++ b/pilosa.go @@ -48,7 +48,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 64 characters") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. @@ -152,7 +152,7 @@ func (cas ColumnAttrSet) MarshalJSON() ([]byte, error) { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// validateName ensures that the name is a valid format. +// validateName ensures that the index or field name is a valid format. func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { return errors.Wrapf(ErrName, "'%s'", name) From 98a864634ebae5fef9849f6b4f8c2dd8e5f78f3e Mon Sep 17 00:00:00 2001 From: Shaquille Wyan Que Date: Tue, 14 May 2019 16:58:03 -0500 Subject: [PATCH 52/73] fixed out of bounds panic to show error --- pql/ast.go | 2 +- pql/parser.go | 3 ++- pql/pqlpeg_test.go | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 20b757946..e14a93fe1 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -160,7 +160,7 @@ func (q *Query) addNumVal(val string) { ival, err = strconv.ParseInt(val, 10, 64) } if err != nil { - panic(err) + panic(fmt.Sprintf("out of bounds: %s", err)) } if elem.inList { if elem.lastCond != ILLEGAL { diff --git a/pql/parser.go b/pql/parser.go index 6a28f560e..c9f92c61a 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -28,6 +28,7 @@ const timeFormat = "2006-01-02T15:04" // duplicateArgErrorMessage is used as an error string in the parser. const duplicateArgErrorMessage = "duplicate argument provided" +const parsingIntErrorMessage = "out of bounds" // parser represents a parser for the PQL language. type parser struct { @@ -71,7 +72,7 @@ func (p *parser) Parse() (*Query, error) { p.Execute() }() if v != nil { - if strings.HasPrefix(v.(string), duplicateArgErrorMessage) { + if strings.HasPrefix(v.(string), duplicateArgErrorMessage) || strings.HasPrefix(v.(string), parsingIntErrorMessage){ return nil, fmt.Errorf("%s", v) } else { panic(v) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index a90d58a5d..c9db07407 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -333,6 +333,12 @@ func TestPEGErrors(t *testing.T) { { name: "RangeTimeOneStamp", input: "Row(a=4, 2010-07-04T00:00)"}, + { + name: "ArgOutOfBounds", + input: "Row(a=9223372036854775808)"}, + { + name: "ArgOutOfBoundsNeg", + input: "Row(a=-9223372036854775809)"}, } for i, test := range tests { From b770167db66aea5205152e54beb7fdffeecf14ab Mon Sep 17 00:00:00 2001 From: Shaquille Wyan Que Date: Wed, 15 May 2019 10:52:15 -0500 Subject: [PATCH 53/73] fixed formatting --- pql/parser.go | 6 +++--- pql/pqlpeg_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pql/parser.go b/pql/parser.go index c9f92c61a..8c3b608c9 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -26,9 +26,9 @@ import ( // timeFormat is the go-style time format used to parse string dates. const timeFormat = "2006-01-02T15:04" -// duplicateArgErrorMessage is used as an error string in the parser. +// error strings in the parser const duplicateArgErrorMessage = "duplicate argument provided" -const parsingIntErrorMessage = "out of bounds" +const outOfBoundsErrorMessage = "out of bounds" // parser represents a parser for the PQL language. type parser struct { @@ -72,7 +72,7 @@ func (p *parser) Parse() (*Query, error) { p.Execute() }() if v != nil { - if strings.HasPrefix(v.(string), duplicateArgErrorMessage) || strings.HasPrefix(v.(string), parsingIntErrorMessage){ + if strings.HasPrefix(v.(string), duplicateArgErrorMessage) || strings.HasPrefix(v.(string), outOfBoundsErrorMessage) { return nil, fmt.Errorf("%s", v) } else { panic(v) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index c9db07407..3ceff075d 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -333,7 +333,7 @@ func TestPEGErrors(t *testing.T) { { name: "RangeTimeOneStamp", input: "Row(a=4, 2010-07-04T00:00)"}, - { + { name: "ArgOutOfBounds", input: "Row(a=9223372036854775808)"}, { From 6ba6218ae439ac3f420b8d077fc9ba572eb4def9 Mon Sep 17 00:00:00 2001 From: Shaquille Wyan Que Date: Wed, 15 May 2019 11:22:04 -0500 Subject: [PATCH 54/73] changed out of range error message name and fixed formatting --- pql/ast.go | 2 +- pql/parser.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index e14a93fe1..49503474c 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -160,7 +160,7 @@ func (q *Query) addNumVal(val string) { ival, err = strconv.ParseInt(val, 10, 64) } if err != nil { - panic(fmt.Sprintf("out of bounds: %s", err)) + panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err)) } if elem.inList { if elem.lastCond != ILLEGAL { diff --git a/pql/parser.go b/pql/parser.go index 8c3b608c9..cef8a9b9b 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -28,7 +28,7 @@ const timeFormat = "2006-01-02T15:04" // error strings in the parser const duplicateArgErrorMessage = "duplicate argument provided" -const outOfBoundsErrorMessage = "out of bounds" +const intOutOfRangeError = "integer is not in signed 64-bit range" // parser represents a parser for the PQL language. type parser struct { @@ -72,7 +72,8 @@ func (p *parser) Parse() (*Query, error) { p.Execute() }() if v != nil { - if strings.HasPrefix(v.(string), duplicateArgErrorMessage) || strings.HasPrefix(v.(string), outOfBoundsErrorMessage) { + errorMessage := v.(string) + if strings.HasPrefix(errorMessage, duplicateArgErrorMessage) || strings.HasPrefix(errorMessage, intOutOfRangeError) { return nil, fmt.Errorf("%s", v) } else { panic(v) From 44088d4f29654537ee9ca9259ab705fcff055fda Mon Sep 17 00:00:00 2001 From: Shaquille Wyan Que Date: Wed, 15 May 2019 12:11:55 -0500 Subject: [PATCH 55/73] added check for unexpected parser error --- pql/parser.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pql/parser.go b/pql/parser.go index cef8a9b9b..e733038d7 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -72,7 +72,10 @@ func (p *parser) Parse() (*Query, error) { p.Execute() }() if v != nil { - errorMessage := v.(string) + errorMessage, ok := v.(string) + if !ok { + return nil, fmt.Errorf("unexpected parser error of type %T: %[1]v", v) + } if strings.HasPrefix(errorMessage, duplicateArgErrorMessage) || strings.HasPrefix(errorMessage, intOutOfRangeError) { return nil, fmt.Errorf("%s", v) } else { From 7ed9fba3359feb87f4fd2b9bd44561e6e69a243a Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 6 Apr 2019 15:13:31 -0600 Subject: [PATCH 56/73] Unbounded BSI w/ sign magnitude This commit implements BSI with variable bit depth using a sign magnitudeto indicate whether a value is positive or negative. This also rearranges the existence bit to be the first bit instead of the last bit. --- api_test.go | 2 +- docs/data-model.md | 8 + encoding/proto/proto.go | 8 +- executor.go | 30 +- executor_test.go | 82 +- field.go | 263 +++-- field_internal_test.go | 107 +- field_test.go | 34 +- fragment.go | 459 ++++++-- fragment_internal_test.go | 79 +- http/client.go | 7 +- http/handler.go | 40 +- index_test.go | 2 +- internal/private.pb.go | 2161 +++++++------------------------------ internal/private.proto | 7 +- internal/public.pb.go | 1158 +++----------------- pilosa.go | 2 - roaring/roaring.go | 44 +- row.go | 10 + view.go | 46 +- 20 files changed, 1373 insertions(+), 3176 deletions(-) diff --git a/api_test.go b/api_test.go index a33b064c1..ef737d596 100644 --- a/api_test.go +++ b/api_test.go @@ -198,7 +198,7 @@ func TestAPI_ImportValue(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(0, 100)) + _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(0)) if err != nil { t.Fatalf("creating field: %v", err) } diff --git a/docs/data-model.md b/docs/data-model.md index c12227a29..7de27b575 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -165,6 +165,14 @@ Set(3, B=6) Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa. + +###### BSI Deprecated Format + +The original implementation of BSI required a fixed bit depth when creating fields because the existence bit was written to the bit above the highest bit. The second version of BSI moves the existence bit to the beginning, adds a negative bit as the second bit, and shifts all remaining bits up by two. + +Pilosa automatically converts all old data to the new format on startup, however, this can cause issues when upgrading Pilosa and then reverting back to an old version. This documentation section exists as a record for anyone who experiences unusual behavior in BSI between versions. + + #### Time Time fields are similar to `set` fields, but in addition to row and column information, they also store a per-bit time value down to a defined granularity. The following example creates a `time` field called "event" which stores timestamp information down to a day granularity. diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index d40db1fb9..24cfcd232 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -530,8 +530,8 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, - Min: o.Min, - Max: o.Max, + Base: o.Base, + BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), Keys: o.Keys, } @@ -798,8 +798,8 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) m.Type = options.Type m.CacheType = options.CacheType m.CacheSize = options.CacheSize - m.Min = options.Min - m.Max = options.Max + m.Base = options.Base + m.BitDepth = uint(options.BitDepth) m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) m.Keys = options.Keys } diff --git a/executor.go b/executor.go index 41077168e..2d05991ae 100644 --- a/executor.go +++ b/executor.go @@ -597,12 +597,12 @@ func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pq return ValCount{}, nil } - vsum, vcount, err := fragment.sum(filter, bsig.BitDepth()) + vsum, vcount, err := fragment.sum(filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } return ValCount{ - Val: int64(vsum) + (int64(vcount) * bsig.Min), + Val: int64(vsum) + (int64(vcount) * bsig.Base), Count: int64(vcount), }, nil } @@ -638,12 +638,12 @@ func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmin, fcount, err := fragment.min(filter, bsig.BitDepth()) + fmin, fcount, err := fragment.min(filter, bsig.BitDepth) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmin) + bsig.Min, + Val: int64(fmin) + bsig.Base, Count: int64(fcount), }, nil } @@ -679,12 +679,12 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmax, fcount, err := fragment.max(filter, bsig.BitDepth()) + fmax, fcount, err := fragment.max(filter, bsig.BitDepth) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmax) + bsig.Min, + Val: int64(fmax) + bsig.Base, Count: int64(fcount), }, nil } @@ -1405,7 +1405,7 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return NewRow(), nil } - return frag.notNull(bsig.BitDepth()) + return frag.notNull() } else if cond.Op == pql.BETWEEN { @@ -1442,11 +1442,11 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. - if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { - return frag.notNull(bsig.BitDepth()) + if predicates[0] <= bsig.Min() && predicates[1] >= bsig.Max() { + return frag.notNull() } - return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) + return frag.rangeBetween(bsig.BitDepth, baseValueMin, baseValueMax) } else { @@ -1474,18 +1474,18 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c } // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. - if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || - (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { - return frag.notNull(bsig.BitDepth()) + if (cond.Op == pql.LT && value > bsig.Max()) || (cond.Op == pql.LTE && value >= bsig.Max()) || + (cond.Op == pql.GT && value < bsig.Min()) || (cond.Op == pql.GTE && value <= bsig.Min()) { + return frag.notNull() } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.notNull(bsig.BitDepth()) + return frag.notNull() } f.Stats.Count("range:bsigroup", 1, 1.0) - return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue) + return frag.rangeOp(cond.Op, bsig.BitDepth, baseValue) } } diff --git a/executor_test.go b/executor_test.go index fd3090401..9473d02ca 100644 --- a/executor_test.go +++ b/executor_test.go @@ -769,7 +769,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) @@ -806,7 +806,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } @@ -1214,7 +1214,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10)); err != nil { t.Fatal(err) } @@ -1262,32 +1262,6 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } } }) - - t.Run("Max", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: 60, cnt: 1}, - {filter: `Row(x=0)`, exp: 60, cnt: 1}, - {filter: `Row(x=1)`, exp: -5, cnt: 1}, - {filter: `Row(x=2)`, exp: 40, cnt: 1}, - } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Max(field=f)` - } else { - pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } - } - }) }) t.Run("ColumnKey", func(t *testing.T) { @@ -1304,7 +1278,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10)); err != nil { t.Fatal(err) } @@ -1397,15 +1371,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } @@ -1455,15 +1429,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } @@ -1857,19 +1831,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100)); err != nil { t.Fatal(err) } @@ -1893,8 +1867,8 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Run("EQ", func(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) { + t.Fatalf("Query().Row.Columns=%#v, expected %#v", got, exp) } }) @@ -1903,20 +1877,20 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // NEQ if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // NEQ - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } }) @@ -1931,8 +1905,8 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Run("LTE", func(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}; !reflect.DeepEqual(got, exp) { + t.Fatalf("unexpected result: got=%v, exp=%v", got, exp) } }) @@ -1940,7 +1914,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) @@ -1948,7 +1922,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) @@ -2051,19 +2025,19 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100)); err != nil { t.Fatal(err) } @@ -2821,7 +2795,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100)) + _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0)) if err != nil { t.Fatal(err) } diff --git a/field.go b/field.go index 3b0b63feb..ec7a1ccd0 100644 --- a/field.go +++ b/field.go @@ -130,17 +130,14 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { // OptFieldTypeInt is a functional option on FieldOptions // used to specify the field as being type `int` and to // provide any respective configuration values. -func OptFieldTypeInt(min, max int64) FieldOption { +func OptFieldTypeInt(base int64) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) } - if min > max { - return ErrInvalidBSIGroupRange - } fo.Type = FieldTypeInt - fo.Min = min - fo.Max = max + fo.Base = base + fo.BitDepth = 1 return nil } } @@ -448,6 +445,22 @@ func (f *Field) openViews() error { if err := view.open(); err != nil { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } + + // Automatically upgrade BSI v1 fragments if they exist & reopen view. + if bsig := f.bsiGroup(f.name); bsig != nil { + if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil { + return errors.Wrap(err, "upgrade view bsi v2") + } else if ok { + if err := view.close(); err != nil { + return errors.Wrap(err, "closing upgraded view") + } + view = f.newView(f.viewPath(name), name) + if err := view.open(); err != nil { + return fmt.Errorf("re-opening view: view=%s, err=%s", view.name, err) + } + } + } + view.rowAttrStore = f.rowAttrStore f.logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name) f.viewMap[view.name] = view @@ -472,12 +485,18 @@ func (f *Field) loadMeta() error { } } + // Convert min/max from deprecated v1 int type. + if pb.Min != 0 || pb.Max != 0 { + pb.Base = pb.Min + pb.BitDepth = uint64(bitDepth(uint64(pb.Max - pb.Min))) + } + // Copy metadata fields. f.options.Type = pb.Type f.options.CacheType = pb.CacheType f.options.CacheSize = pb.CacheSize - f.options.Min = pb.Min - f.options.Max = pb.Max + f.options.Base = pb.Base + f.options.BitDepth = uint(pb.BitDepth) f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) f.options.Keys = pb.Keys f.options.NoStandardView = pb.NoStandardView @@ -521,25 +540,25 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = opt.CacheSize } } - f.options.Min = 0 - f.options.Max = 0 + f.options.Base = 0 + f.options.BitDepth = 0 f.options.TimeQuantum = "" f.options.Keys = opt.Keys case FieldTypeInt: f.options.Type = opt.Type f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 - f.options.Min = opt.Min - f.options.Max = opt.Max + f.options.Base = opt.Base + f.options.BitDepth = opt.BitDepth f.options.TimeQuantum = "" f.options.Keys = opt.Keys // Create new bsiGroup. bsig := &bsiGroup{ - Name: f.name, - Type: bsiGroupTypeInt, - Min: opt.Min, - Max: opt.Max, + Name: f.name, + Type: bsiGroupTypeInt, + Base: opt.Base, + BitDepth: opt.BitDepth, } // Validate bsiGroup. if err := bsig.validate(); err != nil { @@ -552,8 +571,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Type = opt.Type f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 - f.options.Min = 0 - f.options.Max = 0 + f.options.Base = 0 + f.options.BitDepth = 0 f.options.Keys = opt.Keys f.options.NoStandardView = opt.NoStandardView // Set the time quantum. @@ -565,8 +584,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Type = FieldTypeBool f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 - f.options.Min = 0 - f.options.Max = 0 + f.options.Base = 0 + f.options.BitDepth = 0 f.options.TimeQuantum = "" f.options.Keys = false default: @@ -965,25 +984,44 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { return 0, false, nil } - v, exists, err := view.value(columnID, bsig.BitDepth()) + v, exists, err := view.value(columnID, bsig.BitDepth) if err != nil { return 0, false, err } else if !exists { return 0, false, nil } - return int64(v) + bsig.Min, true, nil + return int64(v) + bsig.Base, true, nil } // SetValue sets a field value for a column. func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { - // Fetch bsiGroup and validate value. + // Fetch bsiGroup. bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound - } else if value < bsig.Min { - return false, ErrBSIGroupValueTooLow - } else if value > bsig.Max { - return false, ErrBSIGroupValueTooHigh + } + + // Determine base value to store. + baseValue := int64(value - bsig.Base) + + // Increase bit depth value if the unsigned value is greater. + if value < bsig.Min() || value > bsig.Max() { + if err := func() error { + f.mu.Lock() + defer f.mu.Unlock() + + uvalue := uint64(baseValue) + if value < 0 { + uvalue = uint64(-baseValue) + } + bitDepth := bitDepth(uvalue) + + bsig.BitDepth = bitDepth + f.options.BitDepth = bitDepth + return f.saveMeta() + }(); err != nil { + return false, errors.Wrap(err, "increasing bsi max") + } } // Fetch target view. @@ -992,10 +1030,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) return false, errors.Wrap(err, "creating view") } - // Determine base value to store. - baseValue := uint64(value - bsig.Min) - - return view.setValue(columnID, bsig.BitDepth(), baseValue) + return view.setValue(columnID, bsig.BitDepth, baseValue) } // Sum returns the sum and count for a field. @@ -1011,11 +1046,11 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { return 0, 0, nil } - vsum, vcount, err := view.sum(filter, bsig.BitDepth()) + vsum, vcount, err := view.sum(filter, bsig.BitDepth) if err != nil { return 0, 0, err } - return int64(vsum) + (int64(vcount) * bsig.Min), int64(vcount), nil + return int64(vsum) + (int64(vcount) * bsig.Base), int64(vcount), nil } // Min returns the min for a field. @@ -1031,11 +1066,11 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { return 0, 0, nil } - vmin, vcount, err := view.min(filter, bsig.BitDepth()) + vmin, vcount, err := view.min(filter, bsig.BitDepth) if err != nil { return 0, 0, err } - return int64(vmin) + bsig.Min, int64(vcount), nil + return int64(vmin) + bsig.Base, int64(vcount), nil } // Max returns the max for a field. @@ -1051,11 +1086,11 @@ func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { return 0, 0, nil } - vmax, vcount, err := view.max(filter, bsig.BitDepth()) + vmax, vcount, err := view.max(filter, bsig.BitDepth) if err != nil { return 0, 0, err } - return int64(vmax) + bsig.Min, int64(vcount), nil + return int64(vmax) + bsig.Base, int64(vcount), nil } // Range performs a conditional operation on Field. @@ -1064,7 +1099,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) bsig := f.bsiGroup(name) if bsig == nil { return nil, ErrBSIGroupNotFound - } else if predicate < bsig.Min || predicate > bsig.Max { + } else if predicate < bsig.Min() || predicate > bsig.Max() { return nil, nil } @@ -1079,7 +1114,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) return NewRow(), nil } - return view.rangeOp(op, bsig.BitDepth(), baseValue) + return view.rangeOp(op, bsig.BitDepth, baseValue) } // Import bulk imports data. @@ -1172,15 +1207,40 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return errors.Wrap(ErrBSIGroupNotFound, f.name) } + // Find the lowest/highest values. + var min, max int64 + for i, value := range values { + if i == 0 || value < min { + min = value + } + if i == 0 || value > max { + max = value + } + } + + // Determine the highest bit depth required by the min & max. + requiredDepth := bitDepthInt64(min - bsig.Base) + if v := bitDepthInt64(max - bsig.Base); v > requiredDepth { + requiredDepth = v + } + + // Increase bit depth if required. + if requiredDepth > bsig.BitDepth { + if err := func() error { + f.mu.Lock() + defer f.mu.Unlock() + bsig.BitDepth = requiredDepth + f.options.BitDepth = requiredDepth + return f.saveMeta() + }(); err != nil { + return errors.Wrap(err, "increasing bsi bit depth") + } + } + // Split import data by fragment. dataByFragment := make(map[importKey]importValueData) for i := range columnIDs { columnID, value := columnIDs[i], values[i] - if value > bsig.Max { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value) - } else if value < bsig.Min { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value) - } // Attach value to each bsiGroup view. for _, name := range []string{viewName} { @@ -1194,7 +1254,6 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO // Import into each fragment. for key, data := range dataByFragment { - // The view must already exist (i.e. we can't create it) // because we need to know bitDepth (based on min/max value). view, err := f.createViewIfNotExists(key.View) @@ -1207,12 +1266,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return errors.Wrap(err, "creating fragment") } - baseValues := make([]uint64, len(data.Values)) + baseValues := make([]int64, len(data.Values)) for i, value := range data.Values { - baseValues[i] = uint64(value - bsig.Min) + baseValues[i] = value - bsig.Base } - if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth(), options.Clear); err != nil { + if err := frag.importValue(data.ColumnIDs, baseValues, requiredDepth, options.Clear); err != nil { return err } } @@ -1266,14 +1325,18 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FieldOptions represents options to set when initializing a field. type FieldOptions struct { - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` + Base int64 `json:"base,omitempty"` + BitDepth uint `json:"bitDepth,omitempty"` Keys bool `json:"keys"` NoStandardView bool `json:"noStandardView,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` CacheType string `json:"cacheType,omitempty"` Type string `json:"type,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + + // Deprecated. Use base/bit depth. + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` } // applyDefaultOptions returns a new FieldOptions object @@ -1302,8 +1365,8 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, - Min: o.Min, - Max: o.Max, + Base: o.Base, + BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), Keys: o.Keys, NoStandardView: o.NoStandardView, @@ -1329,14 +1392,14 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { }) case FieldTypeInt: return json.Marshal(struct { - Type string `json:"type"` - Min int64 `json:"min"` - Max int64 `json:"max"` - Keys bool `json:"keys"` + Type string `json:"type"` + Base int64 `json:"base"` + BitDepth uint `json:"bitDepth"` + Keys bool `json:"keys"` }{ o.Type, - o.Min, - o.Max, + o.Base, + o.BitDepth, o.Keys, }) case FieldTypeTime: @@ -1389,20 +1452,20 @@ func isValidBSIGroupType(v string) bool { // bsiGroup represents a group of range-encoded rows on a field. type bsiGroup struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Base int64 `json:"base,omitempty"` + BitDepth uint `json:"bitDepth,omitempty"` } -// BitDepth returns the number of bits required to store a value between min & max. -func (b *bsiGroup) BitDepth() uint { - for i := uint(0); i < 63; i++ { - if b.Max-b.Min < (1 << i) { - return i - } - } - return 63 +// Min returns the lowest possible value for the group based on current bit depth. +func (b *bsiGroup) Min() int64 { + return b.Base - (1 << b.BitDepth) + 1 +} + +// Max returns the highest possible value for the group based on current bit depth. +func (b *bsiGroup) Max() int64 { + return b.Base + (1 << b.BitDepth) - 1 } // baseValue adjusts the value to align with the range for Field for a certain @@ -1417,44 +1480,48 @@ func (b *bsiGroup) BitDepth() uint { // In order to make this work, we effectively need to change the operator to LTE. // Executor.executeBSIGroupRangeShard() takes this into account and returns // `frag.FieldNotNull(bsig.BitDepth())` in such instances. -func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { +func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue int64, outOfRange bool) { + min, max := b.Min(), b.Max() + if op == pql.GT || op == pql.GTE { - if value > b.Max { + if value > max { return baseValue, true - } else if value > b.Min { - baseValue = uint64(value - b.Min) + } else if value > min { + baseValue = int64(value - b.Base) } } else if op == pql.LT || op == pql.LTE { - if value < b.Min { + if value < min { return baseValue, true - } else if value > b.Max { - baseValue = uint64(b.Max - b.Min) + } else if value > max { + baseValue = int64(max - b.Base) } else { - baseValue = uint64(value - b.Min) + baseValue = int64(value - b.Base) } } else if op == pql.EQ || op == pql.NEQ { - if value < b.Min || value > b.Max { + if value < min || value > max { return baseValue, true } - baseValue = uint64(value - b.Min) + baseValue = int64(value - b.Base) } return baseValue, false } // baseValueBetween adjusts the min/max value to align with the range for Field. -func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { - if max < b.Min || min > b.Max { +func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax int64, outOfRange bool) { + bsiMin, bsiMax := b.Min(), b.Max() + + if max < bsiMin || min > bsiMax { return baseValueMin, baseValueMax, true } // Adjust min/max to range. - if min > b.Min { - baseValueMin = uint64(min - b.Min) + if min > bsiMin { + baseValueMin = int64(min - b.Base) } // Make sure the high value of the BETWEEN does not exceed BitDepth. - if max > b.Max { - baseValueMax = uint64(b.Max - b.Min) - } else if max > b.Min { - baseValueMax = uint64(max - b.Min) + if max > bsiMax { + baseValueMax = int64(bsiMax - b.Base) + } else if max > bsiMin { + baseValueMax = int64(max - b.Base) } return baseValueMin, baseValueMax, false } @@ -1464,8 +1531,6 @@ func (b *bsiGroup) validate() error { return ErrBSIGroupNameRequired } else if !isValidBSIGroupType(b.Type) { return ErrInvalidBSIGroupType - } else if b.Min > b.Max { - return ErrInvalidBSIGroupRange } return nil } @@ -1486,3 +1551,21 @@ func isValidCacheType(v string) bool { return false } } + +// bitDepth returns the number of bits required to store a value. +func bitDepth(v uint64) uint { + for i := uint(0); i < 63; i++ { + if v < (1 << i) { + return i + } + } + return 63 +} + +// bitDepthInt64 returns the required bit depth for abs(v). +func bitDepthInt64(v int64) uint { + if v < 0 { + return bitDepth(uint64(-v)) + } + return bitDepth(uint64(v)) +} diff --git a/field_internal_test.go b/field_internal_test.go index ad8337c31..aa8692916 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -28,125 +28,120 @@ import ( // Ensure a bsiGroup can adjust to its baseValue. func TestBSIGroup_BaseValue(t *testing.T) { b0 := &bsiGroup{ - Name: "b0", - Type: bsiGroupTypeInt, - Min: -100, - Max: 900, + Name: "b0", + Type: bsiGroupTypeInt, + Base: -100, + BitDepth: 10, } b1 := &bsiGroup{ - Name: "b1", - Type: bsiGroupTypeInt, - Min: 0, - Max: 1000, + Name: "b1", + Type: bsiGroupTypeInt, + Base: 0, + BitDepth: 8, } b2 := &bsiGroup{ - Name: "b2", - Type: bsiGroupTypeInt, - Min: 100, - Max: 1100, + Name: "b2", + Type: bsiGroupTypeInt, + Base: 100, + BitDepth: 11, } t.Run("Normal Condition", func(t *testing.T) { - - for _, tt := range []struct { + for i, tt := range []struct { f *bsiGroup op pql.Token val int64 - expBaseValue uint64 + expBaseValue int64 expOutOfRange bool }{ // LT {b0, pql.LT, 5, 105, false}, {b0, pql.LT, -8, 92, false}, - {b0, pql.LT, -108, 0, true}, - {b0, pql.LT, 1005, 1000, false}, + {b0, pql.LT, -108, -8, false}, + {b0, pql.LT, 1005, 1023, false}, {b0, pql.LT, 0, 100, false}, {b1, pql.LT, 5, 5, false}, - {b1, pql.LT, -8, 0, true}, - {b1, pql.LT, 1005, 1000, false}, + {b1, pql.LT, -8, -8, false}, + {b1, pql.LT, 1005, 255, false}, {b1, pql.LT, 0, 0, false}, - {b2, pql.LT, 5, 0, true}, - {b2, pql.LT, -8, 0, true}, + {b2, pql.LT, 5, -95, false}, + {b2, pql.LT, -8, -108, false}, {b2, pql.LT, 105, 5, false}, - {b2, pql.LT, 1105, 1000, false}, + {b2, pql.LT, 1105, 1005, false}, // GT - {b0, pql.GT, -105, 0, false}, + {b0, pql.GT, -5, 95, false}, {b0, pql.GT, 5, 105, false}, - {b0, pql.GT, 905, 0, true}, + {b0, pql.GT, 905, 1005, false}, {b0, pql.GT, 0, 100, false}, {b1, pql.GT, 5, 5, false}, - {b1, pql.GT, -8, 0, false}, + {b1, pql.GT, -8, -8, false}, {b1, pql.GT, 1005, 0, true}, {b1, pql.GT, 0, 0, false}, - {b2, pql.GT, 5, 0, false}, - {b2, pql.GT, -8, 0, false}, + {b2, pql.GT, 5, -95, false}, + {b2, pql.GT, -8, -108, false}, {b2, pql.GT, 105, 5, false}, - {b2, pql.GT, 1105, 0, true}, + {b2, pql.GT, 1105, 1005, false}, // EQ - {b0, pql.EQ, -105, 0, true}, + {b0, pql.EQ, -105, -5, false}, {b0, pql.EQ, 5, 105, false}, - {b0, pql.EQ, 905, 0, true}, + {b0, pql.EQ, 905, 1005, false}, {b0, pql.EQ, 0, 100, false}, {b1, pql.EQ, 5, 5, false}, - {b1, pql.EQ, -8, 0, true}, + {b1, pql.EQ, -8, -8, false}, {b1, pql.EQ, 1005, 0, true}, {b1, pql.EQ, 0, 0, false}, - {b2, pql.EQ, 5, 0, true}, - {b2, pql.EQ, -8, 0, true}, + {b2, pql.EQ, 5, -95, false}, + {b2, pql.EQ, -8, -108, false}, {b2, pql.EQ, 105, 5, false}, - {b2, pql.EQ, 1105, 0, true}, + {b2, pql.EQ, 1105, 1005, false}, } { bv, oor := tt.f.baseValue(tt.op, tt.val) - if oor != tt.expOutOfRange { - t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(bv, tt.expBaseValue) { - t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) + if oor != tt.expOutOfRange || !reflect.DeepEqual(bv, tt.expBaseValue) { + t.Errorf("%d. %s) baseValue(%s, %v)=(%v, %v), expected (%v, %v)", i, tt.f.Name, tt.op, tt.val, bv, oor, tt.expBaseValue, tt.expOutOfRange) } } }) - t.Run("Betwween Condition", func(t *testing.T) { - for _, tt := range []struct { + t.Run("Between Condition", func(t *testing.T) { + for i, tt := range []struct { f *bsiGroup predMin int64 predMax int64 - expBaseValueMin uint64 - expBaseValueMax uint64 + expBaseValueMin int64 + expBaseValueMax int64 expOutOfRange bool }{ - {b0, -205, -105, 0, 0, true}, - {b0, -105, 80, 0, 180, false}, + {b0, -205, -105, -105, -5, false}, + {b0, -105, 80, -5, 180, false}, {b0, 5, 20, 105, 120, false}, - {b0, 20, 1005, 120, 1000, false}, + {b0, 20, 1005, 120, 1023, false}, {b0, 1005, 2000, 0, 0, true}, - {b1, -105, -5, 0, 0, true}, - {b1, -5, 20, 0, 20, false}, + {b1, -105, -5, -105, -5, false}, + {b1, -5, 20, -5, 20, false}, {b1, 5, 20, 5, 20, false}, - {b1, 20, 1005, 20, 1000, false}, + {b1, 20, 1005, 20, 255, false}, {b1, 1005, 2000, 0, 0, true}, - {b2, 5, 95, 0, 0, true}, - {b2, 95, 120, 0, 20, false}, + {b2, 5, 95, -95, -5, false}, + {b2, 95, 120, -5, 20, false}, {b2, 105, 120, 5, 20, false}, - {b2, 120, 1105, 20, 1000, false}, - {b2, 1105, 2000, 0, 0, true}, + {b2, 120, 1105, 20, 1005, false}, + {b2, 1105, 2000, 1005, 1900, false}, } { min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax) - if oor != tt.expOutOfRange { - t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { - t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) + if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) || oor != tt.expOutOfRange { + t.Errorf("%d. %s) baseValueBetween(%v, %v)=(%v, %v, %v), expected (%v, %v, %v)", i, tt.f.Name, tt.predMin, tt.predMax, min, max, oor, tt.expBaseValueMin, tt.expBaseValueMax, tt.expOutOfRange) } } }) diff --git a/field_test.go b/field_test.go index 88a3f3569..72e700211 100644 --- a/field_test.go +++ b/field_test.go @@ -30,7 +30,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0)) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0)) if err != nil { t.Fatal(err) } @@ -106,36 +106,6 @@ func TestField_SetValue(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - - t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh { - t.Fatalf("unexpected error: %s", err) - } - }) } func TestField_NameRestriction(t *testing.T) { diff --git a/fragment.go b/fragment.go index 8e8fa0ce5..a67ea7875 100644 --- a/fragment.go +++ b/fragment.go @@ -83,6 +83,14 @@ const ( // Row ids used for boolean fields. falseRowID = uint64(0) trueRowID = uint64(1) + + // BSI bits used to check existence & sign. + bsiExistsBit = 0 + bsiSignBit = 1 + bsiOffsetBit = 2 + + // Roaring bitmap flags. + roaringFlagBSIv2 = 0x01 // indicates version using low bit for existence ) // fragment represents the intersection of a field and shard in an index. @@ -97,6 +105,7 @@ type fragment struct { // File-backed storage path string + flags byte // user-defined flags passed to roaring file *os.File storage *roaring.Bitmap storageData []byte @@ -136,13 +145,14 @@ type fragment struct { } // newFragment returns a new instance of Fragment. -func newFragment(path, index, field, view string, shard uint64) *fragment { +func newFragment(path, index, field, view string, shard uint64, flags byte) *fragment { return &fragment{ path: path, index: index, field: field, view: view, shard: shard, + flags: flags, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, @@ -208,6 +218,7 @@ func (f *fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() + f.storage.Flags = f.flags } // Open the data file to be mmap'd and used as an ops log. file, mustClose, err := syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) @@ -679,12 +690,12 @@ func (f *fragment) bit(rowID, columnID uint64) (bool, error) { } // value uses a column of bits to read a multi-bit value. -func (f *fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +func (f *fragment) value(columnID uint64, bitDepth uint) (value int64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() // If existence bit is unset then ignore remaining bits. - if v, err := f.bit(uint64(bitDepth), columnID); err != nil { + if v, err := f.bit(bsiExistsBit, columnID); err != nil { return 0, false, errors.Wrap(err, "getting existence bit") } else if !v { return 0, false, nil @@ -692,55 +703,75 @@ func (f *fragment) value(columnID uint64, bitDepth uint) (value uint64, exists b // Compute other bits into a value. for i := uint(0); i < bitDepth; i++ { - if v, err := f.bit(uint64(i), columnID); err != nil { + if v, err := f.bit(uint64(bsiOffsetBit+i), columnID); err != nil { return 0, false, errors.Wrapf(err, "getting value bit %d", i) } else if v { value |= (1 << i) } } + // Negate if sign bit set. + if v, err := f.bit(bsiSignBit, columnID); err != nil { + return 0, false, errors.Wrap(err, "getting sign bit") + } else if v { + value = -value + } + return value, true, nil } // clearValue uses a column of bits to clear a multi-bit value. -func (f *fragment) clearValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +func (f *fragment) clearValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { return f.setValueBase(columnID, bitDepth, value, true) } // setValue uses a column of bits to set a multi-bit value. -func (f *fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +func (f *fragment) setValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { return f.setValueBase(columnID, bitDepth, value, false) } -func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value uint64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) { +func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) { + // Convert value to an unsigned representation. + uvalue := uint64(value) + if value < 0 { + uvalue = uint64(-value) + } + + // Mark value as set. + if bit, err := f.pos(bsiExistsBit, columnID); err != nil { + return toSet, toClear, errors.Wrap(err, "getting not-null pos") + } else if clear { + toClear = append(toClear, bit) + } else { + toSet = append(toSet, bit) + } + + // Mark sign. + if bit, err := f.pos(bsiSignBit, columnID); err != nil { + return toSet, toClear, errors.Wrap(err, "getting sign pos") + } else if value >= 0 || clear { + toClear = append(toClear, bit) + } else { + toSet = append(toSet, bit) + } + for i := uint(0); i < bitDepth; i++ { - bit, err := f.pos(uint64(i), columnID) + bit, err := f.pos(uint64(bsiOffsetBit+i), columnID) if err != nil { return toSet, toClear, errors.Wrap(err, "getting pos") } - if value&(1<= 0 || clear { + if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil { + return changed, errors.Wrap(err, "clearing sign") + } else if c { + changed = true + } + } else { + if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil { + return changed, errors.Wrap(err, "marking sign") + } else if c { + changed = true + } + } + return changed, nil } // importSetValue is a more efficient SetValue just for imports. -func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value uint64, clear bool) (changed bool, err error) { // nolint: unparam +func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) { // nolint: unparam + // Convert value to an unsigned representation. + uvalue := uint64(value) + if value < 0 { + uvalue = uint64(-value) + } + for i := uint(0); i < bitDepth; i++ { - if value&(1<= 0 || clear { + if c, err := f.storage.Remove(p); err != nil { + return changed, errors.Wrap(err, "removing sign from storage") + } else if c { + changed = true + } + } else { + if c, err := f.storage.Add(p); err != nil { + return changed, errors.Wrap(err, "adding sign to storage") + } else if c { + changed = true + } + } + return changed, nil } // sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (f *fragment) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err error) { // Compute count based on the existence row. - consider := f.row(uint64(bitDepth)) + consider := f.row(bsiExistsBit) if filter != nil { consider = consider.Intersect(filter) } count = consider.Count() + // Determine positive & negative sets. + nrow := f.row(bsiSignBit) + prow := consider.Difference(nrow) + // Compute the sum based on the bit count of each row multiplied by the // place value of each row. For example, 10 bits in the 1's place plus // 4 bits in the 2's place plus 3 bits in the 4's place equals a total @@ -850,11 +924,16 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // // 10*(2^0) + 4*(2^1) + 3*(2^2) = 30 // - var cnt uint64 + // Execute once for positive numbers and once for negative. Subtract the + // negative sum from the positive sum. for i := uint(0); i < bitDepth; i++ { - row := f.row(uint64(i)) - cnt = row.intersectionCount(consider) - sum += (1 << i) * cnt + row := f.row(uint64(bsiOffsetBit + i)) + + psum := int64((1 << i) * row.intersectionCount(prow)) + nsum := int64((1 << i) * row.intersectionCount(nrow)) + + // Squash to reduce the possibility of overflow. + sum += psum - nsum } return sum, count, nil @@ -862,9 +941,8 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // min returns the min of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error) { - - consider := f.row(uint64(bitDepth)) +func (f *fragment) min(filter *Row, bitDepth uint) (min int64, count uint64, err error) { + consider := f.row(bsiExistsBit) if filter != nil { consider = consider.Intersect(filter) } @@ -874,58 +952,79 @@ func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error return 0, 0, nil } - for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.row(uint64(ii)) + // If we have negative values, we should find the highest unsigned value + // from that set, then negate it, and return it. For example, if values + // (-1, -2) exist, they are stored unsigned (1,2) with a negative sign bit + // set. We take the highest of that set (2) and negate it and return it. + if row := f.row(bsiSignBit).Intersect(consider); row.Any() { + min, count := f.maxUnsigned(row, bitDepth) + return -min, count, nil + } - x := consider.Difference(row) - count = x.Count() + // Otherwise find lowest positive number. + min, count = f.minUnsigned(consider, bitDepth) + return min, count, nil +} + +// minUnsigned the lowest value without considering the sign bit. Filter is required. +func (f *fragment) minUnsigned(filter *Row, bitDepth uint) (min int64, count uint64) { + for i := int(bitDepth - 1); i >= 0; i-- { + row := filter.Difference(f.row(uint64(bsiOffsetBit + i))) + count = row.Count() if count > 0 { - consider = x + filter = row } else { - min += (1 << ii) - if ii == 0 { - count = consider.Count() + min += (1 << uint(i)) + if i == 0 { + count = filter.Count() } } } - - return min, count, nil + return min, count } // max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - - consider := f.row(uint64(bitDepth)) +func (f *fragment) max(filter *Row, bitDepth uint) (max int64, count uint64, err error) { + consider := f.row(bsiExistsBit) if filter != nil { consider = consider.Intersect(filter) } // If there are no columns to consider, return early. - if consider.Count() == 0 { + if !consider.Any() { return 0, 0, nil } - for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.row(uint64(ii)) - - x := row.Intersect(consider) - count = x.Count() - if count > 0 { - max += (1 << ii) - consider = x - } else if ii == 0 { - count = consider.Count() - } + // Find lowest negative number w/o sign and negate, if no positives are available. + pos := consider.Difference(f.row(bsiSignBit)) + if !pos.Any() { + max, count = f.minUnsigned(consider, bitDepth) + return -max, count, nil } + // Otherwise find highest positive number. + max, count = f.maxUnsigned(pos, bitDepth) return max, count, nil } +// maxUnsigned the highest value without considering the sign bit. Filter is required. +func (f *fragment) maxUnsigned(filter *Row, bitDepth uint) (max int64, count uint64) { + for i := int(bitDepth - 1); i >= 0; i-- { + row := f.row(uint64(bsiOffsetBit + i)).Intersect(filter) + count = row.Count() + if count > 0 { + max += (1 << uint(i)) + filter = row + } else if i == 0 { + count = filter.Count() + } + } + return max, count +} + // rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. -func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) { switch op { case pql.EQ: return f.rangeEQ(bitDepth, predicate) @@ -940,14 +1039,23 @@ func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, } } -func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. - b := f.row(uint64(bitDepth)) + b := f.row(bsiExistsBit) + + // Filter to only positive/negative numbers. + upredicate := uint64(predicate) + if predicate < 0 { + upredicate = uint64(-predicate) + b = b.Intersect(f.row(bsiSignBit)) // only negatives + } else { + b = b.Difference(f.row(bsiSignBit)) // only positives + } // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) - bit := (predicate >> uint(i)) & 1 + row := f.row(uint64(bsiOffsetBit + i)) + bit := (upredicate >> uint(i)) & 1 if bit == 1 { b = b.Intersect(row) @@ -959,9 +1067,9 @@ func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. - b := f.row(uint64(bitDepth)) + b := f.row(bsiExistsBit) // Get the equal bitmap. eq, err := f.rangeEQ(bitDepth, predicate) @@ -975,22 +1083,44 @@ func (f *fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - keep := NewRow() - +func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { // Start with set of columns with values set. - b := f.row(uint64(bitDepth)) + b := f.row(bsiExistsBit) + + // Create predicate without sign bit. + upredicate := uint64(predicate) + if predicate < 0 { + upredicate = uint64(-predicate) + } + + // If predicate is positive, return all positives less than predicate and all negatives. + if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { + pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + neg := f.row(bsiSignBit) + return neg.Union(pos), nil + } + + // Otherwise if predicate is negative, return all negatives greater than upredicate. + return f.rangeGTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) +} + +// rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. +func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { + keep := NewRow() // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) + row := f.row(uint64(bsiOffsetBit + i)) bit := (predicate >> uint(i)) & 1 // Remove any columns with higher bits set. if leadingZeros { if bit == 0 { - b = b.Difference(row) + filter = filter.Difference(row) continue } else { leadingZeros = false @@ -1004,32 +1134,54 @@ func (f *fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) if bit == 0 { return keep, nil } - return b.Difference(row.Difference(keep)), nil + return filter.Difference(row.Difference(keep)), nil } // If bit is zero then remove all set columns not in excluded bitmap. if bit == 0 { - b = b.Difference(row.Difference(keep)) + filter = filter.Difference(row.Difference(keep)) continue } // If bit is set then add columns for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { - keep = keep.Union(b.Difference(row)) + keep = keep.Union(filter.Difference(row)) } } - return b, nil + return filter, nil } -func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.row(uint64(bitDepth)) +func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { + b := f.row(bsiExistsBit) + + // Create predicate without sign bit. + upredicate := uint64(predicate) + if predicate < 0 { + upredicate = uint64(-predicate) + } + + // If predicate is positive, return all positives greater than predicate. + if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { + return f.rangeGTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + } + + // If predicate is negative, return all negatives less than than upredicate and all positives. + neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + pos := b.Difference(f.row(bsiSignBit)) + return pos.Union(neg), nil +} + +func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) + row := f.row(uint64(bsiOffsetBit + i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. @@ -1039,68 +1191,102 @@ func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) if bit == 1 { return keep, nil } - return b.Difference(b.Difference(row).Difference(keep)), nil + return filter.Difference(filter.Difference(row).Difference(keep)), nil } // If bit is set then remove all unset columns not already kept. if bit == 1 { - b = b.Difference(b.Difference(row).Difference(keep)) + filter = filter.Difference(filter.Difference(row).Difference(keep)) continue } // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep = keep.Union(b.Intersect(row)) + keep = keep.Union(filter.Intersect(row)) } } - return b, nil + return filter, nil } -// notNull returns the not-null row (stored at bitDepth). -func (f *fragment) notNull(bitDepth uint) (*Row, error) { - return f.row(uint64(bitDepth)), nil +// notNull returns the exists row. +func (f *fragment) notNull() (*Row, error) { + return f.row(bsiExistsBit), nil } // rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. -func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.row(uint64(bitDepth)) +func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) (*Row, error) { + b := f.row(bsiExistsBit) + + // Convert predicates to unsigned values. + upredicateMin, upredicateMax := uint64(predicateMin), uint64(predicateMax) + if predicateMin < 0 { + upredicateMin = uint64(-predicateMin) + } + if predicateMax < 0 { + upredicateMax = uint64(-predicateMax) + } + + // Handle positive-only values. + if predicateMin >= 0 { + return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax) + } + + // Handle negative-only values. Swap unsigned min/max predicates. + if predicateMax < 0 { + return f.rangeBetweenUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMax, upredicateMin) + } + + // If predicate crosses positive/negative boundary then handle separately and union. + pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) + if err != nil { + return nil, err + } + neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) + if err != nil { + return nil, err + } + return pos.Union(neg), nil +} + +// rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. +func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) + row := f.row(uint64(bsiOffsetBit + i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 // GTE predicateMin // If bit is set then remove all unset columns not already kept. if bit1 == 1 { - b = b.Difference(b.Difference(row).Difference(keep1)) + filter = filter.Difference(filter.Difference(row).Difference(keep1)) } else { // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep1 = keep1.Union(b.Intersect(row)) + keep1 = keep1.Union(filter.Intersect(row)) } } // LTE predicateMin // If bit is zero then remove all set bits not in excluded bitmap. if bit2 == 0 { - b = b.Difference(row.Difference(keep2)) + filter = filter.Difference(row.Difference(keep2)) } else { // If bit is set then add columns for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { - keep2 = keep2.Union(b.Difference(row)) + keep2 = keep2.Union(filter.Difference(row)) } } } - return b, nil + return filter, nil } // pos translates the row ID and column ID into a position in the storage bitmap. @@ -1735,7 +1921,7 @@ func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error { return errors.Wrap(f.importPositions(toSet, toClear, rowSet), "importing positions") } -func (f *fragment) importValueSmallWrite(columnIDs, values []uint64, bitDepth uint, clear bool) error { +func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bitDepth uint, clear bool) error { // TODO figure out how to avoid re-allocating these each time. Probably // possible to store them on the fragment with a capacity based on // MaxOpN. For now, we know that the total number of bits to be @@ -1751,6 +1937,7 @@ func (f *fragment) importValueSmallWrite(columnIDs, values []uint64, bitDepth ui if _, ok := colSet[columnID]; ok { continue } + colSet[columnID] = struct{}{} toSet, toClear, err = f.positionsForValue(columnID, bitDepth, value, clear, toSet, toClear) if err != nil { @@ -1772,7 +1959,7 @@ func (f *fragment) importValueSmallWrite(columnIDs, values []uint64, bitDepth ui } // importValue bulk imports a set of range-encoded values. -func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear bool) error { +func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint, clear bool) error { f.mu.Lock() defer f.mu.Unlock() @@ -1783,7 +1970,6 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear if len(columnIDs)*int(bitDepth+1)+f.opN < f.MaxOpN { return errors.Wrap(f.importValueSmallWrite(columnIDs, values, bitDepth, clear), "import small write") - } f.storage.OpWriter = nil @@ -2288,6 +2474,51 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { return rows } +// upgradeRoaringBSIv2 upgrades a fragment that contains old BSI formatting +// to a new BSI format (v2). The new format moves the "exists" bit to the +// beginning & adds a negative sign bit. +func upgradeRoaringBSIv2(f *fragment, bitDepth uint) (string, error) { + // If flag set, already upgraded. Exit. + if f.storage.Flags&roaringFlagBSIv2 == 1 { + return "", nil + } + + other := roaring.NewBitmap() + other.Flags = roaringFlagBSIv2 + func() { + f.mu.Lock() + defer f.mu.Unlock() + + f.storage.ForEach(func(i uint64) { + rowID, columnID := i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth) + if rowID == uint64(bitDepth) { + other.Add(pos(bsiExistsBit, columnID)) // move exists bit to beginning + } else { + other.Add(pos(rowID+bsiOffsetBit, columnID)) // move other bits up + } + }) + }() + + // Create temporary file next to existing file. + newPath := f.path + ".tmp" + file, err := os.OpenFile(newPath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return "", err + } + defer file.Close() + + // Write & flush to temporary file. + if _, err := other.WriteTo(file); err != nil { + return "", err + } else if err := file.Sync(); err != nil { + return "", err + } else if err := file.Close(); err != nil { + return "", err + } + + return newPath, nil +} + type rowIterator struct { f *fragment rowIDs []uint64 diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 43b11357c..ecc506814 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -276,7 +276,7 @@ func TestFragment_SetValue(t *testing.T) { } // Non-existent value. - if value, exists, err := f.value(100, 11); err != nil { + if value, exists, err := f.value(101, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -305,7 +305,7 @@ func TestFragment_SetValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.setValue(columnID, bitDepth, value); err != nil { + if _, err := f.setValue(columnID, bitDepth, int64(value)); err != nil { t.Fatal(err) } } @@ -409,7 +409,7 @@ func TestFragment_MinMax(t *testing.T) { t.Run("Min", func(t *testing.T) { tests := []struct { filter *Row - exp uint64 + exp int64 cnt uint64 }{ {filter: nil, exp: 0, cnt: 1}, @@ -433,7 +433,7 @@ func TestFragment_MinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { tests := []struct { filter *Row - exp uint64 + exp int64 cnt uint64 }{ {filter: nil, exp: 2818, cnt: 2}, @@ -444,12 +444,15 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { + var columns []uint64 + if test.filter != nil { + columns = test.filter.Columns() + } + if max, cnt, err := f.max(test.filter, bitDepth); err != nil { t.Fatal(err) - } else if max != test.exp { - t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) - } else if cnt != test.cnt { - t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } else if max != test.exp || cnt != test.cnt { + t.Errorf("%d. max(%v, %v)=(%v, %v), expected (%v, %v)", i, columns, bitDepth, max, cnt, test.exp, test.cnt) } } }) @@ -657,7 +660,7 @@ func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uin for i := 0; i < b.N; i++ { // We're not checking the error because this is a benchmark. // That does mean the result could be completely wrong... - _, _ = f.setValue(column, bitDepth, uint64(i)) + _, _ = f.setValue(column, bitDepth, int64(i)) column = cfunc(column) } } @@ -686,9 +689,9 @@ func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func( column := uint64(0) b.StopTimer() columns := make([]uint64, b.N) - values := make([]uint64, b.N) + values := make([]int64, b.N) for i := 0; i < b.N; i++ { - values[i] = uint64(i) + values[i] = int64(i) columns[i] = column column = cfunc(column) } @@ -805,23 +808,23 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { initialCols := make([]uint64, 0, ShardWidth) - initialVals := make([]uint64, 0, ShardWidth) + initialVals := make([]int64, 0, ShardWidth) for i := uint64(0); i < ShardWidth; i++ { // every 29 columns, skip between 0 and 12 columns if i%29 == 0 { i += i % 13 } initialCols = append(initialCols, i) - initialVals = append(initialVals, uint64(rand.Int63n(1<<21))) + initialVals = append(initialVals, int64(rand.Int63n(1<<21))) } for _, numUpdates := range []int{100} { for _, valsPerUpdate := range []int{10, 100} { updateCols := make([]uint64, numUpdates*valsPerUpdate) - updateVals := make([]uint64, numUpdates*valsPerUpdate) + updateVals := make([]int64, numUpdates*valsPerUpdate) for i := 0; i < numUpdates*valsPerUpdate; i++ { updateCols[i] = uint64(rand.Int63n(ShardWidth)) - updateVals[i] = uint64(rand.Int63n(1 << 21)) + updateVals[i] = int64(rand.Int63n(1 << 21)) } for _, opN := range []int{1, 5000, 50000} { @@ -1372,7 +1375,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := newFragment(*FragmentPath, "i", "f", viewStandard, 0) + f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1901,7 +1904,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.ReportAllocs() // Open the fragment specified by the path. - f := newFragment(*FragmentPath, "i", "f", viewStandard, 0) + f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -2231,7 +2234,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { } origF.Close() fi.Close() - nf := newFragment(fi.Name(), "i", "f", viewStandard, 0) + nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0) err = nf.Open() if err != nil { b.Fatalf("opening fragment: %v", err) @@ -2268,7 +2271,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { } origF.Close() fi.Close() - nf := newFragment(fi.Name(), "i", "f", viewStandard, 0) + nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0) err = nf.Open() if err != nil { b.Fatalf("opening fragment: %v", err) @@ -2463,7 +2466,7 @@ func mustOpenFragment(index, field, view string, shard uint64, cacheType string) cacheType = DefaultCacheType } - f := newFragment(file.Name(), index, field, view, shard) + f := newFragment(file.Name(), index, field, view, shard, 0) f.CacheType = cacheType f.RowAttrStore = &memAttrStore{ store: make(map[uint64]map[string]interface{}), @@ -2988,7 +2991,7 @@ func TestFragmentPositionsForValue(t *testing.T) { tests := []struct { columnID uint64 bitDepth uint - value uint64 + value int64 clear bool toSet []uint64 toClear []uint64 @@ -2997,43 +3000,43 @@ func TestFragmentPositionsForValue(t *testing.T) { columnID: 0, bitDepth: 1, value: 0, - toSet: []uint64{ShardWidth}, - toClear: []uint64{0}, + toSet: []uint64{0}, // exists bit only + toClear: []uint64{ShardWidth, ShardWidth * 2}, // sign bit & 1-position }, { columnID: 0, bitDepth: 3, value: 0, - toSet: []uint64{ShardWidth * 3}, - toClear: []uint64{0, ShardWidth, ShardWidth * 2}, + toSet: []uint64{0}, // exists bit only + toClear: []uint64{ShardWidth * 1, ShardWidth * 2, ShardWidth * 3, ShardWidth * 4}, // sign bit, 1, 2, 4 }, { columnID: 1, bitDepth: 3, value: 0, - toSet: []uint64{ShardWidth*3 + 1}, - toClear: []uint64{1, ShardWidth + 1, ShardWidth*2 + 1}, + toSet: []uint64{1}, // exists bit only + toClear: []uint64{ShardWidth + 1, ShardWidth*2 + 1, ShardWidth*3 + 1, ShardWidth*4 + 1}, // sign bit, 1, 2, 4 }, { columnID: 0, bitDepth: 1, value: 1, - toSet: []uint64{0, ShardWidth}, - toClear: []uint64{}, + toSet: []uint64{0, ShardWidth * 2}, // exists bit, 1 + toClear: []uint64{ShardWidth}, // sign bit only }, { columnID: 0, bitDepth: 4, value: 10, - toSet: []uint64{ShardWidth, ShardWidth * 3, ShardWidth * 4}, - toClear: []uint64{0, ShardWidth * 2}, + toSet: []uint64{0, ShardWidth * 3, ShardWidth * 5}, // exists bit, 2, 8 + toClear: []uint64{ShardWidth * 1, ShardWidth * 2, ShardWidth * 4}, // sign bit, 1, 4 }, { columnID: 0, bitDepth: 5, value: 10, - toSet: []uint64{ShardWidth, ShardWidth * 3, ShardWidth * 5}, - toClear: []uint64{0, ShardWidth * 2, ShardWidth * 4}, + toSet: []uint64{0, ShardWidth * 3, ShardWidth * 5}, // exists bit, 2, 8 + toClear: []uint64{ShardWidth * 1, ShardWidth * 2, ShardWidth * 4, ShardWidth * 6}, // sign bit, 1, 4, 16 }, } @@ -3138,7 +3141,7 @@ func TestImportClearRestart(t *testing.T) { check(t, f, exp) - f2 := newFragment(f.path, "i", "f", viewStandard, 0) + f2 := newFragment(f.path, "i", "f", viewStandard, 0, 0) f2.MaxOpN = maxOpN f2.CacheType = f.CacheType @@ -3172,7 +3175,7 @@ func TestImportClearRestart(t *testing.T) { check(t, f2, exp) - f3 := newFragment(f2.path, "i", "f", viewStandard, 0) + f3 := newFragment(f2.path, "i", "f", viewStandard, 0, 0) f3.MaxOpN = maxOpN f3.CacheType = f.CacheType @@ -3229,7 +3232,7 @@ func TestImportValueConcurrent(t *testing.T) { i := i eg.Go(func() error { for j := uint64(0); j < 10; j++ { - err := f.importValue([]uint64{j}, []uint64{uint64(rand.Int63n(1000))}, 10, i%2 == 0) + err := f.importValue([]uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) if err != nil { return err } @@ -3246,14 +3249,14 @@ func TestImportValueConcurrent(t *testing.T) { func TestImportMultipleValues(t *testing.T) { tests := []struct { cols []uint64 - vals []uint64 + vals []int64 checkCols []uint64 checkVals []uint64 depth uint }{ { cols: []uint64{0, 0}, - vals: []uint64{97, 100}, + vals: []int64{97, 100}, depth: 7, checkCols: []uint64{0}, checkVals: []uint64{100}, diff --git a/http/client.go b/http/client.go index 74706588d..6e3b2f274 100644 --- a/http/client.go +++ b/http/client.go @@ -799,8 +799,11 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel fieldOpt.CacheType = &opt.CacheType fieldOpt.CacheSize = &opt.CacheSize } else if fieldOpt.Type == "int" { - fieldOpt.Min = &opt.Min - fieldOpt.Max = &opt.Max + if opt.Base == 0 && opt.Min != 0 { + opt.Base = opt.Min + } + fieldOpt.Base = &opt.Base + fieldOpt.BitDepth = &opt.BitDepth } else if fieldOpt.Type == "time" { fieldOpt.TimeQuantum = &opt.TimeQuantum } diff --git a/http/handler.go b/http/handler.go index 78b5f48a9..2271d5ec4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -758,7 +758,13 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { case pilosa.FieldTypeSet: fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeInt: - fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) + var base int64 + if v := req.Options.Base; v != nil { + base = *v + } else if v := req.Options.Min; v != nil { + base = *v + } + fos = append(fos, pilosa.OptFieldTypeInt(base)) case pilosa.FieldTypeTime: fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView)) case pilosa.FieldTypeMutex: @@ -786,11 +792,15 @@ type fieldOptions struct { Type string `json:"type,omitempty"` CacheType *string `json:"cacheType,omitempty"` CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *int64 `json:"min,omitempty"` - Max *int64 `json:"max,omitempty"` + Base *int64 `json:"base,omitempty"` + BitDepth *uint `json:"bitDepth,omitempty"` TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` Keys *bool `json:"keys,omitempty"` NoStandardView bool `json:"noStandardView,omitempty"` + + // Deprecated. Use base/bit depth. + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` } func (o *fieldOptions) validate() error { @@ -812,7 +822,11 @@ func (o *fieldOptions) validate() error { if o.CacheSize == nil { o.CacheSize = &defaultCacheSize } - if o.Min != nil { + if o.Base != nil { + return pilosa.NewBadRequestError(errors.New("base does not apply to field type set")) + } else if o.BitDepth != nil { + return pilosa.NewBadRequestError(errors.New("bit depth does not apply to field type set")) + } else if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) } else if o.Max != nil { return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) @@ -824,10 +838,6 @@ func (o *fieldOptions) validate() error { return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) } else if o.CacheSize != nil { return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.Min == nil { - return pilosa.NewBadRequestError(errors.New("min is required for field type int")) - } else if o.Max == nil { - return pilosa.NewBadRequestError(errors.New("max is required for field type int")) } else if o.TimeQuantum != nil { return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) } @@ -836,6 +846,10 @@ func (o *fieldOptions) validate() error { return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) } else if o.CacheSize != nil { return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + } else if o.Base != nil { + return pilosa.NewBadRequestError(errors.New("base does not apply to field type time")) + } else if o.BitDepth != nil { + return pilosa.NewBadRequestError(errors.New("bitDepth does not apply to field type time")) } else if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) } else if o.Max != nil { @@ -850,7 +864,11 @@ func (o *fieldOptions) validate() error { if o.CacheSize == nil { o.CacheSize = &defaultCacheSize } - if o.Min != nil { + if o.Base != nil { + return pilosa.NewBadRequestError(errors.New("base does not apply to field type mutex")) + } else if o.BitDepth != nil { + return pilosa.NewBadRequestError(errors.New("bitDepth does not apply to field type mutex")) + } else if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) } else if o.Max != nil { return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) @@ -862,6 +880,10 @@ func (o *fieldOptions) validate() error { return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) } else if o.CacheSize != nil { return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Base != nil { + return pilosa.NewBadRequestError(errors.New("base does not apply to field type bool")) + } else if o.BitDepth != nil { + return pilosa.NewBadRequestError(errors.New("bitDepth does not apply to field type bool")) } else if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) } else if o.Max != nil { diff --git a/index_test.go b/index_test.go index 21f9a91d9..61f138de9 100644 --- a/index_test.go +++ b/index_test.go @@ -93,7 +93,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with schema and verify it exists. - if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil { + if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10)); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type: %#v", f.Type()) diff --git a/internal/private.pb.go b/internal/private.pb.go index c5a51741b..fc6c306e9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,48 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto +/* + Package internal is a generated protocol buffer package. + + It is generated from these files: + private.proto + + It has these top-level messages: + IndexMeta + FieldOptions + ImportResponse + BlockDataRequest + BlockDataResponse + Cache + MaxShards + CreateShardMessage + DeleteIndexMessage + CreateIndexMessage + CreateFieldMessage + DeleteFieldMessage + DeleteAvailableShardMessage + Field + Schema + Index + URI + Node + NodeStateMessage + NodeEventMessage + NodeStatus + IndexStatus + FieldStatus + ClusterStatus + BSIGroup + CreateViewMessage + DeleteViewMessage + ResizeInstruction + ResizeSource + ResizeInstructionComplete + SetCoordinatorMessage + UpdateCoordinatorMessage + Topology + RecalculateCaches +*/ package internal import proto "github.com/golang/protobuf/proto" @@ -21,45 +63,14 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` - TrackExistence bool `protobuf:"varint,4,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` + TrackExistence bool `protobuf:"varint,4,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` } -func (m *IndexMeta) Reset() { *m = IndexMeta{} } -func (m *IndexMeta) String() string { return proto.CompactTextString(m) } -func (*IndexMeta) ProtoMessage() {} -func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{0} -} -func (m *IndexMeta) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *IndexMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_IndexMeta.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *IndexMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexMeta.Merge(dst, src) -} -func (m *IndexMeta) XXX_Size() int { - return m.Size() -} -func (m *IndexMeta) XXX_DiscardUnknown() { - xxx_messageInfo_IndexMeta.DiscardUnknown(m) -} - -var xxx_messageInfo_IndexMeta proto.InternalMessageInfo +func (m *IndexMeta) Reset() { *m = IndexMeta{} } +func (m *IndexMeta) String() string { return proto.CompactTextString(m) } +func (*IndexMeta) ProtoMessage() {} +func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } func (m *IndexMeta) GetKeys() bool { if m != nil { @@ -76,51 +87,22 @@ func (m *IndexMeta) GetTrackExistence() bool { } type FieldOptions struct { - Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` - Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` - NoStandardView bool `protobuf:"varint,12,opt,name=NoStandardView,proto3" json:"NoStandardView,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` + NoStandardView bool `protobuf:"varint,12,opt,name=NoStandardView,proto3" json:"NoStandardView,omitempty"` + Base int64 `protobuf:"varint,13,opt,name=Base,proto3" json:"Base,omitempty"` + BitDepth uint64 `protobuf:"varint,14,opt,name=BitDepth,proto3" json:"BitDepth,omitempty"` + Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *FieldOptions) Reset() { *m = FieldOptions{} } -func (m *FieldOptions) String() string { return proto.CompactTextString(m) } -func (*FieldOptions) ProtoMessage() {} -func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{1} -} -func (m *FieldOptions) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FieldOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FieldOptions.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *FieldOptions) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldOptions.Merge(dst, src) -} -func (m *FieldOptions) XXX_Size() int { - return m.Size() -} -func (m *FieldOptions) XXX_DiscardUnknown() { - xxx_messageInfo_FieldOptions.DiscardUnknown(m) -} - -var xxx_messageInfo_FieldOptions proto.InternalMessageInfo +func (m *FieldOptions) Reset() { *m = FieldOptions{} } +func (m *FieldOptions) String() string { return proto.CompactTextString(m) } +func (*FieldOptions) ProtoMessage() {} +func (*FieldOptions) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } func (m *FieldOptions) GetType() string { if m != nil { @@ -143,20 +125,6 @@ func (m *FieldOptions) GetCacheSize() uint32 { return 0 } -func (m *FieldOptions) GetMin() int64 { - if m != nil { - return m.Min - } - return 0 -} - -func (m *FieldOptions) GetMax() int64 { - if m != nil { - return m.Max - } - return 0 -} - func (m *FieldOptions) GetTimeQuantum() string { if m != nil { return m.TimeQuantum @@ -178,45 +146,42 @@ func (m *FieldOptions) GetNoStandardView() bool { return false } -type ImportResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ImportResponse) Reset() { *m = ImportResponse{} } -func (m *ImportResponse) String() string { return proto.CompactTextString(m) } -func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{2} -} -func (m *ImportResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil +func (m *FieldOptions) GetBase() int64 { + if m != nil { + return m.Base } -} -func (dst *ImportResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportResponse.Merge(dst, src) -} -func (m *ImportResponse) XXX_Size() int { - return m.Size() -} -func (m *ImportResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ImportResponse.DiscardUnknown(m) + return 0 } -var xxx_messageInfo_ImportResponse proto.InternalMessageInfo +func (m *FieldOptions) GetBitDepth() uint64 { + if m != nil { + return m.BitDepth + } + return 0 +} + +func (m *FieldOptions) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *FieldOptions) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + +type ImportResponse struct { + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` +} + +func (m *ImportResponse) Reset() { *m = ImportResponse{} } +func (m *ImportResponse) String() string { return proto.CompactTextString(m) } +func (*ImportResponse) ProtoMessage() {} +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{2} } func (m *ImportResponse) GetErr() string { if m != nil { @@ -226,48 +191,17 @@ func (m *ImportResponse) GetErr() string { } type BlockDataRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` - Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` - Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` + Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } -func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } -func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } -func (*BlockDataRequest) ProtoMessage() {} -func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{3} -} -func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BlockDataRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BlockDataRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *BlockDataRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataRequest.Merge(dst, src) -} -func (m *BlockDataRequest) XXX_Size() int { - return m.Size() -} -func (m *BlockDataRequest) XXX_DiscardUnknown() { - xxx_messageInfo_BlockDataRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_BlockDataRequest proto.InternalMessageInfo +func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } +func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } +func (*BlockDataRequest) ProtoMessage() {} +func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} } func (m *BlockDataRequest) GetIndex() string { if m != nil { @@ -305,45 +239,14 @@ func (m *BlockDataRequest) GetBlock() uint64 { } type BlockDataResponse struct { - RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` } -func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } -func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } -func (*BlockDataResponse) ProtoMessage() {} -func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{4} -} -func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BlockDataResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BlockDataResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *BlockDataResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataResponse.Merge(dst, src) -} -func (m *BlockDataResponse) XXX_Size() int { - return m.Size() -} -func (m *BlockDataResponse) XXX_DiscardUnknown() { - xxx_messageInfo_BlockDataResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_BlockDataResponse proto.InternalMessageInfo +func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } +func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } +func (*BlockDataResponse) ProtoMessage() {} +func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} } func (m *BlockDataResponse) GetRowIDs() []uint64 { if m != nil { @@ -360,44 +263,13 @@ func (m *BlockDataResponse) GetColumnIDs() []uint64 { } type Cache struct { - IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` } -func (m *Cache) Reset() { *m = Cache{} } -func (m *Cache) String() string { return proto.CompactTextString(m) } -func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{5} -} -func (m *Cache) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Cache) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Cache.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Cache) XXX_Merge(src proto.Message) { - xxx_messageInfo_Cache.Merge(dst, src) -} -func (m *Cache) XXX_Size() int { - return m.Size() -} -func (m *Cache) XXX_DiscardUnknown() { - xxx_messageInfo_Cache.DiscardUnknown(m) -} - -var xxx_messageInfo_Cache proto.InternalMessageInfo +func (m *Cache) Reset() { *m = Cache{} } +func (m *Cache) String() string { return proto.CompactTextString(m) } +func (*Cache) ProtoMessage() {} +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{5} } func (m *Cache) GetIDs() []uint64 { if m != nil { @@ -407,44 +279,13 @@ func (m *Cache) GetIDs() []uint64 { } type MaxShards struct { - Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxShards) Reset() { *m = MaxShards{} } -func (m *MaxShards) String() string { return proto.CompactTextString(m) } -func (*MaxShards) ProtoMessage() {} -func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{6} -} -func (m *MaxShards) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *MaxShards) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_MaxShards.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *MaxShards) XXX_Merge(src proto.Message) { - xxx_messageInfo_MaxShards.Merge(dst, src) -} -func (m *MaxShards) XXX_Size() int { - return m.Size() -} -func (m *MaxShards) XXX_DiscardUnknown() { - xxx_messageInfo_MaxShards.DiscardUnknown(m) -} - -var xxx_messageInfo_MaxShards proto.InternalMessageInfo +func (m *MaxShards) Reset() { *m = MaxShards{} } +func (m *MaxShards) String() string { return proto.CompactTextString(m) } +func (*MaxShards) ProtoMessage() {} +func (*MaxShards) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } func (m *MaxShards) GetStandard() map[string]uint64 { if m != nil { @@ -454,46 +295,15 @@ func (m *MaxShards) GetStandard() map[string]uint64 { } type CreateShardMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` - Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } -func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } -func (*CreateShardMessage) ProtoMessage() {} -func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{7} -} -func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateShardMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateShardMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateShardMessage.Merge(dst, src) -} -func (m *CreateShardMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateShardMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateShardMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateShardMessage proto.InternalMessageInfo +func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } +func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } +func (*CreateShardMessage) ProtoMessage() {} +func (*CreateShardMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } func (m *CreateShardMessage) GetIndex() string { if m != nil { @@ -517,44 +327,13 @@ func (m *CreateShardMessage) GetShard() uint64 { } type DeleteIndexMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } -func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } -func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteIndexMessage) ProtoMessage() {} -func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{8} -} -func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteIndexMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteIndexMessage.Merge(dst, src) -} -func (m *DeleteIndexMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteIndexMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteIndexMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteIndexMessage proto.InternalMessageInfo +func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } +func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteIndexMessage) ProtoMessage() {} +func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } func (m *DeleteIndexMessage) GetIndex() string { if m != nil { @@ -564,45 +343,14 @@ func (m *DeleteIndexMessage) GetIndex() string { } type CreateIndexMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` } -func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } -func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } -func (*CreateIndexMessage) ProtoMessage() {} -func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{9} -} -func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateIndexMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateIndexMessage.Merge(dst, src) -} -func (m *CreateIndexMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateIndexMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateIndexMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateIndexMessage proto.InternalMessageInfo +func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } +func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } +func (*CreateIndexMessage) ProtoMessage() {} +func (*CreateIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } func (m *CreateIndexMessage) GetIndex() string { if m != nil { @@ -619,46 +367,15 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { } type CreateFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } -func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } -func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } -func (*CreateFieldMessage) ProtoMessage() {} -func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{10} -} -func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateFieldMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateFieldMessage.Merge(dst, src) -} -func (m *CreateFieldMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateFieldMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateFieldMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateFieldMessage proto.InternalMessageInfo +func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } +func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFieldMessage) ProtoMessage() {} +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } func (m *CreateFieldMessage) GetIndex() string { if m != nil { @@ -682,45 +399,14 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { } type DeleteFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` } -func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } -func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteFieldMessage) ProtoMessage() {} -func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{11} -} -func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteFieldMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteFieldMessage.Merge(dst, src) -} -func (m *DeleteFieldMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteFieldMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteFieldMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteFieldMessage proto.InternalMessageInfo +func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } +func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFieldMessage) ProtoMessage() {} +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } func (m *DeleteFieldMessage) GetIndex() string { if m != nil { @@ -737,46 +423,17 @@ func (m *DeleteFieldMessage) GetField() string { } type DeleteAvailableShardMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - ShardID uint64 `protobuf:"varint,3,opt,name=ShardID,proto3" json:"ShardID,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + ShardID uint64 `protobuf:"varint,3,opt,name=ShardID,proto3" json:"ShardID,omitempty"` } func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShardMessage{} } func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{12} + return fileDescriptorPrivate, []int{12} } -func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteAvailableShardMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteAvailableShardMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteAvailableShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteAvailableShardMessage.Merge(dst, src) -} -func (m *DeleteAvailableShardMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteAvailableShardMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteAvailableShardMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteAvailableShardMessage proto.InternalMessageInfo func (m *DeleteAvailableShardMessage) GetIndex() string { if m != nil { @@ -800,46 +457,15 @@ func (m *DeleteAvailableShardMessage) GetShardID() uint64 { } type Field struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } -func (m *Field) Reset() { *m = Field{} } -func (m *Field) String() string { return proto.CompactTextString(m) } -func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{13} -} -func (m *Field) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Field.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(dst, src) -} -func (m *Field) XXX_Size() int { - return m.Size() -} -func (m *Field) XXX_DiscardUnknown() { - xxx_messageInfo_Field.DiscardUnknown(m) -} - -var xxx_messageInfo_Field proto.InternalMessageInfo +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } func (m *Field) GetName() string { if m != nil { @@ -863,44 +489,13 @@ func (m *Field) GetViews() []string { } type Schema struct { - Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` } -func (m *Schema) Reset() { *m = Schema{} } -func (m *Schema) String() string { return proto.CompactTextString(m) } -func (*Schema) ProtoMessage() {} -func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{14} -} -func (m *Schema) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Schema.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(dst, src) -} -func (m *Schema) XXX_Size() int { - return m.Size() -} -func (m *Schema) XXX_DiscardUnknown() { - xxx_messageInfo_Schema.DiscardUnknown(m) -} - -var xxx_messageInfo_Schema proto.InternalMessageInfo +func (m *Schema) Reset() { *m = Schema{} } +func (m *Schema) String() string { return proto.CompactTextString(m) } +func (*Schema) ProtoMessage() {} +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Schema) GetIndexes() []*Index { if m != nil { @@ -910,45 +505,14 @@ func (m *Schema) GetIndexes() []*Index { } type Index struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` } -func (m *Index) Reset() { *m = Index{} } -func (m *Index) String() string { return proto.CompactTextString(m) } -func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{15} -} -func (m *Index) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Index.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Index) XXX_Merge(src proto.Message) { - xxx_messageInfo_Index.Merge(dst, src) -} -func (m *Index) XXX_Size() int { - return m.Size() -} -func (m *Index) XXX_DiscardUnknown() { - xxx_messageInfo_Index.DiscardUnknown(m) -} - -var xxx_messageInfo_Index proto.InternalMessageInfo +func (m *Index) Reset() { *m = Index{} } +func (m *Index) String() string { return proto.CompactTextString(m) } +func (*Index) ProtoMessage() {} +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *Index) GetName() string { if m != nil { @@ -965,46 +529,15 @@ func (m *Index) GetFields() []*Field { } type URI struct { - Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` - Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` - Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` + Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` + Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *URI) Reset() { *m = URI{} } -func (m *URI) String() string { return proto.CompactTextString(m) } -func (*URI) ProtoMessage() {} -func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{16} -} -func (m *URI) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *URI) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_URI.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *URI) XXX_Merge(src proto.Message) { - xxx_messageInfo_URI.Merge(dst, src) -} -func (m *URI) XXX_Size() int { - return m.Size() -} -func (m *URI) XXX_DiscardUnknown() { - xxx_messageInfo_URI.DiscardUnknown(m) -} - -var xxx_messageInfo_URI proto.InternalMessageInfo +func (m *URI) Reset() { *m = URI{} } +func (m *URI) String() string { return proto.CompactTextString(m) } +func (*URI) ProtoMessage() {} +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *URI) GetScheme() string { if m != nil { @@ -1028,47 +561,16 @@ func (m *URI) GetPort() uint32 { } type Node struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` - IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` - State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` + State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` } -func (m *Node) Reset() { *m = Node{} } -func (m *Node) String() string { return proto.CompactTextString(m) } -func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{17} -} -func (m *Node) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Node) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Node.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Node) XXX_Merge(src proto.Message) { - xxx_messageInfo_Node.Merge(dst, src) -} -func (m *Node) XXX_Size() int { - return m.Size() -} -func (m *Node) XXX_DiscardUnknown() { - xxx_messageInfo_Node.DiscardUnknown(m) -} - -var xxx_messageInfo_Node proto.InternalMessageInfo +func (m *Node) Reset() { *m = Node{} } +func (m *Node) String() string { return proto.CompactTextString(m) } +func (*Node) ProtoMessage() {} +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *Node) GetID() string { if m != nil { @@ -1099,45 +601,14 @@ func (m *Node) GetState() string { } type NodeStateMessage struct { - NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } -func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } -func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } -func (*NodeStateMessage) ProtoMessage() {} -func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{18} -} -func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NodeStateMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NodeStateMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *NodeStateMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStateMessage.Merge(dst, src) -} -func (m *NodeStateMessage) XXX_Size() int { - return m.Size() -} -func (m *NodeStateMessage) XXX_DiscardUnknown() { - xxx_messageInfo_NodeStateMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeStateMessage proto.InternalMessageInfo +func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } +func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } +func (*NodeStateMessage) ProtoMessage() {} +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *NodeStateMessage) GetNodeID() string { if m != nil { @@ -1154,45 +625,14 @@ func (m *NodeStateMessage) GetState() string { } type NodeEventMessage struct { - Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` } -func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } -func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } -func (*NodeEventMessage) ProtoMessage() {} -func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{19} -} -func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NodeEventMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NodeEventMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *NodeEventMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeEventMessage.Merge(dst, src) -} -func (m *NodeEventMessage) XXX_Size() int { - return m.Size() -} -func (m *NodeEventMessage) XXX_DiscardUnknown() { - xxx_messageInfo_NodeEventMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeEventMessage proto.InternalMessageInfo +func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } +func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } +func (*NodeEventMessage) ProtoMessage() {} +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeEventMessage) GetEvent() uint32 { if m != nil { @@ -1209,46 +649,15 @@ func (m *NodeEventMessage) GetNode() *Node { } type NodeStatus struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` - Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes" json:"Indexes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` + Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes" json:"Indexes,omitempty"` } -func (m *NodeStatus) Reset() { *m = NodeStatus{} } -func (m *NodeStatus) String() string { return proto.CompactTextString(m) } -func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{20} -} -func (m *NodeStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NodeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NodeStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *NodeStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStatus.Merge(dst, src) -} -func (m *NodeStatus) XXX_Size() int { - return m.Size() -} -func (m *NodeStatus) XXX_DiscardUnknown() { - xxx_messageInfo_NodeStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeStatus proto.InternalMessageInfo +func (m *NodeStatus) Reset() { *m = NodeStatus{} } +func (m *NodeStatus) String() string { return proto.CompactTextString(m) } +func (*NodeStatus) ProtoMessage() {} +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *NodeStatus) GetNode() *Node { if m != nil { @@ -1272,45 +681,14 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { } type IndexStatus struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields" json:"Fields,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields" json:"Fields,omitempty"` } -func (m *IndexStatus) Reset() { *m = IndexStatus{} } -func (m *IndexStatus) String() string { return proto.CompactTextString(m) } -func (*IndexStatus) ProtoMessage() {} -func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{21} -} -func (m *IndexStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *IndexStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_IndexStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *IndexStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexStatus.Merge(dst, src) -} -func (m *IndexStatus) XXX_Size() int { - return m.Size() -} -func (m *IndexStatus) XXX_DiscardUnknown() { - xxx_messageInfo_IndexStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_IndexStatus proto.InternalMessageInfo +func (m *IndexStatus) Reset() { *m = IndexStatus{} } +func (m *IndexStatus) String() string { return proto.CompactTextString(m) } +func (*IndexStatus) ProtoMessage() {} +func (*IndexStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } func (m *IndexStatus) GetName() string { if m != nil { @@ -1327,45 +705,14 @@ func (m *IndexStatus) GetFields() []*FieldStatus { } type FieldStatus struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards" json:"AvailableShards,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards" json:"AvailableShards,omitempty"` } -func (m *FieldStatus) Reset() { *m = FieldStatus{} } -func (m *FieldStatus) String() string { return proto.CompactTextString(m) } -func (*FieldStatus) ProtoMessage() {} -func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{22} -} -func (m *FieldStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FieldStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FieldStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *FieldStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldStatus.Merge(dst, src) -} -func (m *FieldStatus) XXX_Size() int { - return m.Size() -} -func (m *FieldStatus) XXX_DiscardUnknown() { - xxx_messageInfo_FieldStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_FieldStatus proto.InternalMessageInfo +func (m *FieldStatus) Reset() { *m = FieldStatus{} } +func (m *FieldStatus) String() string { return proto.CompactTextString(m) } +func (*FieldStatus) ProtoMessage() {} +func (*FieldStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *FieldStatus) GetName() string { if m != nil { @@ -1382,46 +729,15 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { } type ClusterStatus struct { - ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` } -func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } -func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } -func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{23} -} -func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ClusterStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ClusterStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ClusterStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClusterStatus.Merge(dst, src) -} -func (m *ClusterStatus) XXX_Size() int { - return m.Size() -} -func (m *ClusterStatus) XXX_DiscardUnknown() { - xxx_messageInfo_ClusterStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_ClusterStatus proto.InternalMessageInfo +func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } +func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } +func (*ClusterStatus) ProtoMessage() {} +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *ClusterStatus) GetClusterID() string { if m != nil { @@ -1445,47 +761,16 @@ func (m *ClusterStatus) GetNodes() []*Node { } type BSIGroup struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` - Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` - Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` + Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *BSIGroup) Reset() { *m = BSIGroup{} } -func (m *BSIGroup) String() string { return proto.CompactTextString(m) } -func (*BSIGroup) ProtoMessage() {} -func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{24} -} -func (m *BSIGroup) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BSIGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BSIGroup.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *BSIGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_BSIGroup.Merge(dst, src) -} -func (m *BSIGroup) XXX_Size() int { - return m.Size() -} -func (m *BSIGroup) XXX_DiscardUnknown() { - xxx_messageInfo_BSIGroup.DiscardUnknown(m) -} - -var xxx_messageInfo_BSIGroup proto.InternalMessageInfo +func (m *BSIGroup) Reset() { *m = BSIGroup{} } +func (m *BSIGroup) String() string { return proto.CompactTextString(m) } +func (*BSIGroup) ProtoMessage() {} +func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *BSIGroup) GetName() string { if m != nil { @@ -1516,46 +801,15 @@ func (m *BSIGroup) GetMax() int64 { } type CreateViewMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } -func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } -func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } -func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{25} -} -func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateViewMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateViewMessage.Merge(dst, src) -} -func (m *CreateViewMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateViewMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateViewMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateViewMessage proto.InternalMessageInfo +func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } +func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } +func (*CreateViewMessage) ProtoMessage() {} +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -1579,46 +833,15 @@ func (m *CreateViewMessage) GetView() string { } type DeleteViewMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } -func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } -func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{26} -} -func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteViewMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteViewMessage.Merge(dst, src) -} -func (m *DeleteViewMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteViewMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteViewMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteViewMessage proto.InternalMessageInfo +func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } +func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteViewMessage) ProtoMessage() {} +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -1642,49 +865,18 @@ func (m *DeleteViewMessage) GetView() string { } type ResizeInstruction struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` - Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` - NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` - ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` + NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` + ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` } -func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } -func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } -func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{27} -} -func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ResizeInstruction) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ResizeInstruction.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ResizeInstruction) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstruction.Merge(dst, src) -} -func (m *ResizeInstruction) XXX_Size() int { - return m.Size() -} -func (m *ResizeInstruction) XXX_DiscardUnknown() { - xxx_messageInfo_ResizeInstruction.DiscardUnknown(m) -} - -var xxx_messageInfo_ResizeInstruction proto.InternalMessageInfo +func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } +func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } +func (*ResizeInstruction) ProtoMessage() {} +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -1729,48 +921,17 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { } type ResizeSource struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` - Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *ResizeSource) Reset() { *m = ResizeSource{} } -func (m *ResizeSource) String() string { return proto.CompactTextString(m) } -func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{28} -} -func (m *ResizeSource) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ResizeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ResizeSource.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ResizeSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeSource.Merge(dst, src) -} -func (m *ResizeSource) XXX_Size() int { - return m.Size() -} -func (m *ResizeSource) XXX_DiscardUnknown() { - xxx_messageInfo_ResizeSource.DiscardUnknown(m) -} - -var xxx_messageInfo_ResizeSource proto.InternalMessageInfo +func (m *ResizeSource) Reset() { *m = ResizeSource{} } +func (m *ResizeSource) String() string { return proto.CompactTextString(m) } +func (*ResizeSource) ProtoMessage() {} +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *ResizeSource) GetNode() *Node { if m != nil { @@ -1808,46 +969,17 @@ func (m *ResizeSource) GetShard() uint64 { } type ResizeInstructionComplete struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` } func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComplete{} } func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{29} + return fileDescriptorPrivate, []int{29} } -func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ResizeInstructionComplete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ResizeInstructionComplete.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ResizeInstructionComplete) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstructionComplete.Merge(dst, src) -} -func (m *ResizeInstructionComplete) XXX_Size() int { - return m.Size() -} -func (m *ResizeInstructionComplete) XXX_DiscardUnknown() { - xxx_messageInfo_ResizeInstructionComplete.DiscardUnknown(m) -} - -var xxx_messageInfo_ResizeInstructionComplete proto.InternalMessageInfo func (m *ResizeInstructionComplete) GetJobID() int64 { if m != nil { @@ -1871,44 +1003,13 @@ func (m *ResizeInstructionComplete) GetError() string { } type SetCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` } -func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } -func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } -func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{30} -} -func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SetCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SetCoordinatorMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *SetCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_SetCoordinatorMessage.Merge(dst, src) -} -func (m *SetCoordinatorMessage) XXX_Size() int { - return m.Size() -} -func (m *SetCoordinatorMessage) XXX_DiscardUnknown() { - xxx_messageInfo_SetCoordinatorMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_SetCoordinatorMessage proto.InternalMessageInfo +func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } +func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*SetCoordinatorMessage) ProtoMessage() {} +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { @@ -1918,44 +1019,13 @@ func (m *SetCoordinatorMessage) GetNew() *Node { } type UpdateCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` } -func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } -func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } -func (*UpdateCoordinatorMessage) ProtoMessage() {} -func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{31} -} -func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *UpdateCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_UpdateCoordinatorMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *UpdateCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_UpdateCoordinatorMessage.Merge(dst, src) -} -func (m *UpdateCoordinatorMessage) XXX_Size() int { - return m.Size() -} -func (m *UpdateCoordinatorMessage) XXX_DiscardUnknown() { - xxx_messageInfo_UpdateCoordinatorMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_UpdateCoordinatorMessage proto.InternalMessageInfo +func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } +func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*UpdateCoordinatorMessage) ProtoMessage() {} +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } func (m *UpdateCoordinatorMessage) GetNew() *Node { if m != nil { @@ -1965,45 +1035,14 @@ func (m *UpdateCoordinatorMessage) GetNew() *Node { } type Topology struct { - ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` - NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` } -func (m *Topology) Reset() { *m = Topology{} } -func (m *Topology) String() string { return proto.CompactTextString(m) } -func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{32} -} -func (m *Topology) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Topology) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Topology.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Topology) XXX_Merge(src proto.Message) { - xxx_messageInfo_Topology.Merge(dst, src) -} -func (m *Topology) XXX_Size() int { - return m.Size() -} -func (m *Topology) XXX_DiscardUnknown() { - xxx_messageInfo_Topology.DiscardUnknown(m) -} - -var xxx_messageInfo_Topology proto.InternalMessageInfo +func (m *Topology) Reset() { *m = Topology{} } +func (m *Topology) String() string { return proto.CompactTextString(m) } +func (*Topology) ProtoMessage() {} +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } func (m *Topology) GetClusterID() string { if m != nil { @@ -2020,43 +1059,12 @@ func (m *Topology) GetNodeIDs() []string { } type RecalculateCaches struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } -func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } -func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{33} -} -func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RecalculateCaches) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RecalculateCaches.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *RecalculateCaches) XXX_Merge(src proto.Message) { - xxx_messageInfo_RecalculateCaches.Merge(dst, src) -} -func (m *RecalculateCaches) XXX_Size() int { - return m.Size() -} -func (m *RecalculateCaches) XXX_DiscardUnknown() { - xxx_messageInfo_RecalculateCaches.DiscardUnknown(m) -} - -var xxx_messageInfo_RecalculateCaches proto.InternalMessageInfo +func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } +func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } +func (*RecalculateCaches) ProtoMessage() {} +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{33} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") @@ -2066,7 +1074,6 @@ func init() { proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*MaxShards)(nil), "internal.MaxShards") - proto.RegisterMapType((map[string]uint64)(nil), "internal.MaxShards.StandardEntry") proto.RegisterType((*CreateShardMessage)(nil), "internal.CreateShardMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") @@ -2130,9 +1137,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2204,8 +1208,15 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Base != 0 { + dAtA[i] = 0x68 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Base)) + } + if m.BitDepth != 0 { + dAtA[i] = 0x70 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.BitDepth)) } return i, nil } @@ -2231,9 +1242,6 @@ func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Err))) i += copy(dAtA[i:], m.Err) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2280,9 +1288,6 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2335,9 +1340,6 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j3)) i += copy(dAtA[i:], dAtA4[:j3]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2373,9 +1375,6 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j5)) i += copy(dAtA[i:], dAtA6[:j5]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2410,9 +1409,6 @@ func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(v)) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2448,9 +1444,6 @@ func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2475,9 +1468,6 @@ func (m *DeleteIndexMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2512,9 +1502,6 @@ func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { } i += n7 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2555,9 +1542,6 @@ func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { } i += n8 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2588,9 +1572,6 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2626,9 +1607,6 @@ func (m *DeleteAvailableShardMessage) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2678,9 +1656,6 @@ func (m *Field) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2711,9 +1686,6 @@ func (m *Schema) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2750,9 +1722,6 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2788,9 +1757,6 @@ func (m *URI) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2841,9 +1807,6 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2874,9 +1837,6 @@ func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2910,9 +1870,6 @@ func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { } i += n11 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2963,9 +1920,6 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3002,9 +1956,6 @@ func (m *IndexStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3046,9 +1997,6 @@ func (m *FieldStatus) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j14)) i += copy(dAtA[i:], dAtA15[:j14]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3091,9 +2039,6 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3134,9 +2079,6 @@ func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3173,9 +2115,6 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3212,9 +2151,6 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3290,9 +2226,6 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { } i += n19 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3344,9 +2277,6 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3386,9 +2316,6 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) i += copy(dAtA[i:], m.Error) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3417,9 +2344,6 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { } i += n22 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3448,9 +2372,6 @@ func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { } i += n23 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3490,9 +2411,6 @@ func (m *Topology) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3511,9 +2429,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3527,9 +2442,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { return offset + 1 } func (m *IndexMeta) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Keys { @@ -3538,16 +2450,10 @@ func (m *IndexMeta) Size() (n int) { if m.TrackExistence { n += 2 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *FieldOptions) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.CacheType) @@ -3577,32 +2483,26 @@ func (m *FieldOptions) Size() (n int) { if m.NoStandardView { n += 2 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) + if m.Base != 0 { + n += 1 + sovPrivate(uint64(m.Base)) + } + if m.BitDepth != 0 { + n += 1 + sovPrivate(uint64(m.BitDepth)) } return n } func (m *ImportResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Err) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *BlockDataRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3623,16 +2523,10 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *BlockDataResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.RowIDs) > 0 { @@ -3649,16 +2543,10 @@ func (m *BlockDataResponse) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Cache) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.IDs) > 0 { @@ -3668,16 +2556,10 @@ func (m *Cache) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *MaxShards) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Standard) > 0 { @@ -3688,16 +2570,10 @@ func (m *MaxShards) Size() (n int) { n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateShardMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3711,32 +2587,20 @@ func (m *CreateShardMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteIndexMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateIndexMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3747,16 +2611,10 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateFieldMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3771,16 +2629,10 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteFieldMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3791,16 +2643,10 @@ func (m *DeleteFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteAvailableShardMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3814,16 +2660,10 @@ func (m *DeleteAvailableShardMessage) Size() (n int) { if m.ShardID != 0 { n += 1 + sovPrivate(uint64(m.ShardID)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Field) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -3840,16 +2680,10 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Schema) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Indexes) > 0 { @@ -3858,16 +2692,10 @@ func (m *Schema) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Index) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -3880,16 +2708,10 @@ func (m *Index) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *URI) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Scheme) @@ -3903,16 +2725,10 @@ func (m *URI) Size() (n int) { if m.Port != 0 { n += 1 + sovPrivate(uint64(m.Port)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Node) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.ID) @@ -3930,16 +2746,10 @@ func (m *Node) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *NodeStateMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.NodeID) @@ -3950,16 +2760,10 @@ func (m *NodeStateMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *NodeEventMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Event != 0 { @@ -3969,16 +2773,10 @@ func (m *NodeEventMessage) Size() (n int) { l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *NodeStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Node != nil { @@ -3995,16 +2793,10 @@ func (m *NodeStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *IndexStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -4017,16 +2809,10 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *FieldStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -4040,16 +2826,10 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ClusterStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.ClusterID) @@ -4066,16 +2846,10 @@ func (m *ClusterStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *BSIGroup) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -4092,16 +2866,10 @@ func (m *BSIGroup) Size() (n int) { if m.Max != 0 { n += 1 + sovPrivate(uint64(m.Max)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateViewMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -4116,16 +2884,10 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteViewMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -4140,16 +2902,10 @@ func (m *DeleteViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ResizeInstruction) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.JobID != 0 { @@ -4177,16 +2933,10 @@ func (m *ResizeInstruction) Size() (n int) { l = m.NodeStatus.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ResizeSource) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Node != nil { @@ -4208,16 +2958,10 @@ func (m *ResizeSource) Size() (n int) { if m.Shard != 0 { n += 1 + sovPrivate(uint64(m.Shard)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ResizeInstructionComplete) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.JobID != 0 { @@ -4231,48 +2975,30 @@ func (m *ResizeInstructionComplete) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *SetCoordinatorMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.New != nil { l = m.New.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *UpdateCoordinatorMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.New != nil { l = m.New.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Topology) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.ClusterID) @@ -4285,21 +3011,12 @@ func (m *Topology) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *RecalculateCaches) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -4397,7 +3114,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4620,6 +3336,44 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } } m.NoStandardView = bool(v != 0) + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Base", wireType) + } + m.Base = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Base |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 14: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BitDepth", wireType) + } + m.BitDepth = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BitDepth |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -4632,7 +3386,6 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4712,7 +3465,6 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4888,7 +3640,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4968,17 +3719,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.RowIDs) == 0 { - m.RowIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5041,17 +3781,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.ColumnIDs) == 0 { - m.ColumnIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5085,7 +3814,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5165,17 +3893,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.IDs) == 0 { - m.IDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5209,7 +3926,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5367,7 +4083,6 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5495,7 +4210,6 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5575,7 +4289,6 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5688,7 +4401,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5830,7 +4542,6 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5939,7 +4650,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6067,7 +4777,6 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6209,7 +4918,6 @@ func (m *Field) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6291,7 +4999,6 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6402,7 +5109,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6530,7 +5236,6 @@ func (m *URI) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6692,7 +5397,6 @@ func (m *Node) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6801,7 +5505,6 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6904,7 +5607,6 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7052,7 +5754,6 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7163,7 +5864,6 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7272,17 +5972,6 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.AvailableShards) == 0 { - m.AvailableShards = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -7316,7 +6005,6 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7456,7 +6144,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7603,7 +6290,6 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7741,7 +6427,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7879,7 +6564,6 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8112,7 +6796,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8302,7 +6985,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8434,7 +7116,6 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8518,7 +7199,6 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8602,7 +7282,6 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8711,7 +7390,6 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8762,7 +7440,6 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8877,80 +7554,82 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_8095a89af06a70de) } +func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } -var fileDescriptor_private_8095a89af06a70de = []byte{ - // 1139 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0xc5, - 0x1b, 0xff, 0xef, 0x21, 0x8e, 0xfd, 0x39, 0x4e, 0x93, 0x6d, 0x9b, 0xff, 0x16, 0x50, 0x08, 0xa3, - 0x8a, 0x86, 0x4a, 0x84, 0xaa, 0xe5, 0x82, 0x53, 0xa5, 0x92, 0x38, 0x94, 0xa5, 0x24, 0x94, 0x71, - 0x92, 0x3b, 0x2e, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0x63, 0x76, 0x67, 0x93, 0xb8, 0x17, 0xdc, - 0x82, 0xc4, 0x0b, 0xf0, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x42, 0x15, 0x5e, 0x04, 0xcd, 0x37, 0x33, - 0xbb, 0x6b, 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xbe, 0xdf, 0x77, 0x3e, 0xae, 0x0d, 0x9d, 0x51, 0x16, - 0x9f, 0x30, 0xc9, 0x37, 0x46, 0x99, 0x90, 0x22, 0x68, 0xc6, 0xa9, 0xe4, 0x59, 0xca, 0x12, 0xf2, - 0x1c, 0x5a, 0x51, 0x3a, 0xe0, 0x67, 0x3b, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x05, 0x1f, 0xe7, 0xa1, - 0xb7, 0xe6, 0xac, 0x37, 0x29, 0xbe, 0x83, 0xf7, 0x61, 0x71, 0x2f, 0x63, 0xfd, 0xe3, 0xed, 0xb3, - 0x38, 0x97, 0x3c, 0xed, 0xf3, 0xd0, 0x47, 0xee, 0x14, 0x4a, 0xde, 0x38, 0xb0, 0xf0, 0x55, 0xcc, - 0x93, 0xc1, 0x77, 0x23, 0x19, 0x8b, 0x34, 0x0f, 0xde, 0x81, 0xd6, 0x16, 0xeb, 0x1f, 0xf1, 0xbd, - 0xf1, 0x88, 0xa3, 0xc5, 0x16, 0xad, 0x80, 0x92, 0xdb, 0x8b, 0x5f, 0x6b, 0x8b, 0x1d, 0x5a, 0x01, - 0xc1, 0x1a, 0xb4, 0xf7, 0xe2, 0x21, 0xff, 0xbe, 0x60, 0xa9, 0x2c, 0x86, 0xe1, 0x1c, 0x6a, 0xd7, - 0x21, 0x15, 0x2a, 0x1a, 0x6e, 0x22, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0x3b, 0x71, 0x1a, 0xb6, 0xd6, - 0x9c, 0x75, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x2c, 0x04, 0x83, 0xb0, 0xb3, 0x32, 0xc5, 0xf6, 0x64, - 0x8a, 0xbb, 0xa2, 0x27, 0x59, 0x3a, 0x60, 0xd9, 0xe0, 0x20, 0xe6, 0xa7, 0xe1, 0x82, 0x4e, 0x71, - 0x12, 0x25, 0x04, 0x16, 0xa3, 0xe1, 0x48, 0x64, 0x92, 0xf2, 0x7c, 0x24, 0xd2, 0x1c, 0x3d, 0x6e, - 0x67, 0x59, 0xe8, 0x60, 0x10, 0xea, 0x49, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0xf4, 0x8f, 0xbb, 0x4c, - 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0x3b, 0x30, 0x87, 0x35, 0x36, 0x72, 0x9a, 0x50, 0x28, - 0xd6, 0x2b, 0x74, 0x35, 0x8a, 0x84, 0x42, 0x51, 0x1f, 0x2b, 0xe6, 0x53, 0x4d, 0x28, 0xb4, 0x77, - 0xc4, 0xb2, 0x01, 0x56, 0xca, 0xa7, 0x9a, 0x50, 0xb9, 0x60, 0xb4, 0xba, 0x3c, 0xf8, 0x26, 0x11, - 0x2c, 0xd7, 0xfc, 0x9b, 0x30, 0x57, 0xa0, 0x41, 0xc5, 0x69, 0xd4, 0xcd, 0x43, 0x67, 0xcd, 0x5b, - 0xf7, 0xa9, 0xa1, 0xb0, 0x09, 0x22, 0x29, 0x86, 0xa9, 0x62, 0xb9, 0xc8, 0xaa, 0x00, 0x72, 0x0f, - 0xe6, 0xb0, 0x23, 0x2a, 0xcb, 0x4a, 0x57, 0x3d, 0xc9, 0xcf, 0x0e, 0xb4, 0x76, 0xd8, 0x19, 0x86, - 0x91, 0x07, 0x4f, 0xa1, 0x69, 0xeb, 0x84, 0x42, 0xed, 0xc7, 0xef, 0x6d, 0xd8, 0x01, 0xdb, 0x28, - 0xc5, 0x36, 0xac, 0xcc, 0x76, 0x2a, 0xb3, 0x31, 0x2d, 0x55, 0xde, 0xfa, 0x1c, 0x3a, 0x13, 0x2c, - 0xe5, 0xef, 0x98, 0x8f, 0x6d, 0x55, 0x8f, 0xf9, 0x58, 0xe5, 0x7f, 0xc2, 0x92, 0x82, 0x63, 0xad, - 0x7c, 0xaa, 0x89, 0xcf, 0xdc, 0x4f, 0x1c, 0x72, 0x00, 0xc1, 0x56, 0xc6, 0x99, 0xe4, 0xe8, 0x64, - 0x87, 0xe7, 0x39, 0x7b, 0xc5, 0x2f, 0xaf, 0xb8, 0xae, 0xa2, 0x5b, 0xaf, 0x62, 0xd9, 0x07, 0xaf, - 0xd6, 0x07, 0xf2, 0x10, 0x82, 0x2e, 0x4f, 0xb8, 0xe4, 0x66, 0x3b, 0xfe, 0xc1, 0x2e, 0xe9, 0xd9, - 0x18, 0xae, 0x96, 0x0d, 0x1e, 0x80, 0xaf, 0x56, 0x0d, 0x43, 0x68, 0x3f, 0xbe, 0x5d, 0xd5, 0xa9, - 0xdc, 0x42, 0x8a, 0x02, 0x24, 0xb1, 0x46, 0x31, 0x9e, 0x2b, 0x13, 0x9b, 0x31, 0x4a, 0x0f, 0x8d, - 0x2b, 0x0f, 0x5d, 0xad, 0x54, 0xae, 0xea, 0x6b, 0x6a, 0xbc, 0x3d, 0xb3, 0xe9, 0xde, 0xd4, 0x1b, - 0xe9, 0xc3, 0xdb, 0xda, 0xc2, 0x97, 0x27, 0x2c, 0x4e, 0xd8, 0x61, 0x72, 0xcd, 0x8e, 0xcc, 0x08, - 0x3c, 0x84, 0x79, 0xd4, 0x8d, 0xba, 0x66, 0x0b, 0x2c, 0x49, 0x7e, 0x30, 0xf2, 0x6a, 0xf4, 0x77, - 0xd9, 0x90, 0x1b, 0x6b, 0xf8, 0x2e, 0xf3, 0x75, 0xaf, 0xce, 0x57, 0x39, 0x56, 0xeb, 0xa2, 0x4e, - 0x9d, 0xa7, 0x1c, 0x23, 0x41, 0x9e, 0x40, 0xa3, 0xd7, 0x3f, 0xe2, 0x43, 0x16, 0x7c, 0x00, 0xf3, - 0x18, 0x21, 0xcf, 0xcd, 0x44, 0xdf, 0x9a, 0xea, 0x14, 0xb5, 0x7c, 0xd2, 0x35, 0x99, 0xcd, 0x8c, - 0xe9, 0x01, 0x34, 0xd0, 0x7b, 0x1e, 0xfa, 0xd3, 0x66, 0x10, 0xa7, 0x86, 0x4d, 0xb6, 0xc1, 0xdb, - 0xa7, 0x91, 0xda, 0x54, 0x8c, 0xc0, 0x5a, 0x31, 0x94, 0xb2, 0xfd, 0xb5, 0xc8, 0xa5, 0xa9, 0x13, - 0xbe, 0x15, 0xf6, 0x52, 0x64, 0x12, 0x6b, 0xd4, 0xa1, 0xf8, 0x26, 0x39, 0xf8, 0xbb, 0x62, 0xc0, - 0x83, 0x45, 0x70, 0xa3, 0xae, 0xb1, 0xe1, 0x46, 0xdd, 0xe0, 0x5d, 0x34, 0x6f, 0x4a, 0xd3, 0xa9, - 0x82, 0xd8, 0xa7, 0x11, 0x45, 0xc7, 0xf7, 0xa1, 0x13, 0xe5, 0x5b, 0x42, 0x64, 0x83, 0x38, 0x65, - 0x52, 0x64, 0xe6, 0x1b, 0x30, 0x09, 0xe2, 0x06, 0x49, 0x26, 0xf5, 0xc5, 0x6e, 0x51, 0x4d, 0x90, - 0x67, 0xb0, 0xa4, 0x9c, 0x22, 0x61, 0xfb, 0xbd, 0x02, 0x0d, 0x85, 0x95, 0x41, 0x18, 0xaa, 0xb2, - 0xe0, 0xd6, 0x2d, 0x7c, 0xab, 0x2d, 0x6c, 0x9f, 0xf0, 0x54, 0xd6, 0x26, 0x06, 0x69, 0x34, 0xd0, - 0xa1, 0x9a, 0x08, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x58, 0x65, 0xa2, 0x50, 0x8a, 0x3c, 0xf2, 0xab, - 0x03, 0x60, 0x03, 0x2a, 0xf2, 0x52, 0xc5, 0xb9, 0x5c, 0x25, 0x58, 0xb7, 0x9d, 0x37, 0xdb, 0xb2, - 0x54, 0x49, 0x69, 0x9c, 0xda, 0xc9, 0xf8, 0xa8, 0x9a, 0x0c, 0xdd, 0xd2, 0xbb, 0x53, 0x93, 0xa1, - 0xbd, 0x56, 0xf3, 0xf1, 0x12, 0xda, 0x35, 0x7c, 0xe6, 0x94, 0x7c, 0x58, 0x4e, 0x89, 0x3b, 0x6d, - 0x12, 0x71, 0x63, 0xd2, 0xce, 0xca, 0x0b, 0x68, 0xd7, 0xe0, 0x99, 0x16, 0xd7, 0xe1, 0xd6, 0xe4, - 0x1e, 0xda, 0xfb, 0x3e, 0x0d, 0x93, 0x18, 0x3a, 0x5b, 0x49, 0x91, 0x4b, 0x9e, 0x19, 0x73, 0xea, - 0xa3, 0xa0, 0x81, 0xb2, 0x79, 0x15, 0x30, 0xbb, 0x7f, 0xc1, 0x7d, 0x98, 0x53, 0x65, 0xd4, 0xeb, - 0x74, 0xb1, 0xc6, 0x9a, 0x49, 0x0e, 0xa0, 0xb9, 0xd9, 0x8b, 0x9e, 0x67, 0xa2, 0x18, 0xcd, 0x0c, - 0xda, 0x7e, 0xd3, 0xdd, 0x8b, 0xdf, 0x74, 0xef, 0xc2, 0x37, 0xdd, 0x2f, 0xbf, 0xe9, 0xa4, 0x07, - 0xcb, 0xfa, 0x54, 0xaa, 0x2d, 0xbe, 0xc9, 0xc1, 0xb1, 0x1f, 0x52, 0xaf, 0xf6, 0x21, 0xed, 0xc1, - 0xb2, 0xbe, 0x67, 0xff, 0xa5, 0xd1, 0xdf, 0x5d, 0x58, 0xa6, 0x3c, 0x8f, 0x5f, 0xf3, 0x28, 0xcd, - 0x65, 0x56, 0xf4, 0xd5, 0x4d, 0x52, 0xfa, 0xdf, 0x88, 0x43, 0x53, 0x6d, 0x8f, 0x6a, 0xe2, 0x3a, - 0x93, 0x1e, 0x3c, 0x82, 0xf6, 0xf4, 0xce, 0x5e, 0x14, 0xad, 0x8b, 0x04, 0x8f, 0x60, 0xbe, 0x27, - 0x8a, 0xac, 0x5f, 0x8e, 0x6f, 0xed, 0x4e, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xe0, 0xe9, 0xd4, - 0x80, 0x84, 0x0d, 0xf4, 0xf2, 0xff, 0x4a, 0x6f, 0x82, 0x4d, 0xa7, 0xc6, 0xe9, 0xe3, 0xfa, 0x2e, - 0x86, 0xf3, 0xa8, 0x7b, 0x67, 0x32, 0x42, 0xa3, 0x58, 0x93, 0x23, 0xbf, 0x38, 0xb0, 0x50, 0x0f, - 0xe7, 0x5a, 0x4b, 0x5c, 0x76, 0xc7, 0x9d, 0xd9, 0x1d, 0x6f, 0x56, 0x77, 0xfc, 0xaa, 0x3b, 0xd5, - 0xef, 0x83, 0xb9, 0xda, 0xef, 0x03, 0x72, 0x0c, 0xf7, 0x2e, 0xb4, 0x6c, 0x4b, 0x0c, 0x47, 0x6a, - 0x36, 0xfe, 0x45, 0xeb, 0xd4, 0x79, 0xcb, 0x32, 0xd3, 0xb4, 0x16, 0xd5, 0x04, 0xf9, 0x14, 0xee, - 0xf6, 0xb8, 0xac, 0x35, 0xcc, 0x4e, 0xde, 0x1a, 0x78, 0xbb, 0xfc, 0xf4, 0x92, 0xf4, 0x15, 0x8b, - 0x7c, 0x01, 0xe1, 0xfe, 0x68, 0xc0, 0x24, 0xbf, 0x91, 0xf6, 0x26, 0x34, 0xf7, 0xc4, 0x48, 0x24, - 0xe2, 0xd5, 0xf8, 0x8a, 0x0b, 0x10, 0xc2, 0xbc, 0xbe, 0xe5, 0xfa, 0xa4, 0xb4, 0xa8, 0x25, 0xc9, - 0x6d, 0x35, 0xdc, 0x7d, 0x96, 0xf4, 0x8b, 0x44, 0x85, 0xa1, 0x7e, 0x3b, 0xe6, 0x9b, 0x4b, 0x7f, - 0x9c, 0xaf, 0x3a, 0x7f, 0x9e, 0xaf, 0x3a, 0x6f, 0xce, 0x57, 0x9d, 0xdf, 0xfe, 0x5a, 0xfd, 0xdf, - 0x61, 0x03, 0xff, 0x83, 0x3c, 0xf9, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xd8, 0x6d, 0x1f, 0x94, - 0x0c, 0x00, 0x00, +var fileDescriptorPrivate = []byte{ + // 1180 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, + 0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x75, 0xea, 0x4c, 0xdb, 0xb0, 0x2d, 0x28, 0x98, 0x51, 0x45, + 0x4d, 0x25, 0x42, 0xd5, 0x72, 0xc1, 0xa9, 0x52, 0x71, 0x1c, 0xca, 0x52, 0x12, 0xca, 0x38, 0xc9, + 0x1d, 0x17, 0x13, 0x7b, 0xd4, 0xac, 0xb2, 0xde, 0x35, 0xbb, 0xb3, 0x49, 0xdc, 0x0b, 0x6e, 0x41, + 0xe2, 0x05, 0xfa, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x02, 0x0a, 0x2f, 0x82, 0xe6, 0x9f, 0xd9, 0x83, + 0x1d, 0x87, 0x44, 0x81, 0xbb, 0xf9, 0xbf, 0x7f, 0xfe, 0xf3, 0x61, 0x67, 0xa1, 0x35, 0x49, 0x82, + 0x63, 0x2e, 0xc5, 0xc6, 0x24, 0x89, 0x65, 0x4c, 0xea, 0x41, 0x24, 0x45, 0x12, 0xf1, 0x90, 0x3e, + 0x87, 0x86, 0x1f, 0x8d, 0xc4, 0xe9, 0xb6, 0x90, 0x9c, 0x10, 0x70, 0x5f, 0x88, 0x69, 0xea, 0x39, + 0x1d, 0xab, 0x5b, 0x67, 0x78, 0x26, 0x1f, 0xc0, 0xca, 0x6e, 0xc2, 0x87, 0x47, 0x5b, 0xa7, 0x41, + 0x2a, 0x45, 0x34, 0x14, 0x9e, 0x8b, 0xdc, 0x39, 0x94, 0xbe, 0xb1, 0xe1, 0xc6, 0xd7, 0x81, 0x08, + 0x47, 0xdf, 0x4f, 0x64, 0x10, 0x47, 0x29, 0x79, 0x17, 0x1a, 0x9b, 0x7c, 0x78, 0x28, 0x76, 0xa7, + 0x13, 0x81, 0x1a, 0x1b, 0xac, 0x04, 0x0a, 0xee, 0x20, 0x78, 0xad, 0x35, 0xb6, 0x58, 0x09, 0x90, + 0x0e, 0x34, 0x77, 0x83, 0xb1, 0xf8, 0x21, 0xe3, 0x91, 0xcc, 0xc6, 0xde, 0x12, 0x4a, 0x57, 0x21, + 0xe5, 0x2a, 0x2a, 0xae, 0x23, 0x0b, 0xcf, 0xe4, 0x36, 0x38, 0xdb, 0x41, 0xe4, 0x35, 0x3a, 0x56, + 0xd7, 0xe9, 0xd9, 0x9e, 0xc5, 0x14, 0x89, 0x28, 0x3f, 0xf5, 0xa0, 0x82, 0xf2, 0xd3, 0x22, 0xd4, + 0xe6, 0x6c, 0xa8, 0x3b, 0xf1, 0x40, 0xf2, 0x68, 0xc4, 0x93, 0xd1, 0x7e, 0x20, 0x4e, 0xbc, 0x1b, + 0x3a, 0xd4, 0x59, 0x54, 0xc9, 0xf6, 0x78, 0x2a, 0xbc, 0x96, 0x52, 0xc9, 0xf0, 0x4c, 0xee, 0x41, + 0xbd, 0x17, 0xc8, 0xbe, 0x98, 0xc8, 0x43, 0x6f, 0xa5, 0x63, 0x75, 0x5d, 0x56, 0xd0, 0x94, 0xc2, + 0x8a, 0x3f, 0x9e, 0xc4, 0x89, 0x64, 0x22, 0x9d, 0xc4, 0x51, 0x2a, 0x48, 0x1b, 0x9c, 0xad, 0x24, + 0xf1, 0x2c, 0x74, 0x5e, 0x1d, 0xe9, 0xcf, 0xd0, 0xee, 0x85, 0xf1, 0xf0, 0xa8, 0xcf, 0x25, 0x67, + 0xe2, 0xa7, 0x4c, 0xa4, 0x92, 0xdc, 0x86, 0x25, 0xac, 0x8d, 0xb9, 0xa7, 0x09, 0x85, 0x62, 0x9e, + 0x3d, 0x5b, 0xa3, 0x48, 0x28, 0x14, 0xe5, 0x31, 0xd3, 0x2e, 0xd3, 0x84, 0x42, 0x07, 0x87, 0x3c, + 0x19, 0x61, 0x86, 0x5d, 0xa6, 0x09, 0xe5, 0x3f, 0x46, 0xa7, 0xd3, 0x8a, 0x67, 0xea, 0xc3, 0x6a, + 0xc5, 0xbe, 0x71, 0x73, 0x0d, 0x6a, 0x2c, 0x3e, 0xf1, 0xfb, 0xa9, 0x67, 0x75, 0x9c, 0xae, 0xcb, + 0x0c, 0x85, 0xc5, 0x8b, 0xc3, 0x6c, 0x1c, 0x29, 0x96, 0x8d, 0xac, 0x12, 0xa0, 0x77, 0x61, 0x09, + 0x2b, 0xa9, 0xa2, 0x2c, 0x65, 0xd5, 0x91, 0xfe, 0x62, 0x41, 0x63, 0x9b, 0x9f, 0xa2, 0x1b, 0x29, + 0x79, 0x0a, 0xf5, 0x3c, 0xaf, 0x78, 0xa9, 0xf9, 0xf8, 0xfd, 0x8d, 0xbc, 0x31, 0x37, 0x8a, 0x6b, + 0x1b, 0xf9, 0x9d, 0xad, 0x48, 0x26, 0x53, 0x56, 0x88, 0xdc, 0xfb, 0x02, 0x5a, 0x33, 0x2c, 0x65, + 0xef, 0x48, 0x4c, 0xf3, 0xac, 0x1e, 0x89, 0xa9, 0x8a, 0xff, 0x98, 0x87, 0x99, 0xc0, 0x5c, 0xb9, + 0x4c, 0x13, 0x9f, 0xdb, 0x9f, 0x5a, 0x74, 0x1f, 0xc8, 0x66, 0x22, 0xb8, 0x14, 0x68, 0x64, 0x5b, + 0xa4, 0x29, 0x7f, 0x25, 0x2e, 0xce, 0xb8, 0xce, 0xa2, 0x5d, 0xcd, 0x62, 0x51, 0x07, 0xa7, 0x52, + 0x07, 0xfa, 0x10, 0x48, 0x5f, 0x84, 0x42, 0x0a, 0x33, 0x55, 0xff, 0xa2, 0x97, 0x0e, 0x72, 0x1f, + 0x2e, 0xbf, 0x4b, 0x1e, 0x80, 0xab, 0x46, 0x14, 0x5d, 0x68, 0x3e, 0xbe, 0x55, 0xe6, 0xa9, 0x98, + 0x5e, 0x86, 0x17, 0x68, 0x98, 0x2b, 0x45, 0x7f, 0x2e, 0x0d, 0x6c, 0x41, 0x2b, 0x3d, 0x34, 0xa6, + 0x1c, 0x34, 0xb5, 0x56, 0x9a, 0xaa, 0x8e, 0xb7, 0xb1, 0xf6, 0x2c, 0x0f, 0xf7, 0xba, 0xd6, 0xe8, + 0x10, 0xde, 0xd1, 0x1a, 0xbe, 0x3a, 0xe6, 0x41, 0xc8, 0x0f, 0xc2, 0x2b, 0x56, 0x64, 0x81, 0xe3, + 0x1e, 0x2c, 0xa3, 0xac, 0xdf, 0x37, 0x53, 0x90, 0x93, 0xf4, 0x47, 0x73, 0x5f, 0xb5, 0xfe, 0x0e, + 0x1f, 0x0b, 0xa3, 0x0d, 0xcf, 0x45, 0xbc, 0xf6, 0xe5, 0xf1, 0x2a, 0xc3, 0x6a, 0x5c, 0xd4, 0x8a, + 0x74, 0x94, 0x61, 0x24, 0xe8, 0x13, 0xa8, 0x0d, 0x86, 0x87, 0x62, 0xcc, 0xc9, 0x87, 0xb0, 0x8c, + 0x1e, 0x8a, 0xd4, 0x74, 0xf4, 0xcd, 0xb9, 0x4a, 0xb1, 0x9c, 0x4f, 0xfb, 0x26, 0xb2, 0x85, 0x3e, + 0x3d, 0x80, 0x1a, 0x5a, 0x4f, 0x3d, 0x77, 0x5e, 0x0d, 0xe2, 0xcc, 0xb0, 0xe9, 0x16, 0x38, 0x7b, + 0xcc, 0x57, 0x93, 0x8a, 0x1e, 0xe4, 0x5a, 0x0c, 0xa5, 0x74, 0x7f, 0x13, 0xa7, 0xd2, 0xe4, 0x09, + 0xcf, 0x0a, 0x7b, 0x19, 0x27, 0x12, 0x73, 0xd4, 0x62, 0x78, 0xa6, 0x29, 0xb8, 0x3b, 0xf1, 0x48, + 0x90, 0x15, 0xb0, 0xfd, 0xbe, 0xd1, 0x61, 0xfb, 0x7d, 0xf2, 0x1e, 0xaa, 0x37, 0xa9, 0x69, 0x95, + 0x4e, 0xec, 0x31, 0x9f, 0xa1, 0xe1, 0xfb, 0xd0, 0xf2, 0xd3, 0xcd, 0x38, 0x4e, 0x46, 0x41, 0xc4, + 0x65, 0x9c, 0x98, 0x6f, 0xc7, 0x2c, 0x88, 0x13, 0x24, 0xb9, 0xd4, 0x9b, 0xbe, 0xc1, 0x34, 0x41, + 0x9f, 0x41, 0x5b, 0x19, 0x45, 0x22, 0xaf, 0xf7, 0x1a, 0xd4, 0x14, 0x56, 0x38, 0x61, 0xa8, 0x52, + 0x83, 0x5d, 0xd5, 0xf0, 0x9d, 0xd6, 0xb0, 0x75, 0x2c, 0x22, 0x59, 0xe9, 0x18, 0xa4, 0x51, 0x41, + 0x8b, 0x69, 0x82, 0x50, 0x1d, 0xa0, 0x89, 0x64, 0xa5, 0x8c, 0x44, 0xa1, 0x0c, 0x79, 0xf4, 0x37, + 0x0b, 0x20, 0x77, 0x28, 0x4b, 0x0b, 0x11, 0xeb, 0x62, 0x11, 0xd2, 0xcd, 0x2b, 0x6f, 0xa6, 0xa5, + 0x5d, 0xde, 0xd2, 0x38, 0xcb, 0x3b, 0xe3, 0xe3, 0xb2, 0x33, 0x74, 0x49, 0xef, 0xcc, 0x75, 0x86, + 0xb6, 0x5a, 0xf6, 0xc7, 0x4b, 0x68, 0x56, 0xf0, 0x85, 0x5d, 0xf2, 0x51, 0xd1, 0x25, 0xf6, 0xbc, + 0x4a, 0xc4, 0x8d, 0xca, 0xbc, 0x57, 0x5e, 0x40, 0xb3, 0x02, 0x2f, 0xd4, 0xd8, 0x85, 0x9b, 0xb3, + 0x73, 0x98, 0xef, 0xf7, 0x79, 0x98, 0x06, 0xd0, 0xda, 0x0c, 0xb3, 0x54, 0x8a, 0xc4, 0xa8, 0x53, + 0x1f, 0x05, 0x0d, 0x14, 0xc5, 0x2b, 0x81, 0xc5, 0xf5, 0x23, 0xf7, 0x61, 0x49, 0xa5, 0x51, 0x8f, + 0xd3, 0xf9, 0x1c, 0x6b, 0x26, 0xdd, 0x87, 0x7a, 0x6f, 0xe0, 0x3f, 0x4f, 0xe2, 0x6c, 0xb2, 0xd0, + 0xe9, 0xfc, 0x2d, 0x60, 0x57, 0xde, 0x02, 0x6d, 0xfd, 0x16, 0x70, 0xf0, 0x13, 0x8d, 0xef, 0x80, + 0xb6, 0x7e, 0x07, 0xb8, 0x06, 0xe1, 0x6a, 0xff, 0xae, 0xea, 0x55, 0xa9, 0xa6, 0xf8, 0x3a, 0x0b, + 0x27, 0xff, 0x90, 0x3a, 0x95, 0x0f, 0xe9, 0x00, 0x56, 0xf5, 0x3e, 0xfb, 0x3f, 0x95, 0xfe, 0x6e, + 0xc3, 0x2a, 0x13, 0x69, 0xf0, 0x5a, 0xf8, 0x51, 0x2a, 0x93, 0x6c, 0xa8, 0x76, 0x92, 0x92, 0xff, + 0x36, 0x3e, 0x30, 0xd9, 0x76, 0x98, 0x26, 0xae, 0xd2, 0xe9, 0xe4, 0x11, 0x34, 0xe7, 0x67, 0xf6, + 0xfc, 0xd5, 0xea, 0x15, 0xf2, 0x08, 0x96, 0x07, 0x71, 0x96, 0x0c, 0x8b, 0xf6, 0xad, 0xec, 0x49, + 0xed, 0x99, 0x66, 0xb3, 0xfc, 0x1a, 0x79, 0x3a, 0xd7, 0x20, 0x5e, 0x0d, 0xad, 0xbc, 0x5d, 0xca, + 0xcd, 0xb0, 0xd9, 0x5c, 0x3b, 0x7d, 0x52, 0x9d, 0x45, 0x6f, 0x19, 0x65, 0x6f, 0xcf, 0x7a, 0x68, + 0x04, 0x2b, 0xf7, 0xe8, 0xaf, 0x16, 0xdc, 0xa8, 0xba, 0x73, 0xa5, 0x21, 0x2e, 0xaa, 0x63, 0x2f, + 0xac, 0x8e, 0xb3, 0xa8, 0x3a, 0x6e, 0x59, 0x9d, 0xf2, 0x7d, 0xb0, 0x54, 0x79, 0x1f, 0xd0, 0x23, + 0xb8, 0x7b, 0xae, 0x64, 0x9b, 0xf1, 0x78, 0xa2, 0x7a, 0xe3, 0x3f, 0x94, 0x4e, 0xad, 0xb7, 0x24, + 0x31, 0x45, 0x6b, 0x30, 0x4d, 0xd0, 0xcf, 0xe0, 0xce, 0x40, 0xc8, 0x4a, 0xc1, 0xf2, 0xce, 0xeb, + 0x80, 0xb3, 0x23, 0x4e, 0x2e, 0x08, 0x5f, 0xb1, 0xe8, 0x97, 0xe0, 0xed, 0x4d, 0x46, 0x5c, 0x8a, + 0x6b, 0x49, 0xf7, 0xa0, 0xbe, 0x1b, 0x4f, 0xe2, 0x30, 0x7e, 0x35, 0xbd, 0x64, 0x03, 0x78, 0xb0, + 0xac, 0x77, 0xb9, 0x5e, 0x29, 0x0d, 0x96, 0x93, 0xf4, 0x96, 0x6a, 0xee, 0x21, 0x0f, 0x87, 0x59, + 0xa8, 0xdc, 0x50, 0x6f, 0xc7, 0xb4, 0xd7, 0xfe, 0xe3, 0x6c, 0xdd, 0xfa, 0xf3, 0x6c, 0xdd, 0xfa, + 0xeb, 0x6c, 0xdd, 0x7a, 0xf3, 0xf7, 0xfa, 0x5b, 0x07, 0x35, 0xfc, 0x77, 0x79, 0xf2, 0x4f, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x4f, 0xa0, 0xa9, 0x8b, 0xcc, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 971bb5f69..01890652d 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -11,11 +11,14 @@ message FieldOptions { string Type = 8; string CacheType = 3; uint32 CacheSize = 4; - int64 Min = 9; - int64 Max = 10; string TimeQuantum = 5; bool Keys = 11; bool NoStandardView = 12; + int64 Base = 13; + uint64 BitDepth = 14; + + int64 Min = 9 [deprecated=true]; + int64 Max = 10 [deprecated=true]; } message ImportResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 5cd86a833..708d43e2f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,13 +1,39 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto +/* + Package internal is a generated protocol buffer package. + + It is generated from these files: + public.proto + + It has these top-level messages: + Row + RowIdentifiers + Pair + FieldRow + GroupCount + ValCount + ColumnAttrSet + Attr + AttrMap + QueryRequest + QueryResponse + QueryResult + ImportRequest + ImportValueRequest + TranslateKeysRequest + TranslateKeysResponse + ImportRoaringRequestView + ImportRoaringRequest +*/ package internal import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import encoding_binary "encoding/binary" +import binary "encoding/binary" import io "io" @@ -23,46 +49,15 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Row struct { - Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *Row) Reset() { *m = Row{} } -func (m *Row) String() string { return proto.CompactTextString(m) } -func (*Row) ProtoMessage() {} -func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{0} -} -func (m *Row) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Row.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Row) XXX_Merge(src proto.Message) { - xxx_messageInfo_Row.Merge(dst, src) -} -func (m *Row) XXX_Size() int { - return m.Size() -} -func (m *Row) XXX_DiscardUnknown() { - xxx_messageInfo_Row.DiscardUnknown(m) -} - -var xxx_messageInfo_Row proto.InternalMessageInfo +func (m *Row) Reset() { *m = Row{} } +func (m *Row) String() string { return proto.CompactTextString(m) } +func (*Row) ProtoMessage() {} +func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} } func (m *Row) GetColumns() []uint64 { if m != nil { @@ -86,45 +81,14 @@ func (m *Row) GetAttrs() []*Attr { } type RowIdentifiers struct { - Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` - Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` + Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` } -func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } -func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } -func (*RowIdentifiers) ProtoMessage() {} -func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{1} -} -func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RowIdentifiers) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RowIdentifiers.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *RowIdentifiers) XXX_Merge(src proto.Message) { - xxx_messageInfo_RowIdentifiers.Merge(dst, src) -} -func (m *RowIdentifiers) XXX_Size() int { - return m.Size() -} -func (m *RowIdentifiers) XXX_DiscardUnknown() { - xxx_messageInfo_RowIdentifiers.DiscardUnknown(m) -} - -var xxx_messageInfo_RowIdentifiers proto.InternalMessageInfo +func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } +func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } +func (*RowIdentifiers) ProtoMessage() {} +func (*RowIdentifiers) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } func (m *RowIdentifiers) GetRows() []uint64 { if m != nil { @@ -141,46 +105,15 @@ func (m *RowIdentifiers) GetKeys() []string { } type Pair struct { - ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` - Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` - Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *Pair) Reset() { *m = Pair{} } -func (m *Pair) String() string { return proto.CompactTextString(m) } -func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{2} -} -func (m *Pair) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Pair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Pair.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Pair) XXX_Merge(src proto.Message) { - xxx_messageInfo_Pair.Merge(dst, src) -} -func (m *Pair) XXX_Size() int { - return m.Size() -} -func (m *Pair) XXX_DiscardUnknown() { - xxx_messageInfo_Pair.DiscardUnknown(m) -} - -var xxx_messageInfo_Pair proto.InternalMessageInfo +func (m *Pair) Reset() { *m = Pair{} } +func (m *Pair) String() string { return proto.CompactTextString(m) } +func (*Pair) ProtoMessage() {} +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } func (m *Pair) GetID() uint64 { if m != nil { @@ -204,46 +137,15 @@ func (m *Pair) GetCount() uint64 { } type FieldRow struct { - Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` - RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` - RowKey string `protobuf:"bytes,3,opt,name=RowKey,proto3" json:"RowKey,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` + RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` + RowKey string `protobuf:"bytes,3,opt,name=RowKey,proto3" json:"RowKey,omitempty"` } -func (m *FieldRow) Reset() { *m = FieldRow{} } -func (m *FieldRow) String() string { return proto.CompactTextString(m) } -func (*FieldRow) ProtoMessage() {} -func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{3} -} -func (m *FieldRow) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FieldRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FieldRow.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *FieldRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldRow.Merge(dst, src) -} -func (m *FieldRow) XXX_Size() int { - return m.Size() -} -func (m *FieldRow) XXX_DiscardUnknown() { - xxx_messageInfo_FieldRow.DiscardUnknown(m) -} - -var xxx_messageInfo_FieldRow proto.InternalMessageInfo +func (m *FieldRow) Reset() { *m = FieldRow{} } +func (m *FieldRow) String() string { return proto.CompactTextString(m) } +func (*FieldRow) ProtoMessage() {} +func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } func (m *FieldRow) GetField() string { if m != nil { @@ -267,45 +169,14 @@ func (m *FieldRow) GetRowKey() string { } type GroupCount struct { - Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` - Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *GroupCount) Reset() { *m = GroupCount{} } -func (m *GroupCount) String() string { return proto.CompactTextString(m) } -func (*GroupCount) ProtoMessage() {} -func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{4} -} -func (m *GroupCount) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *GroupCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_GroupCount.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *GroupCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_GroupCount.Merge(dst, src) -} -func (m *GroupCount) XXX_Size() int { - return m.Size() -} -func (m *GroupCount) XXX_DiscardUnknown() { - xxx_messageInfo_GroupCount.DiscardUnknown(m) -} - -var xxx_messageInfo_GroupCount proto.InternalMessageInfo +func (m *GroupCount) Reset() { *m = GroupCount{} } +func (m *GroupCount) String() string { return proto.CompactTextString(m) } +func (*GroupCount) ProtoMessage() {} +func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *GroupCount) GetGroup() []*FieldRow { if m != nil { @@ -322,45 +193,14 @@ func (m *GroupCount) GetCount() uint64 { } type ValCount struct { - Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` - Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *ValCount) Reset() { *m = ValCount{} } -func (m *ValCount) String() string { return proto.CompactTextString(m) } -func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{5} -} -func (m *ValCount) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ValCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ValCount.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ValCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValCount.Merge(dst, src) -} -func (m *ValCount) XXX_Size() int { - return m.Size() -} -func (m *ValCount) XXX_DiscardUnknown() { - xxx_messageInfo_ValCount.DiscardUnknown(m) -} - -var xxx_messageInfo_ValCount proto.InternalMessageInfo +func (m *ValCount) Reset() { *m = ValCount{} } +func (m *ValCount) String() string { return proto.CompactTextString(m) } +func (*ValCount) ProtoMessage() {} +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -377,46 +217,15 @@ func (m *ValCount) GetCount() int64 { } type ColumnAttrSet struct { - ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` - Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } -func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } -func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{6} -} -func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ColumnAttrSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ColumnAttrSet.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ColumnAttrSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_ColumnAttrSet.Merge(dst, src) -} -func (m *ColumnAttrSet) XXX_Size() int { - return m.Size() -} -func (m *ColumnAttrSet) XXX_DiscardUnknown() { - xxx_messageInfo_ColumnAttrSet.DiscardUnknown(m) -} - -var xxx_messageInfo_ColumnAttrSet proto.InternalMessageInfo +func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } +func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } +func (*ColumnAttrSet) ProtoMessage() {} +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -440,49 +249,18 @@ func (m *ColumnAttrSet) GetAttrs() []*Attr { } type Attr struct { - Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` - Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` - StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` - IntValue int64 `protobuf:"varint,4,opt,name=IntValue,proto3" json:"IntValue,omitempty"` - BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` - FloatValue float64 `protobuf:"fixed64,6,opt,name=FloatValue,proto3" json:"FloatValue,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + IntValue int64 `protobuf:"varint,4,opt,name=IntValue,proto3" json:"IntValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` + FloatValue float64 `protobuf:"fixed64,6,opt,name=FloatValue,proto3" json:"FloatValue,omitempty"` } -func (m *Attr) Reset() { *m = Attr{} } -func (m *Attr) String() string { return proto.CompactTextString(m) } -func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{7} -} -func (m *Attr) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Attr) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Attr.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Attr) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attr.Merge(dst, src) -} -func (m *Attr) XXX_Size() int { - return m.Size() -} -func (m *Attr) XXX_DiscardUnknown() { - xxx_messageInfo_Attr.DiscardUnknown(m) -} - -var xxx_messageInfo_Attr proto.InternalMessageInfo +func (m *Attr) Reset() { *m = Attr{} } +func (m *Attr) String() string { return proto.CompactTextString(m) } +func (*Attr) ProtoMessage() {} +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *Attr) GetKey() string { if m != nil { @@ -527,44 +305,13 @@ func (m *Attr) GetFloatValue() float64 { } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *AttrMap) Reset() { *m = AttrMap{} } -func (m *AttrMap) String() string { return proto.CompactTextString(m) } -func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{8} -} -func (m *AttrMap) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *AttrMap) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_AttrMap.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *AttrMap) XXX_Merge(src proto.Message) { - xxx_messageInfo_AttrMap.Merge(dst, src) -} -func (m *AttrMap) XXX_Size() int { - return m.Size() -} -func (m *AttrMap) XXX_DiscardUnknown() { - xxx_messageInfo_AttrMap.DiscardUnknown(m) -} - -var xxx_messageInfo_AttrMap proto.InternalMessageInfo +func (m *AttrMap) Reset() { *m = AttrMap{} } +func (m *AttrMap) String() string { return proto.CompactTextString(m) } +func (*AttrMap) ProtoMessage() {} +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -574,49 +321,18 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` - ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` - ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` - ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` + ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` } -func (m *QueryRequest) Reset() { *m = QueryRequest{} } -func (m *QueryRequest) String() string { return proto.CompactTextString(m) } -func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{9} -} -func (m *QueryRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *QueryRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryRequest.Merge(dst, src) -} -func (m *QueryRequest) XXX_Size() int { - return m.Size() -} -func (m *QueryRequest) XXX_DiscardUnknown() { - xxx_messageInfo_QueryRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryRequest proto.InternalMessageInfo +func (m *QueryRequest) Reset() { *m = QueryRequest{} } +func (m *QueryRequest) String() string { return proto.CompactTextString(m) } +func (*QueryRequest) ProtoMessage() {} +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -661,46 +377,15 @@ func (m *QueryRequest) GetExcludeColumns() bool { } type QueryResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` } -func (m *QueryResponse) Reset() { *m = QueryResponse{} } -func (m *QueryResponse) String() string { return proto.CompactTextString(m) } -func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{10} -} -func (m *QueryResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *QueryResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResponse.Merge(dst, src) -} -func (m *QueryResponse) XXX_Size() int { - return m.Size() -} -func (m *QueryResponse) XXX_DiscardUnknown() { - xxx_messageInfo_QueryResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryResponse proto.InternalMessageInfo +func (m *QueryResponse) Reset() { *m = QueryResponse{} } +func (m *QueryResponse) String() string { return proto.CompactTextString(m) } +func (*QueryResponse) ProtoMessage() {} +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -724,52 +409,21 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` - RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` } -func (m *QueryResult) Reset() { *m = QueryResult{} } -func (m *QueryResult) String() string { return proto.CompactTextString(m) } -func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{11} -} -func (m *QueryResult) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryResult.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *QueryResult) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResult.Merge(dst, src) -} -func (m *QueryResult) XXX_Size() int { - return m.Size() -} -func (m *QueryResult) XXX_DiscardUnknown() { - xxx_messageInfo_QueryResult.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryResult proto.InternalMessageInfo +func (m *QueryResult) Reset() { *m = QueryResult{} } +func (m *QueryResult) String() string { return proto.CompactTextString(m) } +func (*QueryResult) ProtoMessage() {} +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -835,51 +489,20 @@ func (m *QueryResult) GetRowIdentifiers() *RowIdentifiers { } type ImportRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` - ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` - Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` + RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` + ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` + Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` } -func (m *ImportRequest) Reset() { *m = ImportRequest{} } -func (m *ImportRequest) String() string { return proto.CompactTextString(m) } -func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{12} -} -func (m *ImportRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRequest.Merge(dst, src) -} -func (m *ImportRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportRequest proto.InternalMessageInfo +func (m *ImportRequest) Reset() { *m = ImportRequest{} } +func (m *ImportRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRequest) ProtoMessage() {} +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -938,49 +561,18 @@ func (m *ImportRequest) GetTimestamps() []int64 { } type ImportValueRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` - Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` + Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` } -func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } -func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } -func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{13} -} -func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportValueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportValueRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportValueRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportValueRequest.Merge(dst, src) -} -func (m *ImportValueRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportValueRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportValueRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportValueRequest proto.InternalMessageInfo +func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } +func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } +func (*ImportValueRequest) ProtoMessage() {} +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -1025,46 +617,15 @@ func (m *ImportValueRequest) GetValues() []int64 { } type TranslateKeysRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` } -func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } -func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } -func (*TranslateKeysRequest) ProtoMessage() {} -func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{14} -} -func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *TranslateKeysRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_TranslateKeysRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *TranslateKeysRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysRequest.Merge(dst, src) -} -func (m *TranslateKeysRequest) XXX_Size() int { - return m.Size() -} -func (m *TranslateKeysRequest) XXX_DiscardUnknown() { - xxx_messageInfo_TranslateKeysRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_TranslateKeysRequest proto.InternalMessageInfo +func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } +func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysRequest) ProtoMessage() {} +func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{14} } func (m *TranslateKeysRequest) GetIndex() string { if m != nil { @@ -1088,44 +649,13 @@ func (m *TranslateKeysRequest) GetKeys() []string { } type TranslateKeysResponse struct { - IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` } -func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } -func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } -func (*TranslateKeysResponse) ProtoMessage() {} -func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{15} -} -func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *TranslateKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_TranslateKeysResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *TranslateKeysResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysResponse.Merge(dst, src) -} -func (m *TranslateKeysResponse) XXX_Size() int { - return m.Size() -} -func (m *TranslateKeysResponse) XXX_DiscardUnknown() { - xxx_messageInfo_TranslateKeysResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_TranslateKeysResponse proto.InternalMessageInfo +func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } +func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysResponse) ProtoMessage() {} +func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{15} } func (m *TranslateKeysResponse) GetIDs() []uint64 { if m != nil { @@ -1135,45 +665,14 @@ func (m *TranslateKeysResponse) GetIDs() []uint64 { } type ImportRoaringRequestView struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` } -func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} } -func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } -func (*ImportRoaringRequestView) ProtoMessage() {} -func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{16} -} -func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src) -} -func (m *ImportRoaringRequestView) XXX_Size() int { - return m.Size() -} -func (m *ImportRoaringRequestView) XXX_DiscardUnknown() { - xxx_messageInfo_ImportRoaringRequestView.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportRoaringRequestView proto.InternalMessageInfo +func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} } +func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequestView) ProtoMessage() {} +func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{16} } func (m *ImportRoaringRequestView) GetName() string { if m != nil { @@ -1190,45 +689,14 @@ func (m *ImportRoaringRequestView) GetData() []byte { } type ImportRoaringRequest struct { - Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` - Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` + Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` } -func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } -func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } -func (*ImportRoaringRequest) ProtoMessage() {} -func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{17} -} -func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequest.Merge(dst, src) -} -func (m *ImportRoaringRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportRoaringRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportRoaringRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportRoaringRequest proto.InternalMessageInfo +func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } +func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequest) ProtoMessage() {} +func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{17} } func (m *ImportRoaringRequest) GetClear() bool { if m != nil { @@ -1323,9 +791,6 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1376,9 +841,6 @@ func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1413,9 +875,6 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) i += copy(dAtA[i:], m.Key) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1451,9 +910,6 @@ func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) i += copy(dAtA[i:], m.RowKey) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1489,9 +945,6 @@ func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1520,9 +973,6 @@ func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1564,9 +1014,6 @@ func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) i += copy(dAtA[i:], m.Key) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1620,12 +1067,9 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) i += 8 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1656,9 +1100,6 @@ func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1740,9 +1181,6 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1791,9 +1229,6 @@ func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1903,9 +1338,6 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i += n11 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2023,9 +1455,6 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2111,9 +1540,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2159,9 +1585,6 @@ func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2197,9 +1620,6 @@ func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j22)) i += copy(dAtA[i:], dAtA23[:j22]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2230,9 +1650,6 @@ func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) i += copy(dAtA[i:], m.Data) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2273,9 +1690,6 @@ func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2289,9 +1703,6 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { return offset + 1 } func (m *Row) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Columns) > 0 { @@ -2313,16 +1724,10 @@ func (m *Row) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *RowIdentifiers) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Rows) > 0 { @@ -2338,16 +1743,10 @@ func (m *RowIdentifiers) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Pair) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.ID != 0 { @@ -2360,16 +1759,10 @@ func (m *Pair) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *FieldRow) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Field) @@ -2383,16 +1776,10 @@ func (m *FieldRow) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *GroupCount) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Group) > 0 { @@ -2404,16 +1791,10 @@ func (m *GroupCount) Size() (n int) { if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ValCount) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Val != 0 { @@ -2422,16 +1803,10 @@ func (m *ValCount) Size() (n int) { if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ColumnAttrSet) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.ID != 0 { @@ -2447,16 +1822,10 @@ func (m *ColumnAttrSet) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Attr) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Key) @@ -2479,16 +1848,10 @@ func (m *Attr) Size() (n int) { if m.FloatValue != 0 { n += 9 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *AttrMap) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Attrs) > 0 { @@ -2497,16 +1860,10 @@ func (m *AttrMap) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *QueryRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Query) @@ -2532,16 +1889,10 @@ func (m *QueryRequest) Size() (n int) { if m.ExcludeColumns { n += 2 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *QueryResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Err) @@ -2560,16 +1911,10 @@ func (m *QueryResponse) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *QueryResult) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Row != nil { @@ -2612,16 +1957,10 @@ func (m *QueryResult) Size() (n int) { l = m.RowIdentifiers.Size() n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -2668,16 +2007,10 @@ func (m *ImportRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportValueRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -2711,16 +2044,10 @@ func (m *ImportValueRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *TranslateKeysRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -2737,16 +2064,10 @@ func (m *TranslateKeysRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *TranslateKeysResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.IDs) > 0 { @@ -2756,16 +2077,10 @@ func (m *TranslateKeysResponse) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportRoaringRequestView) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -2776,16 +2091,10 @@ func (m *ImportRoaringRequestView) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportRoaringRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Clear { @@ -2797,9 +2106,6 @@ func (m *ImportRoaringRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -2886,17 +2192,6 @@ func (m *Row) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Columns) == 0 { - m.Columns = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -2990,7 +2285,6 @@ func (m *Row) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3070,17 +2364,6 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Rows) == 0 { - m.Rows = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -3143,7 +2426,6 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3261,7 +2543,6 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3389,7 +2670,6 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3490,7 +2770,6 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3579,7 +2858,6 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3709,7 +2987,6 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3872,7 +3149,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 m.FloatValue = float64(math.Float64frombits(v)) default: @@ -3887,7 +3164,6 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3969,7 +3245,6 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4078,17 +3353,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Shards) == 0 { - m.Shards = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4202,7 +3466,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4344,7 +3607,6 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4579,17 +3841,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.RowIDs) == 0 { - m.RowIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4687,7 +3938,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4844,17 +4094,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.RowIDs) == 0 { - m.RowIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4917,17 +4156,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.ColumnIDs) == 0 { - m.ColumnIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4990,17 +4218,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Timestamps) == 0 { - m.Timestamps = make([]int64, 0, elementCount) - } for iNdEx < postIndex { var v int64 for shift := uint(0); ; shift += 7 { @@ -5092,7 +4309,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5249,17 +4465,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.ColumnIDs) == 0 { - m.ColumnIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5322,17 +4527,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Values) == 0 { - m.Values = make([]int64, 0, elementCount) - } for iNdEx < postIndex { var v int64 for shift := uint(0); ; shift += 7 { @@ -5395,7 +4589,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5533,7 +4726,6 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5613,17 +4805,6 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.IDs) == 0 { - m.IDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5657,7 +4838,6 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5768,7 +4948,6 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5870,7 +5049,6 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5985,9 +5163,9 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_f65cfea24ac19f54) } +func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } -var fileDescriptor_public_f65cfea24ac19f54 = []byte{ +var fileDescriptorPublic = []byte{ // 880 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x8e, 0x1b, 0x45, 0x10, 0xa6, 0x3d, 0x63, 0x7b, 0x5c, 0x5e, 0x9b, 0xa8, 0xe5, 0x84, 0x11, 0x8a, 0x8c, 0x35, 0x42, diff --git a/pilosa.go b/pilosa.go index 42ab3d3c1..41434f52f 100644 --- a/pilosa.go +++ b/pilosa.go @@ -40,8 +40,6 @@ var ( ErrInvalidBSIGroupType = errors.New("invalid bsigroup type") ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range") ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type") - ErrBSIGroupValueTooLow = errors.New("bsigroup value too low") - ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high") ErrInvalidRangeOperation = errors.New("invalid range operation") ErrInvalidBetweenValue = errors.New("invalid value for between operation") diff --git a/roaring/roaring.go b/roaring/roaring.go index 78b5f4ca8..2d1a2d08a 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -31,16 +31,18 @@ const ( // MagicNumber is an identifier, in bytes 0-1 of the file. MagicNumber = uint32(12348) - // storageVersion indicates the storage version, in bytes 2-3. + // storageVersion indicates the storage version, in byte 2. storageVersion = uint32(0) - // cookie is the first four bytes in a roaring bitmap file, + // NOTE: byte 3 stores user-defined flags. + + // cookie is the first 3 bytes in a roaring bitmap file, // formed by joining MagicNumber and storageVersion cookie = MagicNumber + storageVersion<<16 - // headerBaseSize is the size in bytes of the cookie and key count at the - // beginning of a file. - headerBaseSize = 4 + 4 + // headerBaseSize is the size in bytes of the cookie, flags, and key count + // at the beginning of a file. + headerBaseSize = 3 + 1 + 4 // runCountHeaderSize is the size in bytes of the run count stored // at the beginning of every serialized run container. @@ -123,6 +125,9 @@ type ContainerIterator interface { type Bitmap struct { Containers Containers + // User-defined flags. + Flags byte + // Number of bit change operations written to the writer. Some operations // contain multiple values, each of those counts the number of values rather // than counting as one operation. @@ -970,7 +975,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { n: 0, } - ew.WriteUint32(byte4, cookie) + ew.WriteUint32(byte4, cookie|(uint32(b.Flags)<<24)) ew.WriteUint32(byte4, uint32(containerCount)) // Descriptive header section: encode keys and cardinality. @@ -1032,7 +1037,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion. fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) - fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4])) + b.Flags = data[2] + fileVersion := uint32(data[3]) if fileMagic != MagicNumber { return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) } @@ -1041,8 +1047,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { return fmt.Errorf("wrong roaring version, file is v%d, server requires v%d", fileVersion, storageVersion) } - // Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)). - keyN := binary.LittleEndian.Uint32(data[4:8]) + // Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)). + keyN := binary.LittleEndian.Uint32(data[3+1 : 8]) headerSize := headerBaseSize b.Containers.Reset() @@ -4166,11 +4172,11 @@ const ( serialCookie = 12347 // runs, arrays, and bitmaps ) -func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) { +func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, flags byte, haveRuns bool, err error) { statsHit("readOfficialHeader") if len(buf) < 8 { err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } cf := func(index uint, card int) (newType byte) { newType = containerBitmap @@ -4180,7 +4186,8 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint return newType } containerTyper = cf - cookie := binary.LittleEndian.Uint32(buf) + cookie := binary.LittleEndian.Uint32(buf) & 0xFFFFFF + flags = buf[3] pos += 4 // cookie header @@ -4195,7 +4202,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint isRunBitmapSize := (int(size) + 7) / 8 if pos+isRunBitmapSize > len(buf) { err = fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize) - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } isRunBitmap := buf[pos : pos+isRunBitmapSize] @@ -4208,22 +4215,22 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint } } else { err = fmt.Errorf("did not find expected serialCookie in header") - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } header = pos if size > (1 << 16) { err = fmt.Errorf("it is logically impossible to have more than (1<<16) containers") - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } // descriptive header if pos+2*2*int(size) > len(buf) { err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } pos += 2 * 2 * int(size) // moving pos past keycount - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } // UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in @@ -4240,10 +4247,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") } - keyN, containerTyper, header, pos, haveRuns, err := readOfficialHeader(data) + keyN, containerTyper, header, pos, flags, haveRuns, err := readOfficialHeader(data) if err != nil { return errors.Wrap(err, "reading roaring header") } + b.Flags = flags b.Containers.Reset() // Descriptive header section: Read container keys and cardinalities. diff --git a/row.go b/row.go index e393a9db4..a2e938434 100644 --- a/row.go +++ b/row.go @@ -113,6 +113,16 @@ func (r *Row) Intersect(other *Row) *Row { return &Row{segments: segments} } +// Any returns true if row contains any bits. +func (r *Row) Any() bool { + for _, s := range r.segments { + if s.data.Any() { + return true + } + } + return false +} + // Xor returns the xor of r and other. func (r *Row) Xor(other *Row) *Row { var segments []rowSegment diff --git a/view.go b/view.go index e8894db7d..eee1ab810 100644 --- a/view.go +++ b/view.go @@ -166,6 +166,15 @@ func (v *view) close() error { return nil } +// flags returns a set of flags for the underlying fragments. +func (v *view) flags() byte { + var flag byte + if v.fieldType == FieldTypeInt { + flag |= roaringFlagBSIv2 + } + return flag +} + // availableShards returns a bitmap of shards which contain data. func (v *view) availableShards() *roaring.Bitmap { v.mu.RLock() @@ -254,7 +263,7 @@ func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { } func (v *view) newFragment(path string, shard uint64) *fragment { - frag := newFragment(path, v.index, v.field, v.name, shard) + frag := newFragment(path, v.index, v.field, v.name, shard, v.flags()) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.Logger = v.logger @@ -331,7 +340,7 @@ func (v *view) clearBit(rowID, columnID uint64) (changed bool, err error) { } // value uses a column of bits to read a multi-bit value. -func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +func (v *view) value(columnID uint64, bitDepth uint) (value int64, exists bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { @@ -341,7 +350,7 @@ func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, } // setValue uses a column of bits to set a multi-bit value. -func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +func (v *view) setValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { @@ -351,7 +360,7 @@ func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed b } // sum returns the sum & count of a field. -func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (v *view) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err error) { for _, f := range v.allFragments() { fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { @@ -364,7 +373,7 @@ func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { } // min returns the min and count of a field. -func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) { +func (v *view) min(filter *Row, bitDepth uint) (min int64, count uint64, err error) { var minHasValue bool for _, f := range v.allFragments() { fmin, fcount, err := f.min(filter, bitDepth) @@ -392,7 +401,7 @@ func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) { } // max returns the max and count of a field. -func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (v *view) max(filter *Row, bitDepth uint) (max int64, count uint64, err error) { for _, f := range v.allFragments() { fmax, fcount, err := f.max(filter, bitDepth) if err != nil { @@ -407,7 +416,7 @@ func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) { } // rangeOp returns rows with a field value encoding matching the predicate. -func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) { r := NewRow() for _, frag := range v.allFragments() { other, err := frag.rangeOp(op, bitDepth, predicate) @@ -419,6 +428,29 @@ func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err return r, nil } +// upgradeViewBSIv2 upgrades the fragments of v. Returns ok true if any fragment upgraded. +func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) { + // If reading from an old formatted BSI roaring bitmap, upgrade and reload. + for _, frag := range v.allFragments() { + if frag.storage.Flags&roaringFlagBSIv2 == 1 { + continue // already upgraded, skip + } + ok = true // mark as upgraded, requires reload + + oldPath := frag.path + if newPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil { + return ok, errors.Wrap(err, "upgrading bsi v2") + } else if err := frag.closeStorage(); err != nil { + return ok, errors.Wrap(err, "closing after bsi v2 upgrade") + } else if err := os.Rename(oldPath, newPath); err != nil { + return ok, errors.Wrap(err, "renaming after bsi v2 upgrade") + } else if err := frag.openStorage(); err != nil { + return ok, errors.Wrap(err, "re-opening after bsi v2 upgrade") + } + } + return ok, nil +} + // ViewInfo represents schema information for a view. type ViewInfo struct { Name string `json:"name"` From d4de122549782e5f796de53b33dc20d3f88d90fe Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 13 May 2019 16:00:11 -0600 Subject: [PATCH 57/73] Add min/max constraints --- api_test.go | 3 +- encoding/proto/proto.go | 2 + executor.go | 7 +- executor_test.go | 43 +++++------ field.go | 83 ++++++++++++--------- field_internal_test.go | 19 +++-- field_test.go | 5 +- fragment.go | 6 +- http/client_test.go | 6 +- http/handler.go | 3 +- index_test.go | 3 +- internal/private.proto | 155 ++++++++++++++++++++-------------------- server/handler_test.go | 2 +- 13 files changed, 185 insertions(+), 152 deletions(-) diff --git a/api_test.go b/api_test.go index ef737d596..45aec0419 100644 --- a/api_test.go +++ b/api_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "context" "fmt" + "math" "reflect" "strings" "testing" @@ -198,7 +199,7 @@ func TestAPI_ImportValue(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(0)) + _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) if err != nil { t.Fatalf("creating field: %v", err) } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 24cfcd232..f8a40badf 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -798,6 +798,8 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) m.Type = options.Type m.CacheType = options.CacheType m.CacheSize = options.CacheSize + m.Min = options.Min + m.Max = options.Max m.Base = options.Base m.BitDepth = uint(options.BitDepth) m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) diff --git a/executor.go b/executor.go index 2d05991ae..07a83496d 100644 --- a/executor.go +++ b/executor.go @@ -1408,7 +1408,6 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return frag.notNull() } else if cond.Op == pql.BETWEEN { - predicates, err := cond.IntSliceValue() if err != nil { return nil, errors.Wrap(err, "getting condition value") @@ -1442,7 +1441,7 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. - if predicates[0] <= bsig.Min() && predicates[1] >= bsig.Max() { + if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { return frag.notNull() } @@ -1474,8 +1473,8 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c } // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. - if (cond.Op == pql.LT && value > bsig.Max()) || (cond.Op == pql.LTE && value >= bsig.Max()) || - (cond.Op == pql.GT && value < bsig.Min()) || (cond.Op == pql.GTE && value <= bsig.Min()) { + if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || + (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { return frag.notNull() } diff --git a/executor_test.go b/executor_test.go index 9473d02ca..ae66c7ff6 100644 --- a/executor_test.go +++ b/executor_test.go @@ -20,6 +20,7 @@ import ( "flag" "fmt" "io/ioutil" + "math" "math/rand" "reflect" "strconv" @@ -769,7 +770,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) @@ -806,7 +807,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1214,7 +1215,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1278,7 +1279,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1371,15 +1372,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1429,15 +1430,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1831,19 +1832,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1942,7 +1943,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { {q: `Row(1000 < other <= 1000)`, exp: false}, {q: `Row(1000 < other < 2000)`, exp: false}, - {q: `Row(1000 <= other < 2000)`, exp: true}, + {q: `Row(1000 <= other < 20000)`, exp: true}, {q: `Row(1000 <= other <= 2000)`, exp: true}, {q: `Row(1000 < other <= 2000)`, exp: false}, } @@ -1955,7 +1956,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result for query: %s", test.q) + t.Fatalf("unexpected result for query: %s (%#v)", test.q, result.Results[0].(*pilosa.Row).Columns()) } }) } @@ -2025,19 +2026,19 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -2795,7 +2796,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0)) + _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } diff --git a/field.go b/field.go index ec7a1ccd0..9a33812c3 100644 --- a/field.go +++ b/field.go @@ -130,13 +130,15 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { // OptFieldTypeInt is a functional option on FieldOptions // used to specify the field as being type `int` and to // provide any respective configuration values. -func OptFieldTypeInt(base int64) FieldOption { +func OptFieldTypeInt(base, min, max int64) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) } fo.Type = FieldTypeInt fo.Base = base + fo.Min = min + fo.Max = max fo.BitDepth = 1 return nil } @@ -485,16 +487,21 @@ func (f *Field) loadMeta() error { } } - // Convert min/max from deprecated v1 int type. - if pb.Min != 0 || pb.Max != 0 { + // Initialize "base" to "min" when upgrading from v1 BSI format. + if pb.BitDepth == 0 { pb.Base = pb.Min - pb.BitDepth = uint64(bitDepth(uint64(pb.Max - pb.Min))) + pb.BitDepth = uint64(bitDepthInt64(pb.Max - pb.Min)) + if pb.BitDepth == 0 { + pb.BitDepth = 1 + } } // Copy metadata fields. f.options.Type = pb.Type f.options.CacheType = pb.CacheType f.options.CacheSize = pb.CacheSize + f.options.Min = pb.Min + f.options.Max = pb.Max f.options.Base = pb.Base f.options.BitDepth = uint(pb.BitDepth) f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) @@ -540,6 +547,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = opt.CacheSize } } + f.options.Min = 0 + f.options.Max = 0 f.options.Base = 0 f.options.BitDepth = 0 f.options.TimeQuantum = "" @@ -548,6 +557,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Type = opt.Type f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 + f.options.Min = opt.Min + f.options.Max = opt.Max f.options.Base = opt.Base f.options.BitDepth = opt.BitDepth f.options.TimeQuantum = "" @@ -557,6 +568,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { bsig := &bsiGroup{ Name: f.name, Type: bsiGroupTypeInt, + Min: opt.Min, + Max: opt.Max, Base: opt.Base, BitDepth: opt.BitDepth, } @@ -571,6 +584,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Type = opt.Type f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 f.options.Base = 0 f.options.BitDepth = 0 f.options.Keys = opt.Keys @@ -584,6 +599,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Type = FieldTypeBool f.options.CacheType = CacheTypeNone f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 f.options.Base = 0 f.options.BitDepth = 0 f.options.TimeQuantum = "" @@ -995,7 +1012,7 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { // SetValue sets a field value for a column. func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { - // Fetch bsiGroup. + // Fetch bsiGroup & validate min/max. bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound @@ -1003,9 +1020,10 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) // Determine base value to store. baseValue := int64(value - bsig.Base) + requiredBitDepth := bitDepthInt64(baseValue) // Increase bit depth value if the unsigned value is greater. - if value < bsig.Min() || value > bsig.Max() { + if requiredBitDepth > bsig.BitDepth { if err := func() error { f.mu.Lock() defer f.mu.Unlock() @@ -1099,7 +1117,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) bsig := f.bsiGroup(name) if bsig == nil { return nil, ErrBSIGroupNotFound - } else if predicate < bsig.Min() || predicate > bsig.Max() { + } else if predicate < bsig.Min || predicate > bsig.Max { return nil, nil } @@ -1365,6 +1383,8 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, Base: o.Base, BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), @@ -1454,20 +1474,12 @@ func isValidBSIGroupType(v string) bool { type bsiGroup struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` Base int64 `json:"base,omitempty"` BitDepth uint `json:"bitDepth,omitempty"` } -// Min returns the lowest possible value for the group based on current bit depth. -func (b *bsiGroup) Min() int64 { - return b.Base - (1 << b.BitDepth) + 1 -} - -// Max returns the highest possible value for the group based on current bit depth. -func (b *bsiGroup) Max() int64 { - return b.Base + (1 << b.BitDepth) - 1 -} - // baseValue adjusts the value to align with the range for Field for a certain // operation type. // Note: There is an edge case for GT and LT where this returns a baseValue @@ -1481,7 +1493,7 @@ func (b *bsiGroup) Max() int64 { // Executor.executeBSIGroupRangeShard() takes this into account and returns // `frag.FieldNotNull(bsig.BitDepth())` in such instances. func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue int64, outOfRange bool) { - min, max := b.Min(), b.Max() + min, max := b.bitDepthMin(), b.bitDepthMax() if op == pql.GT || op == pql.GTE { if value > max { @@ -1507,23 +1519,20 @@ func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue int64, outOfR } // baseValueBetween adjusts the min/max value to align with the range for Field. -func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax int64, outOfRange bool) { - bsiMin, bsiMax := b.Min(), b.Max() +func (b *bsiGroup) baseValueBetween(lo, hi int64) (baseValueLo, baseValueHi int64, outOfRange bool) { + min, max := b.bitDepthMin(), b.bitDepthMax() + if hi < min || lo > max { + return 0, 0, true + } - if max < bsiMin || min > bsiMax { - return baseValueMin, baseValueMax, true + // Limit lo/hi to possible bit range. + if lo < min { + lo = min } - // Adjust min/max to range. - if min > bsiMin { - baseValueMin = int64(min - b.Base) + if hi > max { + hi = max } - // Make sure the high value of the BETWEEN does not exceed BitDepth. - if max > bsiMax { - baseValueMax = int64(bsiMax - b.Base) - } else if max > bsiMin { - baseValueMax = int64(max - b.Base) - } - return baseValueMin, baseValueMax, false + return lo - b.Base, hi - b.Base, false } func (b *bsiGroup) validate() error { @@ -1535,6 +1544,16 @@ func (b *bsiGroup) validate() error { return nil } +// bitDepthMin returns the minimum value possible for the current bit depth. +func (b *bsiGroup) bitDepthMin() int64 { + return b.Base - (1 << b.BitDepth) + 1 +} + +// bitDepthMax returns the maximum value possible for the current bit depth. +func (b *bsiGroup) bitDepthMax() int64 { + return b.Base + (1 << b.BitDepth) - 1 +} + // Cache types. const ( CacheTypeLRU = "lru" diff --git a/field_internal_test.go b/field_internal_test.go index aa8692916..13851a493 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -15,7 +15,9 @@ package pilosa import ( + "fmt" "io/ioutil" + "math" "os" "reflect" "testing" @@ -32,19 +34,24 @@ func TestBSIGroup_BaseValue(t *testing.T) { Type: bsiGroupTypeInt, Base: -100, BitDepth: 10, + Min: -1000, + Max: 1000, } b1 := &bsiGroup{ Name: "b1", Type: bsiGroupTypeInt, Base: 0, BitDepth: 8, + Min: -255, + Max: 255, } - b2 := &bsiGroup{ Name: "b2", Type: bsiGroupTypeInt, Base: 100, BitDepth: 11, + Min: math.MinInt64, + Max: math.MaxInt64, } t.Run("Normal Condition", func(t *testing.T) { @@ -104,10 +111,12 @@ func TestBSIGroup_BaseValue(t *testing.T) { {b2, pql.EQ, 105, 5, false}, {b2, pql.EQ, 1105, 1005, false}, } { - bv, oor := tt.f.baseValue(tt.op, tt.val) - if oor != tt.expOutOfRange || !reflect.DeepEqual(bv, tt.expBaseValue) { - t.Errorf("%d. %s) baseValue(%s, %v)=(%v, %v), expected (%v, %v)", i, tt.f.Name, tt.op, tt.val, bv, oor, tt.expBaseValue, tt.expOutOfRange) - } + t.Run(fmt.Sprint(i), func(t *testing.T) { + bv, oor := tt.f.baseValue(tt.op, tt.val) + if oor != tt.expOutOfRange || !reflect.DeepEqual(bv, tt.expBaseValue) { + t.Errorf("%s) baseValue(%s, %v)=(%v, %v), expected (%v, %v)", tt.f.Name, tt.op, tt.val, bv, oor, tt.expBaseValue, tt.expOutOfRange) + } + }) } }) diff --git a/field_test.go b/field_test.go index 72e700211..d73af7ada 100644 --- a/field_test.go +++ b/field_test.go @@ -16,6 +16,7 @@ package pilosa_test import ( "io/ioutil" + "math" "testing" "github.com/google/go-cmp/cmp" @@ -30,7 +31,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } @@ -63,7 +64,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index a67ea7875..d9b9e8128 100644 --- a/fragment.go +++ b/fragment.go @@ -1273,7 +1273,7 @@ func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin } } - // LTE predicateMin + // LTE predicateMax // If bit is zero then remove all set bits not in excluded bitmap. if bit2 == 0 { filter = filter.Difference(row.Difference(keep2)) @@ -2492,9 +2492,9 @@ func upgradeRoaringBSIv2(f *fragment, bitDepth uint) (string, error) { f.storage.ForEach(func(i uint64) { rowID, columnID := i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth) if rowID == uint64(bitDepth) { - other.Add(pos(bsiExistsBit, columnID)) // move exists bit to beginning + _, _ = other.Add(pos(bsiExistsBit, columnID)) // move exists bit to beginning } else { - other.Add(pos(rowID+bsiOffsetBit, columnID)) // move other bits up + _, _ = other.Add(pos(rowID+bsiOffsetBit, columnID)) // move other bits up } }) }() diff --git a/http/client_test.go b/http/client_test.go index 7fa398efa..2146d2ff7 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -705,7 +705,7 @@ func TestClient_ImportKeys(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) - field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(0, -100, 100)) if err != nil { t.Fatal(err) } @@ -790,7 +790,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(0, -100, 100)) if err != nil { t.Fatal(err) } @@ -933,7 +933,7 @@ func TestClient_ImportExistence(t *testing.T) { fldName := "fint" index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true}) - field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(0, -100, 100)) if err != nil { t.Fatal(err) } diff --git a/http/handler.go b/http/handler.go index 2271d5ec4..16a44c975 100644 --- a/http/handler.go +++ b/http/handler.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "io/ioutil" + "math" "net" "net/http" _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. @@ -764,7 +765,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { } else if v := req.Options.Min; v != nil { base = *v } - fos = append(fos, pilosa.OptFieldTypeInt(base)) + fos = append(fos, pilosa.OptFieldTypeInt(base, math.MinInt64, math.MaxInt64)) case pilosa.FieldTypeTime: fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView)) case pilosa.FieldTypeMutex: diff --git a/index_test.go b/index_test.go index 61f138de9..381a3adce 100644 --- a/index_test.go +++ b/index_test.go @@ -16,6 +16,7 @@ package pilosa_test import ( "io/ioutil" + "math" "reflect" "testing" @@ -93,7 +94,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with schema and verify it exists. - if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10)); err != nil { + if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type: %#v", f.Type()) diff --git a/internal/private.proto b/internal/private.proto index 01890652d..a327b835e 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -8,17 +8,16 @@ message IndexMeta { } message FieldOptions { - string Type = 8; + string Type = 8; string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; - bool Keys = 11; - bool NoStandardView = 12; - int64 Base = 13; - uint64 BitDepth = 14; - - int64 Min = 9 [deprecated=true]; - int64 Max = 10 [deprecated=true]; + int64 Min = 9; + int64 Max = 10; + bool Keys = 11; + bool NoStandardView = 12; + int64 Base = 13; + uint64 BitDepth = 14; } message ImportResponse { @@ -43,154 +42,154 @@ message Cache { } message MaxShards { - map Standard = 1; + map Standard = 1; } message CreateShardMessage { - string Index = 1; - string Field = 3; - uint64 Shard = 2; + string Index = 1; + string Field = 3; + uint64 Shard = 2; } message DeleteIndexMessage { - string Index = 1; + string Index = 1; } message CreateIndexMessage { - string Index = 1; - IndexMeta Meta = 2; + string Index = 1; + IndexMeta Meta = 2; } message CreateFieldMessage { - string Index = 1; - string Field = 2; - FieldOptions Meta = 3; + string Index = 1; + string Field = 2; + FieldOptions Meta = 3; } message DeleteFieldMessage { - string Index = 1; - string Field = 2; + string Index = 1; + string Field = 2; } message DeleteAvailableShardMessage { - string Index = 1; - string Field = 2; - uint64 ShardID = 3; + string Index = 1; + string Field = 2; + uint64 ShardID = 3; } message Field { - string Name = 1; - FieldOptions Meta = 2; - repeated string Views = 3; + string Name = 1; + FieldOptions Meta = 2; + repeated string Views = 3; } message Schema { - repeated Index Indexes = 1; + repeated Index Indexes = 1; } message Index { - string Name = 1; - repeated Field Fields = 4; + string Name = 1; + repeated Field Fields = 4; } message URI { - string Scheme = 1; - string Host = 2; - uint32 Port = 3; + string Scheme = 1; + string Host = 2; + uint32 Port = 3; } message Node { - string ID = 1; - URI URI = 2; - bool IsCoordinator = 3; - string State = 4; + string ID = 1; + URI URI = 2; + bool IsCoordinator = 3; + string State = 4; } message NodeStateMessage { - string NodeID = 1; - string State = 2; + string NodeID = 1; + string State = 2; } message NodeEventMessage { - uint32 Event = 1; - Node Node = 2; + uint32 Event = 1; + Node Node = 2; } message NodeStatus { - Node Node = 1; - Schema Schema = 3; - repeated IndexStatus Indexes = 4; + Node Node = 1; + Schema Schema = 3; + repeated IndexStatus Indexes = 4; } message IndexStatus { - string Name = 1; - repeated FieldStatus Fields = 2; + string Name = 1; + repeated FieldStatus Fields = 2; } message FieldStatus { - string Name = 1; - repeated uint64 AvailableShards = 2; + string Name = 1; + repeated uint64 AvailableShards = 2; } message ClusterStatus { - string ClusterID = 1; - string State = 2; - repeated Node Nodes = 3; + string ClusterID = 1; + string State = 2; + repeated Node Nodes = 3; } message BSIGroup { - string Name = 1; - string Type = 2; - int64 Min = 3; - int64 Max = 4; + string Name = 1; + string Type = 2; + int64 Min = 3; + int64 Max = 4; } message CreateViewMessage { - string Index = 1; - string Field = 2; - string View = 3; + string Index = 1; + string Field = 2; + string View = 3; } message DeleteViewMessage { - string Index = 1; - string Field = 2; - string View = 3; + string Index = 1; + string Field = 2; + string View = 3; } message ResizeInstruction { - int64 JobID = 1; - Node Node = 2; - Node Coordinator = 3; - repeated ResizeSource Sources = 4; - NodeStatus NodeStatus = 7; - ClusterStatus ClusterStatus = 6; + int64 JobID = 1; + Node Node = 2; + Node Coordinator = 3; + repeated ResizeSource Sources = 4; + NodeStatus NodeStatus = 7; + ClusterStatus ClusterStatus = 6; } message ResizeSource { - Node Node = 1; - string Index = 2; - string Field = 3; - string View = 4; - uint64 Shard = 5; + Node Node = 1; + string Index = 2; + string Field = 3; + string View = 4; + uint64 Shard = 5; } message ResizeInstructionComplete { - int64 JobID = 1; - Node Node = 2; - string Error = 3; + int64 JobID = 1; + Node Node = 2; + string Error = 3; } message SetCoordinatorMessage { - Node New = 1; + Node New = 1; } message UpdateCoordinatorMessage { - Node New = 1; + Node New = 1; } message Topology { - string ClusterID = 1; - repeated string NodeIDs = 2; + string ClusterID = 1; + repeated string NodeIDs = 2; } message RecalculateCaches {} diff --git a/server/handler_test.go b/server/handler_test.go index fc832269a..86398eb01 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -230,7 +230,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) { // Roaring import into a non-set field should fail. - if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil { + if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 0, 1)); err != nil { t.Fatal(err) } w := httptest.NewRecorder() From 40803372dd855fe0d71c3de32741fff26d155b39 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sun, 19 May 2019 16:05:22 -0600 Subject: [PATCH 58/73] Add min/max constraints; fix tests --- api_test.go | 2 +- encoding/proto/proto.go | 2 ++ executor_test.go | 42 ++++++++++++++++++++--------------------- field.go | 27 +++++++++++++++++--------- field_test.go | 34 +++++++++++++++++++++++++++++++-- http/client.go | 7 ++----- http/client_test.go | 14 +++++++------- http/handler.go | 36 +++++------------------------------ index_test.go | 3 +-- pilosa.go | 2 ++ server/handler_test.go | 2 +- 11 files changed, 92 insertions(+), 79 deletions(-) diff --git a/api_test.go b/api_test.go index 45aec0419..0cde55d59 100644 --- a/api_test.go +++ b/api_test.go @@ -199,7 +199,7 @@ func TestAPI_ImportValue(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) + _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatalf("creating field: %v", err) } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index f8a40badf..a04ab929c 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -530,6 +530,8 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, Base: o.Base, BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), diff --git a/executor_test.go b/executor_test.go index ae66c7ff6..60a92db2f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -770,7 +770,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) @@ -807,7 +807,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1215,7 +1215,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { t.Fatal(err) } @@ -1279,7 +1279,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1110, 1000)); err != nil { t.Fatal(err) } @@ -1372,15 +1372,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1430,15 +1430,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1832,19 +1832,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-900, 1000)); err != nil { t.Fatal(err) } @@ -1997,7 +1997,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -200)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2026,19 +2026,19 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, math.MinInt64, math.MaxInt64)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { t.Fatal(err) } @@ -2163,7 +2163,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -200)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -1200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2796,7 +2796,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) + _, err := index.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } diff --git a/field.go b/field.go index 9a33812c3..73c09f14c 100644 --- a/field.go +++ b/field.go @@ -130,16 +130,14 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { // OptFieldTypeInt is a functional option on FieldOptions // used to specify the field as being type `int` and to // provide any respective configuration values. -func OptFieldTypeInt(base, min, max int64) FieldOption { +func OptFieldTypeInt(min, max int64) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) } fo.Type = FieldTypeInt - fo.Base = base fo.Min = min fo.Max = max - fo.BitDepth = 1 return nil } } @@ -1016,6 +1014,10 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound + } else if value < bsig.Min { + return false, ErrBSIGroupValueTooLow + } else if value > bsig.Max { + return false, ErrBSIGroupValueTooHigh } // Determine base value to store. @@ -1259,6 +1261,11 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO dataByFragment := make(map[importKey]importValueData) for i := range columnIDs { columnID, value := columnIDs[i], values[i] + if value > bsig.Max { + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value) + } else if value < bsig.Min { + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value) + } // Attach value to each bsiGroup view. for _, name := range []string{viewName} { @@ -1345,16 +1352,14 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } type FieldOptions struct { Base int64 `json:"base,omitempty"` BitDepth uint `json:"bitDepth,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` Keys bool `json:"keys"` NoStandardView bool `json:"noStandardView,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` CacheType string `json:"cacheType,omitempty"` Type string `json:"type,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - - // Deprecated. Use base/bit depth. - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` } // applyDefaultOptions returns a new FieldOptions object @@ -1383,10 +1388,10 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, - Min: o.Min, - Max: o.Max, Base: o.Base, BitDepth: uint64(o.BitDepth), + Min: o.Min, + Max: o.Max, TimeQuantum: string(o.TimeQuantum), Keys: o.Keys, NoStandardView: o.NoStandardView, @@ -1415,11 +1420,15 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { Type string `json:"type"` Base int64 `json:"base"` BitDepth uint `json:"bitDepth"` + Min int64 `json:"min"` + Max int64 `json:"max"` Keys bool `json:"keys"` }{ o.Type, o.Base, o.BitDepth, + o.Min, + o.Max, o.Keys, }) case FieldTypeTime: diff --git a/field_test.go b/field_test.go index d73af7ada..c60c9a833 100644 --- a/field_test.go +++ b/field_test.go @@ -31,7 +31,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } @@ -64,7 +64,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, math.MinInt64, math.MaxInt64)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } @@ -107,6 +107,36 @@ func TestField_SetValue(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) + + t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh { + t.Fatalf("unexpected error: %s", err) + } + }) } func TestField_NameRestriction(t *testing.T) { diff --git a/http/client.go b/http/client.go index 6e3b2f274..74706588d 100644 --- a/http/client.go +++ b/http/client.go @@ -799,11 +799,8 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel fieldOpt.CacheType = &opt.CacheType fieldOpt.CacheSize = &opt.CacheSize } else if fieldOpt.Type == "int" { - if opt.Base == 0 && opt.Min != 0 { - opt.Base = opt.Min - } - fieldOpt.Base = &opt.Base - fieldOpt.BitDepth = &opt.BitDepth + fieldOpt.Min = &opt.Min + fieldOpt.Max = &opt.Max } else if fieldOpt.Type == "time" { fieldOpt.TimeQuantum = &opt.TimeQuantum } diff --git a/http/client_test.go b/http/client_test.go index 2146d2ff7..b4c32932d 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -705,7 +705,7 @@ func TestClient_ImportKeys(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) - field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(0, -100, 100)) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) if err != nil { t.Fatal(err) } @@ -790,7 +790,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(0, -100, 100)) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) if err != nil { t.Fatal(err) } @@ -832,8 +832,8 @@ func TestClient_ImportValue(t *testing.T) { if err != nil { t.Fatal(err) } - if min != -100 || cnt != 0 { - t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + if min != 0 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=0, cnt=0", min, cnt) } // Verify Max. @@ -871,8 +871,8 @@ func TestClient_ImportValue(t *testing.T) { if err != nil { t.Fatal(err) } - if min != -100 || cnt != 0 { - t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + if min != 0 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=0, cnt=0", min, cnt) } // Verify Max. @@ -933,7 +933,7 @@ func TestClient_ImportExistence(t *testing.T) { fldName := "fint" index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true}) - field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(0, -100, 100)) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) if err != nil { t.Fatal(err) } diff --git a/http/handler.go b/http/handler.go index 16a44c975..0ce3ee6cd 100644 --- a/http/handler.go +++ b/http/handler.go @@ -759,13 +759,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { case pilosa.FieldTypeSet: fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeInt: - var base int64 - if v := req.Options.Base; v != nil { - base = *v - } else if v := req.Options.Min; v != nil { - base = *v - } - fos = append(fos, pilosa.OptFieldTypeInt(base, math.MinInt64, math.MaxInt64)) + fos = append(fos, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) case pilosa.FieldTypeTime: fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView)) case pilosa.FieldTypeMutex: @@ -793,15 +787,11 @@ type fieldOptions struct { Type string `json:"type,omitempty"` CacheType *string `json:"cacheType,omitempty"` CacheSize *uint32 `json:"cacheSize,omitempty"` - Base *int64 `json:"base,omitempty"` - BitDepth *uint `json:"bitDepth,omitempty"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` Keys *bool `json:"keys,omitempty"` NoStandardView bool `json:"noStandardView,omitempty"` - - // Deprecated. Use base/bit depth. - Min *int64 `json:"min,omitempty"` - Max *int64 `json:"max,omitempty"` } func (o *fieldOptions) validate() error { @@ -823,11 +813,7 @@ func (o *fieldOptions) validate() error { if o.CacheSize == nil { o.CacheSize = &defaultCacheSize } - if o.Base != nil { - return pilosa.NewBadRequestError(errors.New("base does not apply to field type set")) - } else if o.BitDepth != nil { - return pilosa.NewBadRequestError(errors.New("bit depth does not apply to field type set")) - } else if o.Min != nil { + if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) } else if o.Max != nil { return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) @@ -847,10 +833,6 @@ func (o *fieldOptions) validate() error { return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) } else if o.CacheSize != nil { return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) - } else if o.Base != nil { - return pilosa.NewBadRequestError(errors.New("base does not apply to field type time")) - } else if o.BitDepth != nil { - return pilosa.NewBadRequestError(errors.New("bitDepth does not apply to field type time")) } else if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) } else if o.Max != nil { @@ -865,11 +847,7 @@ func (o *fieldOptions) validate() error { if o.CacheSize == nil { o.CacheSize = &defaultCacheSize } - if o.Base != nil { - return pilosa.NewBadRequestError(errors.New("base does not apply to field type mutex")) - } else if o.BitDepth != nil { - return pilosa.NewBadRequestError(errors.New("bitDepth does not apply to field type mutex")) - } else if o.Min != nil { + if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) } else if o.Max != nil { return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) @@ -881,10 +859,6 @@ func (o *fieldOptions) validate() error { return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) } else if o.CacheSize != nil { return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) - } else if o.Base != nil { - return pilosa.NewBadRequestError(errors.New("base does not apply to field type bool")) - } else if o.BitDepth != nil { - return pilosa.NewBadRequestError(errors.New("bitDepth does not apply to field type bool")) } else if o.Min != nil { return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) } else if o.Max != nil { diff --git a/index_test.go b/index_test.go index 381a3adce..ee5625cf3 100644 --- a/index_test.go +++ b/index_test.go @@ -16,7 +16,6 @@ package pilosa_test import ( "io/ioutil" - "math" "reflect" "testing" @@ -94,7 +93,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with schema and verify it exists. - if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, math.MinInt64, math.MaxInt64)); err != nil { + if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type: %#v", f.Type()) diff --git a/pilosa.go b/pilosa.go index 41434f52f..42ab3d3c1 100644 --- a/pilosa.go +++ b/pilosa.go @@ -40,6 +40,8 @@ var ( ErrInvalidBSIGroupType = errors.New("invalid bsigroup type") ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range") ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type") + ErrBSIGroupValueTooLow = errors.New("bsigroup value too low") + ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high") ErrInvalidRangeOperation = errors.New("invalid range operation") ErrInvalidBetweenValue = errors.New("invalid value for between operation") diff --git a/server/handler_test.go b/server/handler_test.go index 86398eb01..fc832269a 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -230,7 +230,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) { // Roaring import into a non-set field should fail. - if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 0, 1)); err != nil { + if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil { t.Fatal(err) } w := httptest.NewRecorder() From dd4227f5e365b1fe2351f7ee85d9a09f3a01e850 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 13 May 2019 13:51:54 -0600 Subject: [PATCH 59/73] Improve TopN() errors This commit improves field not found, integer field, and cache errors for the `TopN()` command. --- executor.go | 15 ++++++++---- executor_test.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/executor.go b/executor.go index 07a83496d..607840a42 100644 --- a/executor.go +++ b/executor.go @@ -766,11 +766,14 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard") defer span.Finish() - field, _ := c.Args["_field"].(string) + fieldName, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNShard: %v", err) + } else if f := e.Holder.Field(index, fieldName); f != nil && f.Type() == FieldTypeInt { + return nil, fmt.Errorf("cannot compute TopN() on integer field: %q", fieldName) } + attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { @@ -799,13 +802,15 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca } // Set default field. - if field == "" { - field = defaultField + if fieldName == "" { + fieldName = defaultField } - f := e.Holder.fragment(index, field, viewStandard, shard) + f := e.Holder.fragment(index, fieldName, viewStandard, shard) if f == nil { return nil, nil + } else if f.CacheType == CacheTypeNone { + return nil, fmt.Errorf("cannot compute TopN(), field has no cache: %q", fieldName) } if minThreshold == 0 { @@ -2623,7 +2628,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if fieldName := callArgString(call, "_field"); fieldName != "" { field := idx.Field(fieldName) if field == nil { - return nil, ErrFieldNotFound + return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { other := make([]Pair, len(result)) diff --git a/executor_test.go b/executor_test.go index 60a92db2f..3565c40fd 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1058,6 +1058,65 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(diff) } }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set data on the "f" field. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(0, f=1) + `}); err != nil { + t.Fatal(err) + } else if err := c[0].RecalculateCaches(); err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + // Attempt to query the "g" field. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(g, n=2)`}); err == nil || err.Error() != `executing: field "g" not found` { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("ErrBSIField", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create BSI "f" field. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("ErrCacheNone", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0)); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(0, f=1) + `}); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` { + t.Fatalf("unexpected error: %v", err) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { From 62e2b16b8861cf54c24fa918879e8ce25893b486 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 27 May 2019 15:49:10 +0300 Subject: [PATCH 60/73] set defaults for int field min and max --- http/handler.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index 0ce3ee6cd..0d4691d78 100644 --- a/http/handler.go +++ b/http/handler.go @@ -759,7 +759,13 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { case pilosa.FieldTypeSet: fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeInt: - fos = append(fos, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) + if req.Options.Min == nil { + *req.Options.Min = math.MinInt64 + } + if req.Options.Max == nil { + *req.Options.Max = math.MaxInt64 + } + fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) case pilosa.FieldTypeTime: fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView)) case pilosa.FieldTypeMutex: From b5e4b90438c3406c4f2382ebd3400c52e1c3a56a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 27 May 2019 17:43:53 +0300 Subject: [PATCH 61/73] fixes #1977 --- field.go | 3 +++ http/handler.go | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/field.go b/field.go index 73c09f14c..3e71af589 100644 --- a/field.go +++ b/field.go @@ -135,6 +135,9 @@ func OptFieldTypeInt(min, max int64) FieldOption { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) } + if min > max { + return errors.New("int field min cannot be greater than max") + } fo.Type = FieldTypeInt fo.Min = min fo.Max = max diff --git a/http/handler.go b/http/handler.go index 0d4691d78..1334af4d5 100644 --- a/http/handler.go +++ b/http/handler.go @@ -760,10 +760,12 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeInt: if req.Options.Min == nil { - *req.Options.Min = math.MinInt64 + min := int64(math.MinInt64) + req.Options.Min = &min } if req.Options.Max == nil { - *req.Options.Max = math.MaxInt64 + max := int64(math.MaxInt64) + req.Options.Max = &max } fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) case pilosa.FieldTypeTime: From 5f4c5d4d35a7d0eeccc27f2e5927caa0ed3f6406 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 28 May 2019 13:54:12 +0300 Subject: [PATCH 62/73] added test for 1977 fix --- api.go | 2 +- http/handler.go | 6 +++ server/handler_test.go | 116 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 2b96d023c..bdaa35b03 100644 --- a/api.go +++ b/api.go @@ -213,7 +213,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str for _, opt := range opts { err := opt(&fo) if err != nil { - return nil, errors.Wrap(err, "applying option") + return nil, NewBadRequestError(errors.Wrap(err, "applying option")) } } diff --git a/http/handler.go b/http/handler.go index 1334af4d5..0d52a270b 100644 --- a/http/handler.go +++ b/http/handler.go @@ -782,6 +782,12 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { } _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + if err != nil { + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } resp.write(w, err) } diff --git a/server/handler_test.go b/server/handler_test.go index fc832269a..dc4f571d5 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "io/ioutil" + "math" gohttp "net/http" "net/http/httptest" "reflect" @@ -542,6 +543,104 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("Query int field unbounded", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int"}}`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + rsp := getSchemaResponse{} + if err := json.Unmarshal([]byte(w.Body.String()), &rsp); err != nil { + t.Fatalf("json decode: %s", err) + } + field := rsp.findField("i0", fieldName) + if field == nil { + t.Fatalf("field not found: %s", fieldName) + } + if math.MinInt64 != field.Options.Min { + t.Fatalf("field min %d != %d", math.MinInt64, field.Options.Min) + } + if math.MaxInt64 != field.Options.Max { + t.Fatalf("field max %d != %d", math.MaxInt64, field.Options.Max) + } + }) + + t.Run("Query int field unbounded min", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound-min" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int", "max": 10}}`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + rsp := getSchemaResponse{} + if err := json.Unmarshal([]byte(w.Body.String()), &rsp); err != nil { + t.Fatalf("json decode: %s", err) + } + field := rsp.findField("i0", fieldName) + if field == nil { + t.Fatalf("field not found: %s", fieldName) + } + if math.MinInt64 != field.Options.Min { + t.Fatalf("field min %d != %d", math.MinInt64, field.Options.Min) + } + if 10 != field.Options.Max { + t.Fatalf("field max %d != %d", 10, field.Options.Max) + } + }) + + t.Run("Query int field unbounded max", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound-max" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int", "min": -10}}`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + rsp := getSchemaResponse{} + if err := json.Unmarshal([]byte(w.Body.String()), &rsp); err != nil { + t.Fatalf("json decode: %s", err) + } + field := rsp.findField("i0", fieldName) + if field == nil { + t.Fatalf("field not found: %s", fieldName) + } + if -10 != field.Options.Min { + t.Fatalf("field min %d != %d", 10, field.Options.Min) + } + if math.MaxInt64 != field.Options.Max { + t.Fatalf("field max %d != %d", math.MaxInt64, field.Options.Max) + } + }) + + t.Run("Query int field min > max return 400", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound-err" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int", "min": 10, "max": -10}}`))) + fmt.Println("body", w.Body.String()) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) + t.Run("Method not allowed", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil)) @@ -999,3 +1098,20 @@ func mustJSONDecodeSlice(t *testing.T, r io.Reader) (ret []interface{}) { } return ret } + +type getSchemaResponse struct { + Indexes []*pilosa.IndexInfo `json:"indexes"` +} + +func (r getSchemaResponse) findField(indexName, fieldName string) *pilosa.FieldInfo { + for _, index := range r.Indexes { + if index.Name == indexName { + for _, field := range index.Fields { + if field.Name == fieldName { + return field + } + } + } + } + return nil +} From 5e102154caf5efd3395282a5cdeb1d3ac8505fa0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 28 May 2019 14:12:04 +0300 Subject: [PATCH 63/73] make linter happy --- http/handler.go | 8 +++----- server/handler_test.go | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/http/handler.go b/http/handler.go index 0d52a270b..8e932271a 100644 --- a/http/handler.go +++ b/http/handler.go @@ -782,11 +782,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { } _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) - if err != nil { - if _, ok := err.(pilosa.BadRequestError); ok { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + return } resp.write(w, err) } diff --git a/server/handler_test.go b/server/handler_test.go index dc4f571d5..c830c94d7 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -557,7 +557,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } rsp := getSchemaResponse{} - if err := json.Unmarshal([]byte(w.Body.String()), &rsp); err != nil { + if err := json.Unmarshal(w.Body.Bytes(), &rsp); err != nil { t.Fatalf("json decode: %s", err) } field := rsp.findField("i0", fieldName) @@ -586,7 +586,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } rsp := getSchemaResponse{} - if err := json.Unmarshal([]byte(w.Body.String()), &rsp); err != nil { + if err := json.Unmarshal(w.Body.Bytes(), &rsp); err != nil { t.Fatalf("json decode: %s", err) } field := rsp.findField("i0", fieldName) @@ -615,7 +615,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } rsp := getSchemaResponse{} - if err := json.Unmarshal([]byte(w.Body.String()), &rsp); err != nil { + if err := json.Unmarshal(w.Body.Bytes(), &rsp); err != nil { t.Fatalf("json decode: %s", err) } field := rsp.findField("i0", fieldName) From c8a3dc8c185ffe3961f977e6de503b910493ec96 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 28 May 2019 14:25:46 +0300 Subject: [PATCH 64/73] fix int min max test for 32bit --- server/handler_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index c830c94d7..c4e98d7f3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -565,10 +565,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("field not found: %s", fieldName) } if math.MinInt64 != field.Options.Min { - t.Fatalf("field min %d != %d", math.MinInt64, field.Options.Min) + t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min) } if math.MaxInt64 != field.Options.Max { - t.Fatalf("field max %d != %d", math.MaxInt64, field.Options.Max) + t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max) } }) @@ -594,7 +594,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("field not found: %s", fieldName) } if math.MinInt64 != field.Options.Min { - t.Fatalf("field min %d != %d", math.MinInt64, field.Options.Min) + t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min) } if 10 != field.Options.Max { t.Fatalf("field max %d != %d", 10, field.Options.Max) @@ -626,7 +626,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("field min %d != %d", 10, field.Options.Min) } if math.MaxInt64 != field.Options.Max { - t.Fatalf("field max %d != %d", math.MaxInt64, field.Options.Max) + t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max) } }) From c587dbc94dcef89dd68d4c6b8e70ebf9d0cb7df8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 13 May 2019 12:36:03 -0500 Subject: [PATCH 65/73] drop enterprise/b We added the containers_btree implementation to roaring/, which makes it silly to keep this one. Also, this one is the only reason that container.Mapped needed to be exported. --- enterprise/b/btree.go | 953 ------------------------------- enterprise/b/containers_btree.go | 212 ------- enterprise/enterprise.go | 10 - 3 files changed, 1175 deletions(-) delete mode 100644 enterprise/b/btree.go delete mode 100644 enterprise/b/containers_btree.go diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go deleted file mode 100644 index 2fa5c24e8..000000000 --- a/enterprise/b/btree.go +++ /dev/null @@ -1,953 +0,0 @@ -// This file is a modified redistribution of b (https://github.com/cznic/b), -// which is governed by the following license notice: -// -// Copyright (c) 2014 The b Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the names of the authors nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package b - -import ( - "io" - "sync" - - "github.com/pilosa/pilosa/roaring" -) - -const ( - // kx must be >= 2 - kx = 128 //TODO benchmark tune this number if using custom key/value type(s). - // kd must be >= 1 - kd = 128 //TODO benchmark tune this number if using custom key/value type(s). -) - -var ( - btDPool = sync.Pool{New: func() interface{} { return &d{} }} - btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}} - btTPool = btTpool{sync.Pool{New: func() interface{} { return &tree{} }}} - btXPool = sync.Pool{New: func() interface{} { return &x{} }} -) - -type btTpool struct{ sync.Pool } - -func (p *btTpool) get(cmp Cmp) *tree { - x := p.Get().(*tree) - x.cmp = cmp - return x -} - -type btEpool struct{ sync.Pool } - -func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *tree, ver int64) *enumerator { - x := p.Get().(*enumerator) - x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver - return x -} - -type ( - // Cmp compares a and b. Return value is: - // - // < 0 if a < b - // 0 if a == b - // > 0 if a > b - // - Cmp func(a, b uint64) int64 - - d struct { // data page - c int - d [2*kd + 1]de - n *d - p *d - } - - de struct { // d element - k uint64 - v *roaring.Container - } - - // enumerator captures the state of enumerating a tree. It is returned - // from the Seek* methods. The enumerator is aware of any mutations - // made to the tree in the process of enumerating it and automatically - // resumes the enumeration at the proper key, if possible. - // - // However, once an enumerator returns io.EOF to signal "no more - // items", it does no more attempt to "resync" on tree mutation(s). In - // other words, io.EOF from an enumerator is "sticky" (idempotent). - enumerator struct { - err error - hit bool - i int - k uint64 - q *d - t *tree - ver int64 - } - - // tree is a B+tree. - tree struct { - c int - cmp Cmp - first *d - last *d - r interface{} - ver int64 - } - - xe struct { // x element - ch interface{} - k uint64 - } - - x struct { // index page - c int - x [2*kx + 2]xe - } -) - -var ( // R/O zero values - zd d - zde de - ze enumerator - zk uint64 - zt tree - zx x - zxe xe -) - -func clr(q interface{}) { - switch x := q.(type) { - case *x: - for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn - clr(x.x[i].ch) - } - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } -} - -// -------------------------------------------------------------------------- x - -func newX(ch0 interface{}) *x { - r := btXPool.Get().(*x) - r.x[0].ch = ch0 - return r -} - -func (q *x) extract(i int) { - q.c-- - if i < q.c { - copy(q.x[i:], q.x[i+1:q.c+1]) - q.x[q.c].ch = q.x[q.c+1].ch - q.x[q.c].k = zk // GC - q.x[q.c+1] = zxe // GC - } -} - -func (q *x) insert(i int, k uint64, ch interface{}) *x { - c := q.c - if i < c { - q.x[c+1].ch = q.x[c].ch - copy(q.x[i+2:], q.x[i+1:c]) - q.x[i+1].k = q.x[i].k - } - c++ - q.c = c - q.x[i].k = k - q.x[i+1].ch = ch - return q -} - -func (q *x) siblings(i int) (l, r *d) { - if i >= 0 { - if i > 0 { - l = q.x[i-1].ch.(*d) - } - if i < q.c { - r = q.x[i+1].ch.(*d) - } - } - return l, r -} - -// -------------------------------------------------------------------------- d - -func (l *d) mvL(r *d, c int) { - copy(l.d[l.c:], r.d[:c]) - copy(r.d[:], r.d[c:r.c]) - // Zero out the de's here to prevent reading bad data - // and to avoid creating non-collectible (GC) references. - for i := 1; i < c; i++ { - r.d[r.c-i] = zde - } - l.c += c - r.c -= c -} - -func (l *d) mvR(r *d, c int) { - copy(r.d[c:], r.d[:r.c]) - copy(r.d[:c], l.d[l.c-c:]) - // Zero out the de's here to prevent reading bad data - // and to avoid creating non-collectible (GC) references. - for i := 1; i < c; i++ { - l.d[l.c-c+i] = zde - } - r.c += c - l.c -= c -} - -// ----------------------------------------------------------------------- Tree - -// treeNew returns a newly created, empty Tree. The compare function is used -// for key collation. -func treeNew(cmp Cmp) *tree { - return btTPool.get(cmp) -} - -// Clear removes all K/V pairs from the tree. -func (t *tree) Clear() { - if t.r == nil { - return - } - - clr(t.r) - t.c, t.first, t.last, t.r = 0, nil, nil, nil - t.ver++ -} - -// Close performs Clear and recycles t to a pool for possible later reuse. No -// references to t should exist or such references must not be used afterwards. -func (t *tree) Close() { - t.Clear() - *t = zt - btTPool.Put(t) -} - -func (t *tree) cat(p *x, q, r *d, pi int) { - t.ver++ - q.mvL(r, r.c) - if r.n != nil { - r.n.p = q - } else { - t.last = q - } - q.n = r.n - *r = zd - btDPool.Put(r) - if p.c > 1 { - p.extract(pi) - p.x[pi].ch = q - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -func (t *tree) catX(p, q, r *x, pi int) { - t.ver++ - q.x[q.c].k = p.x[pi].k - copy(q.x[q.c+1:], r.x[:r.c]) - q.c += r.c + 1 - q.x[q.c].ch = r.x[r.c].ch - *r = zx - btXPool.Put(r) - if p.c > 1 { - p.c-- - pc := p.c - if pi < pc { - p.x[pi].k = p.x[pi+1].k - copy(p.x[pi+1:], p.x[pi+2:pc+1]) - p.x[pc].ch = p.x[pc+1].ch - p.x[pc].k = zk // GC - p.x[pc+1].ch = nil // GC - } - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -// Delete removes the k's KV pair, if it exists, in which case Delete returns -// true. -func (t *tree) Delete(k uint64) (ok bool) { - pi := -1 - var p *x - q := t.r - if q == nil { - return false - } - - for { - var i int - i, ok = t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[pi].ch - continue - case *d: - t.extract(x, i) - if x.c >= kd { - return true - } - - if q != t.r { - t.underflow(p, x, pi) - } else if t.c == 0 { - t.Clear() - } - return true - } - } - - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - return false - } - } -} - -func (t *tree) extract(q *d, i int) { // (r *container) { - t.ver++ - //r = q.d[i].v // prepared for Extract - q.c-- - if i < q.c { - copy(q.d[i:], q.d[i+1:q.c+1]) - } - q.d[q.c] = zde // GC - t.c-- -} - -func (t *tree) find(q interface{}, k uint64) (i int, ok bool) { - var mk uint64 - l := 0 - switch x := q.(type) { - case *x: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.x[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - case *d: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.d[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - } - return l, false -} - -// First returns the first item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *tree) First() (k uint64, v *roaring.Container) { - if q := t.first; q != nil { - q := &q.d[0] - k, v = q.k, q.v - } - return k, v -} - -// Get returns the value associated with k and true if it exists. Otherwise Get -// returns (zero-value, false). -func (t *tree) Get(k uint64) (v *roaring.Container, ok bool) { - q := t.r - if q == nil { - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return x.d[i].v, true - } - } - switch x := q.(type) { - case *x: - q = x.x[i].ch - default: - return - } - } -} - -func (t *tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { - t.ver++ - c := q.c - if i < c { - copy(q.d[i+1:], q.d[i:c]) - } - c++ - q.c = c - q.d[i].k, q.d[i].v = k, v - t.c++ - return q -} - -// Last returns the last item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *tree) Last() (k uint64, v *roaring.Container) { - if q := t.last; q != nil { - q := &q.d[q.c-1] - k, v = q.k, q.v - } - return k, v -} - -// Len returns the number of items in the tree. -func (t *tree) Len() int { - return t.c -} - -func (t *tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { - t.ver++ - l, r := p.siblings(pi) - - // s is the number of items to shift out of the full data container to - // allow for the new data item. This logic shifts by half the available - // space plus one. In the case where the new item is to be inserted within - // the calculated shift space, then s is reduced to include only the - // data items up to the index of the new data item. - if l != nil && l.c < 2*kd && i != 0 { - s := (2*kd-l.c)/2 + 1 // half plus one - //s := 2*kd - l.c // all available - if i < s { - s = i - } - l.mvL(q, s) - t.insert(q, i-s, k, v) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && r.c < 2*kd { - if i < 2*kd { - s := (2*kd-r.c)/2 + 1 // half plus one - //s := 2*kd - r.c // all available - if 2*kd-i < s { - s = 2*kd - i - } - q.mvR(r, s) - t.insert(q, i, k, v) - p.x[pi].k = r.d[0].k - return - } - - t.insert(r, 0, k, v) - p.x[pi].k = k - return - } - - t.split(p, q, pi, i, k, v) -} - -// Seek returns an Enumerator positioned on an item such that k >= item's key. -// ok reports if k == item.key The Enumerator's position is possibly after the -// last item in the tree. -func (t *tree) Seek(k uint64) (e *enumerator, ok bool) { - q := t.r - if q == nil { - e = btEPool.get(nil, false, 0, k, nil, t, t.ver) - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), true - } - } - - switch x := q.(type) { - case *x: - q = x.x[i].ch - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), false - } - } -} - -// SeekFirst returns an enumerator positioned on the first KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *tree) SeekFirst() (e *enumerator, err error) { - q := t.first - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil -} - -// SeekLast returns an enumerator positioned on the last KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *tree) SeekLast() (e *enumerator, err error) { - q := t.last - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil -} - -// Set sets the value associated with k. -func (t *tree) Set(k uint64, v *roaring.Container) { - //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) - //defer func() { - // dbg("--- POST\n%s\n====\n", t.dump()) - //}() - - pi := -1 - var p *x - q := t.r - if q == nil { - z := t.insert(btDPool.Get().(*d), 0, k, v) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - i++ - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - continue - case *d: - x.d[i].v = v - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - switch { - case x.c < 2*kd: - t.insert(x, i, k, v) - default: - t.overflow(p, x, pi, i, k, v) - } - return - } - } -} - -// Put combines Get and Set in a more efficient way where the tree is walked -// only once. The upd(ater) receives (old-value, true) if a KV pair for k -// exists or (zero-value, false) otherwise. It can then return a (new-value, -// true) to create or overwrite the existing value in the KV pair, or -// (whatever, false) if it decides not to create or not to update the value of -// the KV pair. -// -// tree.Set(k, v) call conceptually equals calling -// -// tree.Put(k, func(uint64, bool){ return v, true }) -// -// modulo the differing return values. -func (t *tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { - pi := -1 - var p *x - q := t.r - var newV *roaring.Container - if q == nil { - // new KV pair in empty tree - newV, written = upd(newV, false) - if !written { - return - } - - z := t.insert(btDPool.Get().(*d), 0, k, newV) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - i++ - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - continue - case *d: - oldV = x.d[i].v - newV, written = upd(oldV, true) - if !written { - return - } - - x.d[i].v = newV - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: // new KV pair - newV, written = upd(newV, false) - if !written { - return - } - - switch { - case x.c < 2*kd: - t.insert(x, i, k, newV) - default: - t.overflow(p, x, pi, i, k, newV) - } - return - } - } -} - -func (t *tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { - t.ver++ - r := btDPool.Get().(*d) - if q.n != nil { - r.n = q.n - r.n.p = r - } else { - t.last = r - } - q.n = r - r.p = q - - copy(r.d[:], q.d[kd:2*kd]) - for i := range q.d[kd:] { - q.d[kd+i] = zde - } - q.c = kd - r.c = kd - var done bool - if i > kd { - done = true - t.insert(r, i-kd, k, v) - } - if pi >= 0 { - p.insert(pi, r.d[0].k, r) - } else { - t.r = newX(q).insert(0, r.d[0].k, r) - } - if done { - return - } - - t.insert(q, i, k, v) -} - -func (t *tree) splitX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - r := btXPool.Get().(*x) - copy(r.x[:], q.x[kx+1:]) - q.c = kx - r.c = kx - if pi >= 0 { - p.insert(pi, q.x[kx].k, r) - } else { - t.r = newX(q).insert(0, q.x[kx].k, r) - } - - q.x[kx].k = zk - for i := range q.x[kx+1:] { - q.x[kx+i+1] = zxe - } - if i > kx { - q = r - i -= kx + 1 - } - - return q, i -} - -func (t *tree) underflow(p *x, q *d, pi int) { - t.ver++ - l, r := p.siblings(pi) - - if l != nil && l.c+q.c >= 2*kd { - l.mvR(q, 1) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && q.c+r.c >= 2*kd { - q.mvL(r, 1) - p.x[pi].k = r.d[0].k - r.d[r.c] = zde // GC - return - } - - if l != nil { - t.cat(p, l, q, pi-1) - return - } - - t.cat(p, q, r, pi) -} - -func (t *tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - var l, r *x - - if pi >= 0 { - if pi > 0 { - l = p.x[pi-1].ch.(*x) - } - if pi < p.c { - r = p.x[pi+1].ch.(*x) - } - } - - if l != nil && l.c > kx { - q.x[q.c+1].ch = q.x[q.c].ch - copy(q.x[1:], q.x[:q.c]) - q.x[0].ch = l.x[l.c].ch - q.x[0].k = p.x[pi-1].k - q.c++ - i++ - l.c-- - p.x[pi-1].k = l.x[l.c].k - return q, i - } - - if r != nil && r.c > kx { - q.x[q.c].k = p.x[pi].k - q.c++ - q.x[q.c].ch = r.x[0].ch - p.x[pi].k = r.x[0].k - copy(r.x[:], r.x[1:r.c]) - r.c-- - rc := r.c - r.x[rc].ch = r.x[rc+1].ch - r.x[rc].k = zk - r.x[rc+1].ch = nil - return q, i - } - - if l != nil { - i += l.c + 1 - t.catX(p, l, q, pi-1) - q = l - return q, i - } - - t.catX(p, q, r, pi) - return q, i -} - -// ----------------------------------------------------------------- Enumerator - -// Close recycles e to a pool for possible later reuse. No references to e -// should exist or such references must not be used afterwards. -func (e *enumerator) Close() { - *e = ze - btEPool.Put(e) -} - -// Next returns the currently enumerated item, if it exists and moves to the -// next item in the key collation order. If there is no item to return, err == -// io.EOF is returned. -func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) { - if err = e.err; err != nil { - return 0, nil, err - } - - if e.ver != e.t.ver { - f, _ := e.t.Seek(e.k) - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return 0, nil, err - } - - if e.i >= e.q.c { - if err = e.next(); err != nil { - return 0, nil, err - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, true - _ = e.next() - return k, v, nil -} - -func (e *enumerator) next() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i < e.q.c-1: - e.i++ - default: - if e.q, e.i = e.q.n, 0; e.q == nil { - e.err = io.EOF - } - } - return e.err -} - -// Prev returns the currently enumerated item, if it exists and moves to the -// previous item in the key collation order. If there is no item to return, err -// == io.EOF is returned. -func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) { - if err = e.err; err != nil { - return 0, nil, err - } - - if e.ver != e.t.ver { - f, _ := e.t.Seek(e.k) - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return 0, nil, err - } - - if !e.hit { - // move to previous because Seek overshoots if there's no hit - if err = e.prev(); err != nil { - return 0, nil, err - } - } - - if e.i >= e.q.c { - if err = e.prev(); err != nil { - return 0, nil, err - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, true - _ = e.prev() - return k, v, err -} - -func (e *enumerator) prev() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i > 0: - e.i-- - default: - if e.q = e.q.p; e.q == nil { - e.err = io.EOF - break - } - - e.i = e.q.c - 1 - } - return e.err -} diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go deleted file mode 100644 index a051d7cde..000000000 --- a/enterprise/b/containers_btree.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) 2018 Pilosa Corp. All rights reserved. -// -// This file is part of Pilosa Enterprise Edition. -// -// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Pilosa Enterprise Edition is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with Pilosa Enterprise Edition. If not, see . - -package b - -import ( - "io" - - "github.com/pilosa/pilosa/roaring" -) - -func cmp(a, b uint64) int64 { - return int64(a - b) -} - -type bTreeContainers struct { - tree *tree - - lastKey uint64 - lastContainer *roaring.Container -} - -func newBTreeContainers() *bTreeContainers { - return &bTreeContainers{ - tree: treeNew(cmp), - } -} - -func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { - b := &roaring.Bitmap{ - Containers: newBTreeContainers(), - } - // TODO: there's no way to report an error here - _, _ = b.Add(a...) - return b -} - -func (btc *bTreeContainers) Get(key uint64) *roaring.Container { - // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { - return btc.lastContainer - } - - var c *roaring.Container - el, ok := btc.tree.Get(key) - if ok { - c = el - btc.lastKey = key - btc.lastContainer = c - } - return c -} - -func (btc *bTreeContainers) Put(key uint64, c *roaring.Container) { - // If a mapped container is added to the tree, reset the - // lastContainer cache so that the cache is not pointing - // at a read-only mmap. - if c.Mapped() { - btc.lastContainer = nil - } - btc.tree.Set(key, c) -} - -func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) { - // update the existing container - if exists { - oldV.Update(u.containerType, u.n, u.mapped) - return oldV, false - } - cont := roaring.NewContainer() - cont.Update(u.containerType, u.n, u.mapped) - return cont, true -} - -// this struct is added to prevent the closure locals from being escaped out to the heap -type updater struct { - key uint64 - n int32 - containerType byte - mapped bool -} - -func (btc *bTreeContainers) PutContainerValues(key uint64, typ byte, n int, mapped bool) { - a := updater{key, int32(n), typ, mapped} - btc.tree.Put(key, a.update) -} - -func (btc *bTreeContainers) Remove(key uint64) { - btc.tree.Delete(key) -} - -func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container { - // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { - return btc.lastContainer - } - - btc.lastKey = key - v, ok := btc.tree.Get(key) - if !ok { - cont := roaring.NewContainerArray(nil) - btc.tree.Set(key, cont) - btc.lastContainer = cont - return cont - } - - btc.lastContainer = v - return btc.lastContainer -} - -func (btc *bTreeContainers) Count() (n uint64) { - e, _ := btc.tree.Seek(0) - _, c, err := e.Next() - for err != io.EOF { - n += uint64(c.N()) - _, c, err = e.Next() - } - return n -} - -func (btc *bTreeContainers) Clone() roaring.Containers { - nbtc := newBTreeContainers() - - itr, err := btc.tree.SeekFirst() - if err == io.EOF { - return nbtc - } - for { - k, v, err := itr.Next() - if err == io.EOF { - break - } - nbtc.tree.Set(k, v.Clone()) - } - return nbtc -} - -func (btc *bTreeContainers) Last() (key uint64, c *roaring.Container) { - if btc.tree.Len() == 0 { - return 0, nil - } - k, v := btc.tree.Last() - return k, v -} - -func (btc *bTreeContainers) Size() int { - return btc.tree.Len() -} - -func (btc *bTreeContainers) Reset() { - btc.tree = treeNew(cmp) - btc.lastKey = 0 - btc.lastContainer = nil -} - -func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { - e, ok := btc.tree.Seek(key) - if ok { - found = true - } - - return &btcIterator{ - e: e, - }, found -} - -func (btc *bTreeContainers) Repair() { - e, _ := btc.tree.Seek(0) - _, c, err := e.Next() - for err != io.EOF { - c.Repair() - _, c, err = e.Next() - } -} - -type btcIterator struct { - e *enumerator - key uint64 - val *roaring.Container -} - -func (i *btcIterator) Next() bool { - - k, v, err := i.e.Next() - if err == io.EOF { - return false - } - i.key = k - i.val = v - return true -} - -func (i *btcIterator) Value() (uint64, *roaring.Container) { - if i.val == nil { - return 0, nil - } - return i.key, i.val -} diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go index a75223801..f3a897551 100644 --- a/enterprise/enterprise.go +++ b/enterprise/enterprise.go @@ -21,13 +21,3 @@ // "ENTERPRISE=1 make install". These features were dual-licensed separately // from Pilosa community edition under the AGPL and Pilosa's commercial license. package enterprise - -import ( - "github.com/pilosa/pilosa/enterprise/b" - "github.com/pilosa/pilosa/roaring" -) - -func init() { // nolint: gochecknoinits - // Replace Bitmap constructor with B+Tree implementation - roaring.NewFileBitmap = b.NewBTreeBitmap -} From 63120e37156b093de78cfdbb3b0d092aebc05d3e Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 22 May 2019 14:52:15 -0500 Subject: [PATCH 66/73] rename slice containers source file descriptively The containers.go file contains one of two Containers implementations, it should have a name reflecting this. --- roaring/{containers.go => containers_slice.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roaring/{containers.go => containers_slice.go} (100%) diff --git a/roaring/containers.go b/roaring/containers_slice.go similarity index 100% rename from roaring/containers.go rename to roaring/containers_slice.go From c133ce037661422d51c949664b399d8be5d8d2c4 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 22 May 2019 14:53:07 -0500 Subject: [PATCH 67/73] Make containers copy-on-write This patch replaces a lot of circumstances in which containers were being copied with circumstances in which they are shared, using copy-on-write semantics. To achieve this, we emulate somewhat the design of go's native `append` function. Operations on a container may optionally yield a new container. A container can be marked "frozen", after which no operation should ever write to it in any way; that applies both to the container itself and the backing store it refers to, if any. So for instance, instead of: c.arrayToBitmap() we now write: c = c.arrayToBitmap() Operations which need to modify a container in any way need to be able to return a new container, which is a modified copy of the previous container. This applies to operations like add/remove, but also to things like unmapping memory-mapped storage, or changing a container's type. Bitmaps do not support the same copy-on-write semantics, currently, but "copying" a bitmap and sharing the containers instead of duplicating them is *much* cheaper than copying the containers. Bitmaps do support a .Freeze method, which currently copies the previous bitmap, making a new one with the same container pointers, and freezes the individual containers. Use this if you need a writeable copy of a bitmap -- the resulting bitmap can safely have its set of containers modified, and bitmap operators that would want to modify the containers will use copy-on-write for that. The primary motivation of this is to reduce the cost of the row cache used by fragments. As a secondary issue, the row cache is no longer updated on writes -- that update was actually a race condition waiting to happen. Rather, writes to a row invalidate the cache entry for that row. The row cache is created by creating a new bitmap, and freezing the relevant containers from the fragment's storage. In the case where nothing is being written, the row cache grows to contain bitmaps containing all those containers, but never copies any containers. If nothing's being read, the row cache is never created, and the containers are in general not getting frozen. The only circumstance where copies have to happen is when things are read (and thus stored in the row cache) and later modified. In that case, each read freezes objects, and the first write to a container after it's been frozen will create a new copy. We drop the enterprise/b btree implementation, because we don't really need it anymore -- we now provide that implementation by default in the open source product anyway. Along with this, there's a lot of other changes which improve support for nil containers, as a cheaper representation for empty containers. Operations which we know will provide an empty container can always short-circuit and just yield a nil *Container. Similarly, operations which would provide a full container can return a single shared full container object (which is frozen). The higher-level (non type-specific) container ops are now using that logic to short-circuit operations for empty and full containers. (For instance, difference of anything minus an empty container is the original thing, union of anything and empty is the original thing, and so on.) The Containers interface adds "Update" and "UpdateEvery" methods, based in part on the "Put" interface provided by the underlying btree implementation; Update performs a possible update in-place of a container for a given key, bypassing the need to replicate the search for that key in the container. UpdateEvery loops through all the containers. Containers do not strictly guarantee that they won't return nil `*Container` objects. However, the container iterators won't return those -- empty containers aren't interesting. Some tests are updated to reflect this. Some of the container internals, like N(), or the isArray() and related functions, accept nil container pointers. Some, like Thaw(), do not. For the array(), bitmap(), and runs() methods, roaringparanoia enables an explicit panic on a nil container explaining the problem, but the intent is that those should never be called unless you already know you have the right kind of container, so by default they don't perform the extra checks. In most cases, this is already covered because a nil container is empty, and there's no operation we can perform that requires us to inspect the contents of an empty container. This is passing a fair amount of testing, but the testing may not be comprehensive enough. The overall impact of this is pretty trivial performance-wise. In our default roaring/ benchmarks, a few things get a few percent faster, or slower. The advantage is that, with read-heavy workloads, the row cache no longer eats up incredible amounts of memory. For a smallish test case, pilosa's memory usage (RES in top) after startup was ~2.5GB. Without this patch, simply reading every row a few times got memory usage to about 9GB, which seemed reasonably stable. With this patch, memory usage went to about 3GB. This will be less noticeable in mixed read/write loads, but it should be consistently significantly lower. In addition to dropping things from the rowCache on modifications, we also stopped performing a full count on a modified row when not using a cache of a kind that would use that count, and don't repopulate the rowCache regardless. We don't want every write to imply a corresponding read after it. There's a lot of room for possible future optimizations in terms of things like in-place operations, and some of the row/rowSegment code is a little suspicious to me, but I don't think it should be *worse* in any cases. --- cache.go | 9 +- fragment.go | 49 +- fragment_internal_test.go | 28 + roaring/btree.go | 61 +- roaring/btree_test.go | 18 +- roaring/container_stash.go | 436 ++++++++++--- roaring/containers_btree.go | 71 ++- roaring/containers_slice.go | 87 ++- roaring/containers_test.go | 24 +- roaring/roaring.go | 1016 ++++++++++++++++++------------ roaring/roaring_helpers_test.go | 3 +- roaring/roaring_internal_test.go | 357 +++++------ roaring/roaring_test.go | 7 +- row.go | 81 ++- 14 files changed, 1456 insertions(+), 791 deletions(-) diff --git a/cache.go b/cache.go index 40509ab64..0bf4cd09d 100644 --- a/cache.go +++ b/cache.go @@ -452,9 +452,14 @@ func (s *simpleCache) Fetch(id uint64) (*Row, bool) { return m, ok } -// Add adds the bitmap to the cache, keyed on the id. +// Add adds the bitmap to the cache, keyed on the id. A nil row means +// deleting the row from the cache. func (s *simpleCache) Add(id uint64, b *Row) { - s.cache[id] = b + if b != nil { + s.cache[id] = b + } else { + delete(s.cache, id) + } } // nopCache represents a no-op Cache implementation. diff --git a/fragment.go b/fragment.go index d9b9e8128..eaccf7b86 100644 --- a/fragment.go +++ b/fragment.go @@ -422,19 +422,18 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row { func (f *fragment) rowFromStorage(rowID uint64) *Row { // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by container width. + // + // Note that OffsetRange now returns a new bitmap which uses frozen + // containers which will use copy-on-write semantics. The actual bitmap + // and Containers object are new and not shared, but the containers are + // shared. data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) - // Reference bitmap subrange in storage. We Clone() data because otherwise - // row will contain pointers to containers in storage. This causes - // unexpected results when we cache the row and try to use it later. - // Basically, since we return the Row and release the fragment lock, the - // underlying fragment storage could be changed or snapshotted and thrown - // out at any point. row := &Row{ segments: []rowSegment{{ - data: *data.Clone(), + data: data, shard: f.shard, - writable: false, // this Row will probably be cached and shared, so it must be read only. + writable: true, }}, } row.invalidateCount() @@ -505,12 +504,15 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err return false, errors.Wrap(err, "incrementing") } - // Get the row from row cache or fragment.storage. - row := f.unprotectedRow(rowID) - row.SetBit(columnID) - - // Update the cache. - f.cache.Add(rowID, row.Count()) + // If we're using a cache, update it. Otherwise skip the + // possibly-expensive count operation. + if f.CacheType != CacheTypeNone { + n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.Add(rowID, n) + } + // Drop the rowCache entry; it's wrong, and we don't want to force + // a new copy if no one's reading it. + f.rowCache.Add(rowID, nil) f.stats.Count("setBit", 1, 0.001) @@ -566,12 +568,15 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er return false, errors.Wrap(err, "incrementing") } - // Get the row from cache or fragment.storage. - row := f.unprotectedRow(rowID) - row.clearBit(columnID) - - // Update the cache. - f.cache.Add(rowID, row.Count()) + // If we're using a cache, update it. Otherwise skip the + // possibly-expensive count operation. + if f.CacheType != CacheTypeNone { + n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.Add(rowID, n) + } + // Drop the rowCache entry; it's wrong, and we don't want to force + // a new copy if no one's reading it. + f.rowCache.Add(rowID, nil) f.stats.Count("clearBit", 1, 1.0) @@ -1849,9 +1854,7 @@ func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct f.cache.BulkAdd(rowID, n) if smallWrite { - if _, ok := f.rowCache.Fetch(rowID); ok { // we won't update the rowCache if it wasn't already in there. - f.rowCache.Add(rowID, f.rowFromStorage(rowID)) - } + f.rowCache.Add(rowID, nil) } } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index ecc506814..d815f41bd 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -34,6 +34,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // Test flags @@ -3292,3 +3293,30 @@ func TestImportMultipleValues(t *testing.T) { } } } + +func TestFragmentConcurrentReadWrite(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + defer f.Clean(t) + + eg := &errgroup.Group{} + eg.Go(func() error { + for i := uint64(0); i < 1000; i++ { + _, err := f.setBit(i%4, i) + if err != nil { + return errors.Wrap(err, "setting bit") + } + } + return nil + }) + + acc := uint64(0) + for i := uint64(0); i < 100; i++ { + r := f.row(i % 4) + acc += r.Count() + } + if err := eg.Wait(); err != nil { + t.Errorf("error from setting a bit: %v", err) + } + + t.Logf("%d", acc) +} diff --git a/roaring/btree.go b/roaring/btree.go index 192993c3b..53489af89 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -570,6 +570,13 @@ func (t *tree) Set(k uint64, v *Container) { //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) //}() + // we don't want to store nil containers; if you try to set a + // container to nil, that's equivalent to not having one at that + // location. + if v == nil { + _ = t.Delete(k) + return + } pi := -1 var p *x @@ -642,7 +649,9 @@ func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta if !written { return } - + if newV == nil { + return + } z := t.insert(btDPool.Get().(*d), 0, k, newV) t.r, t.first, t.last = z, z, z return @@ -667,7 +676,11 @@ func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta if !written { return } - + // delete nil containers rather than storing them. + if newV == nil { + t.Delete(k) + return + } x.d[i].v = newV } return @@ -686,6 +699,10 @@ func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta if !written { return } + // nil values don't need to exist, and break iteration later. + if newV == nil { + return + } switch { case x.c < 2*kd: @@ -876,6 +893,46 @@ func (e *enumerator) Next() (k uint64, v *Container, err error) { return k, v, err } +// Every iterates over a tree. +func (e *enumerator) Every(upd func(oldV *Container, exists bool) (newV *Container, write bool)) error { + if err := e.err; err != nil { + return err + } + + if e.ver != e.t.ver { + f, _ := e.t.Seek(e.k) + *e = *f + f.Close() + } + + for { + if e.q == nil { + e.err = io.EOF + return e.err + } + + if e.i >= e.q.c { + if err := e.next(); err != nil { + e.err = err + return e.err + } + } + + i := e.q.d[e.i] + nv, write := upd(i.v, true) + if write { + if nv == nil { + e.t.Delete(e.q.d[e.i].k) + } else { + e.q.d[e.i].v = nv + } + } + // Any error returned would be stashed in e.err, and would come up + // on the next call. + _ = e.next() + } +} + func (e *enumerator) next() error { if e.q == nil { e.err = io.EOF diff --git a/roaring/btree_test.go b/roaring/btree_test.go index c12892f62..422a7b767 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -467,13 +467,14 @@ func benchmarkSetRnd(b *testing.B, n int) { a[i] = rng.Next() } b.ResetTimer() + c := getDummyC(1) for i := 0; i < b.N; i++ { b.StopTimer() r := treeNew() debug.FreeOSMemory() b.StartTimer() for _, v := range a { - r.Set(uint64(v), nil) + r.Set(uint64(v), c) } b.StopTimer() r.Close() @@ -504,8 +505,9 @@ func benchmarkGetRnd(b *testing.B, n int) { for i := range a { a[i] = rng.Next() } + c := getDummyC(1) for _, v := range a { - r.Set(uint64(v), nil) + r.Set(uint64(v), c) } debug.FreeOSMemory() b.ResetTimer() @@ -1392,7 +1394,7 @@ func TestBtreePut(t *testing.T) { t.Fatal(iTest, g, e) } } - return nil, test.write + return getDummyC(99), test.write }) if test.exists { if g, e := oldV, getDummyC(test.oldV); g != e { @@ -1427,6 +1429,8 @@ func TestBtreePut(t *testing.T) { var e *Container if test.post[i+1] != -1 { e = getDummyC(test.post[i+1]) + } else { + e = getDummyC(99) } if g := v; g != e { t.Fatal(iTest, g, e) @@ -1445,7 +1449,7 @@ func TestBtreeSeek(t *testing.T) { tr := treeNew() for i := 0; i < N; i++ { k := 2*i + 1 - tr.Set(uint64(k), nil) + tr.Set(uint64(k), getDummyC(1)) } for i := 0; i < N; i++ { k := 2 * i @@ -1476,14 +1480,14 @@ func TestBtreePR4(t *testing.T) { tr := treeNew() for i := 0; i < 2*kd+1; i++ { k := 1000 * i - tr.Set(uint64(k), nil) + tr.Set(uint64(k), getDummyC(1)) } tr.Delete(1000 * kd) for i := 0; i < kd; i++ { - tr.Set(uint64(1000*(kd+1)-1-i), nil) + tr.Set(uint64(1000*(kd+1)-1-i), getDummyC(1)) } k := 1000*(kd+1) - 1 - kd - tr.Set(uint64(k), nil) + tr.Set(uint64(k), getDummyC(1)) if _, ok := tr.Get(uint64(k)); !ok { t.Fatalf("key lost: %v", k) } diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 728730d64..46725238f 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -15,6 +15,7 @@ package roaring import ( + "fmt" "reflect" "runtime" "unsafe" @@ -39,24 +40,84 @@ type Container struct { pointer *uint16 // the data pointer len, cap int32 // length and cap n int32 // number of integers in container - mapped bool // mapped directly to a byte slice when true - typ byte // array, bitmap, or run + flags containerFlags // internal flags + typeID byte // array, bitmap, or run data [stashedArraySize]uint16 // immediate data for small arrays or runs } +type containerFlags uint8 + +const ( + flagMapped = containerFlags(1 << iota) + flagFrozen +) + +func (c *Container) String() string { + if c == nil { + return "" + } + froze := "" + switch c.flags { + case flagFrozen: + froze = "frozen " + case flagMapped: + froze = "mapped " + case flagFrozen | flagMapped: + froze = "frozen/mapped" + } + switch c.typeID { + case containerArray: + return fmt.Sprintf("<%sarray container, N=%d>", froze, c.N()) + case containerBitmap: + return fmt.Sprintf("<%sbitmap container, N=%d, len %dx uint64>", + froze, c.N(), len(c.bitmap())) + case containerRun: + return fmt.Sprintf("<%srun container, N=%d, len %dx interval>", + froze, c.N(), len(c.runs())) + default: + return fmt.Sprintf("", froze, c.typeID, c.N()) + } +} + // NewContainer returns a new instance of container. This trivial function // may later become more interesting. func NewContainer() *Container { statsHit("NewContainer") - c := &Container{typ: containerArray, len: 0, cap: stashedArraySize} + c := &Container{typeID: containerArray, len: 0, cap: stashedArraySize} c.pointer = (*uint16)(unsafe.Pointer(&c.data[0])) return c } // NewContainerBitmap makes a bitmap container using the provided bitmap, or // an empty one if provided bitmap is nil. If the provided bitmap is too short, -// it will be padded. -func NewContainerBitmap(n int32, bitmap []uint64) *Container { +// it will be padded. This function's API is wrong; it should have been +// written as NewContainerBitmapN, and this should not take the n argument, +// but I did it wrong initially and now that would be a breaking change. +func NewContainerBitmap(n int, bitmap []uint64) *Container { + if bitmap == nil { + return NewContainerBitmapN(nil, 0) + } + // pad to required length + if len(bitmap) < bitmapN { + bm2 := make([]uint64, bitmapN) + copy(bm2, bitmap) + bitmap = bm2 + } + c := &Container{typeID: containerBitmap} + c.setBitmap(bitmap) + // set n based on bitmap contents. + if n < 0 { + c.bitmapRepair() + } else { + c.setN(int32(n)) + } + return c +} + +// NewContainerBitmapN makes a bitmap container using the provided bitmap, or +// an empty one if provided bitmap is nil. If the provided bitmap is too short, +// it will be padded. The container's count is specified directly. +func NewContainerBitmapN(bitmap []uint64, n int32) *Container { if bitmap == nil { bitmap = make([]uint64, bitmapN) } @@ -66,23 +127,40 @@ func NewContainerBitmap(n int32, bitmap []uint64) *Container { copy(bm2, bitmap) bitmap = bm2 } - c := &Container{typ: containerBitmap, n: n} + c := &Container{typeID: containerBitmap, n: n} c.setBitmap(bitmap) return c } -// NewContainerArray returns an array using the provided set of values. It's -// okay if the slice is nil; that's a length of zero. +// NewContainerArray returns an array container using the provided set of +// values. It's okay if the slice is nil; that's a length of zero. func NewContainerArray(set []uint16) *Container { - c := &Container{typ: containerArray, n: int32(len(set))} + c := &Container{typeID: containerArray, n: int32(len(set))} c.setArray(set) return c } -// NewContainerRun creates a new run array using a provided (possibly nil) +// NewContainerArrayCopy returns an array container using the provided set of +// values. It's okay if the slice is nil; that's a length of zero. It copies +// the provided slice to new storage. +func NewContainerArrayCopy(set []uint16) *Container { + c := &Container{typeID: containerArray, n: int32(len(set))} + c.setArrayMaybeCopy(set, true) + return c +} + +// NewContainerArrayN returns an array container using the specified +// set of values, but overriding n. +func NewContainerArrayN(set []uint16, n int32) *Container { + c := &Container{typeID: containerArray, n: n} + c.setArray(set) + return c +} + +// NewContainerRun creates a new run container using a provided (possibly nil) // slice of intervals. func NewContainerRun(set []interval16) *Container { - c := &Container{typ: containerRun} + c := &Container{typeID: containerRun} c.setRuns(set) for _, run := range set { c.n += int32(run.last-run.start) + 1 @@ -90,61 +168,242 @@ func NewContainerRun(set []interval16) *Container { return c } +// NewContainerRunCopy creates a new run container using a provided (possibly nil) +// slice of intervals. It copies the provided slice to new storage. +func NewContainerRunCopy(set []interval16) *Container { + c := &Container{typeID: containerRun} + c.setRunsMaybeCopy(set, true) + for _, run := range set { + c.n += int32(run.last-run.start) + 1 + } + return c +} + +// NewContainerRunN creates a new run array using a provided (possibly nil) +// slice of intervals. It overrides n using the provided value. +func NewContainerRunN(set []interval16, n int32) *Container { + c := &Container{typeID: containerRun, n: n} + c.setRuns(set) + return c +} + // Mapped returns the internal mapped field, which indicates whether the // slice's backing store is believed to be associated with unwriteable // mmapped space. func (c *Container) Mapped() bool { - return c.mapped + if c == nil { + return false + } + return (c.flags & flagMapped) != 0 } -// N returns the internal n field. +// frozen() returns the internal frozen state. It isn't exported because +// nothing outside this package should be thinking about this. +func (c *Container) frozen() bool { + if c == nil { + return true + } + return (c.flags & flagFrozen) != 0 +} + +// N returns the 1-count of the container. func (c *Container) N() int32 { + if c == nil { + return 0 + } return c.n } +func (c *Container) setN(n int32) { + if c == nil { + if roaringParanoia { + panic("trying to setN on a nil container") + } + return + } + c.n = n +} + +func (c *Container) typ() byte { + if c == nil { + return containerNil + } + return c.typeID +} + +// setTyp should only be called if you already know that c is a +// non-nil, non-frozen, container. +func (c *Container) setTyp(newType byte) { + if roaringParanoia { + if c == nil || c.frozen() { + panic("setTyp on nil or frozen container") + } + } + c.typeID = newType +} + +func (c *Container) setMapped(mapped bool) { + if roaringParanoia { + if c == nil || c.frozen() { + panic("setMapped on nil or frozen container") + } + } + if mapped { + c.flags |= flagMapped + } else { + c.flags &^= flagMapped + } +} + +// Freeze returns an unmodifiable container identical to c. This might +// be c, now marked unmodifiable, or might be a new container. +func (c *Container) Freeze() *Container { + if c == nil { + return nil + } + c.flags |= flagFrozen + return c +} + +// Thaw returns a modifiable container identical to c. This may be c, or it +// may be a new container with distinct backing store. +func (c *Container) Thaw() *Container { + if roaringParanoia { + if c == nil { + panic("trying to thaw a nil container") + } + } + if c.flags&(flagFrozen|flagMapped) == 0 { + return c + } + return c.unmapOrClone() +} + +func (c *Container) unmapOrClone() *Container { + if c.flags&flagFrozen != 0 { + // Caqn't modify this container, therefore, we have to make a + // copy. + return c.Clone() + } + c.flags &^= flagMapped + // mapped: we want to unmap the storage. + switch c.typeID { + case containerArray: + // mapped flag is wrong here + if c.pointer == (*uint16)(unsafe.Pointer(&c.data)) { + return c + } + // maybe it fits in storage + if c.len <= stashedArraySize { + copy(c.data[:stashedArraySize], c.array()) + c.pointer, c.cap = (*uint16)(unsafe.Pointer(&c.data)), stashedArraySize + return c + } + array := c.array() + tmp := make([]uint16, c.len) + copy(tmp, array) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) + runtime.KeepAlive(&tmp) + case containerRun: + // mapped flag is wrong here + if c.pointer == (*uint16)(unsafe.Pointer(&c.data)) { + return c + } + oldRuns := c.runs() + // maybe it fits in storage + if c.len <= stashedRunSize { + c.pointer, c.cap = (*uint16)(unsafe.Pointer(&c.data)), stashedRunSize + copy(c.runs(), oldRuns) + return c + } + tmp := make([]interval16, c.len) + copy(tmp, oldRuns) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) + runtime.KeepAlive(&tmp) + case containerBitmap: + bitmap := c.bitmap() + tmp := make([]uint64, bitmapN) + copy(tmp, bitmap) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), bitmapN, bitmapN + runtime.KeepAlive(&tmp) + default: + panic(fmt.Sprintf("can't thaw invalid container, type %d", c.typeID)) + } + return c +} + // array yields the data viewed as a slice of uint16 values. func (c *Container) array() []uint16 { if roaringParanoia { - if c.typ != containerArray { + if c == nil { + panic("attempt to read a nil container's array") + } + if c.typeID != containerArray { panic("attempt to read non-array's array") } } return *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) } -// setArray stores a set of uint16s as data. -func (c *Container) setArray(array []uint16) { +// setArrayMaybeCopy stores a set of uint16s as data. c must not be frozen. +// If doCopy is set, it will ensure that the data get copied (possibly to +// its internal stash.) +func (c *Container) setArrayMaybeCopy(array []uint16, doCopy bool) { if roaringParanoia { - if c.typ != containerArray { + if c == nil || c.frozen() { + panic("setArray on nil or frozen container") + } + if c.typeID != containerArray { panic("attempt to write non-array's array") } } // no array: start with our default 5-value array if array == nil { c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedArraySize + c.n = c.len return } h := (*reflect.SliceHeader)(unsafe.Pointer(&array)) if h.Data == uintptr(unsafe.Pointer(c.pointer)) { // nothing to do but update length c.len = int32(h.Len) + c.n = c.len return } // array we can fit in data store: if len(array) <= stashedArraySize { copy(c.data[:stashedArraySize], array) c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(array)), stashedArraySize - c.mapped = false // this is no longer using a hypothetical mmapped input array + c.n = c.len + c.flags &^= flagMapped // this is no longer using a hypothetical mmapped input array return } + // copy the array + if doCopy { + a2 := make([]uint16, len(array)) + copy(a2, array) + h = (*reflect.SliceHeader)(unsafe.Pointer(&a2)) + } c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) + c.n = c.len runtime.KeepAlive(&array) } +// setArrayMaybeCopy stores a set of uint16s as data. c must not be frozen. +func (c *Container) setArray(array []uint16) { + c.setArrayMaybeCopy(array, false) +} + // bitmap yields the data viewed as a slice of uint64s holding bits. func (c *Container) bitmap() []uint64 { if roaringParanoia { - if c.typ != containerBitmap { + if c == nil { + panic("attempt to read nil container's bitmap") + } + if c.typeID != containerBitmap { panic("attempt to read non-bitmap's bitmap") } } @@ -153,8 +412,11 @@ func (c *Container) bitmap() []uint64 { // setBitmap stores a set of uint64s as data. func (c *Container) setBitmap(bitmap []uint64) { + if c == nil || c.frozen() { + panic("setBitmap on nil or frozen container") + } if roaringParanoia { - if c.typ != containerBitmap { + if c.typeID != containerBitmap { panic("attempt to write non-bitmap's bitmap") } } @@ -166,17 +428,29 @@ func (c *Container) setBitmap(bitmap []uint64) { // runs yields the data viewed as a slice of intervals. func (c *Container) runs() []interval16 { if roaringParanoia { - if c.typ != containerRun { + if c == nil { + panic("attempt to read nil container's runs") + } + if c.typeID != containerRun { panic("attempt to read non-run's runs") } } return *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) } -// setRuns stores a set of intervals as data. +// setRuns stores a set of intervals as data. c must not be frozen. func (c *Container) setRuns(runs []interval16) { + c.setRunsMaybeCopy(runs, false) +} + +// setRunsMaybeCopy stores a set of intervals as data. c must not be frozen. +// If doCopy is set, the values will be copied to different storage. +func (c *Container) setRunsMaybeCopy(runs []interval16, doCopy bool) { if roaringParanoia { - if c.typ != containerRun { + if c == nil || c.frozen() { + panic("setRuns on nil or frozen container") + } + if c.typeID != containerRun { panic("attempt to write non-run's runs") } } @@ -197,20 +471,64 @@ func (c *Container) setRuns(runs []interval16) { newRuns := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&c.data[0])), Len: stashedRunSize, Cap: stashedRunSize})) copy(newRuns, runs) c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(runs)), stashedRunSize - c.mapped = false // this is no longer using a hypothetical mmapped input array + c.flags &^= flagMapped // this is no longer using a hypothetical mmapped input array return } + if doCopy { + r2 := make([]interval16, len(runs)) + copy(r2, runs) + h = (*reflect.SliceHeader)(unsafe.Pointer(&r2)) + } c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) runtime.KeepAlive(&runs) } -// Update updates the container -func (c *Container) Update(typ byte, n int32, mapped bool) { - c.typ = typ +// UpdateOrMake updates the container, yielding a new container if necessary. +func (c *Container) UpdateOrMake(typ byte, n int32, mapped bool) *Container { + if c == nil { + switch typ { + case containerRun: + c = NewContainerRunN(nil, n) + case containerBitmap: + c = NewContainerBitmapN(nil, n) + default: + c = NewContainerArrayN(nil, n) + } + c.flags |= flagMapped + return c + } + // ensure that we are allowed to modify this container + c = c.Thaw() + c.typeID = typ c.n = n - c.mapped = mapped + // note: this probably shouldn't be happening, the decision should be getting + // made when we specify the storage. + c.setMapped(mapped) // we don't know that any existing slice is usable, so let's ditch it - switch c.typ { + switch c.typeID { + case containerArray: + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(0), stashedArraySize + case containerRun: + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedRunSize + default: + c.pointer, c.len, c.cap = nil, 0, 0 + } + return c +} + +// Update updates the container if possible. It is an error to +// call Update on a frozen container. +func (c *Container) Update(typ byte, n int32, mapped bool) { + if c == nil || c.frozen() { + panic("cannot Update a nil or frozen container") + } + c.typeID = typ + c.n = n + // note: this probably shouldn't be happening, the decision should be getting + // made when we specify the storage. + c.setMapped(mapped) + // we don't know that any existing slice is usable, so let's ditch it + switch c.typeID { case containerArray: c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(0), stashedArraySize case containerRun: @@ -222,56 +540,30 @@ func (c *Container) Update(typ byte, n int32, mapped bool) { // isArray returns true if the container is an array container. func (c *Container) isArray() bool { - return c.typ == containerArray + if roaringParanoia { + if c == nil { + panic("calling isArray on nil container") + } + } + return c.typeID == containerArray } // isBitmap returns true if the container is a bitmap container. func (c *Container) isBitmap() bool { - return c.typ == containerBitmap + if roaringParanoia { + if c == nil { + panic("calling isBitmap on nil container") + } + } + return c.typeID == containerBitmap } // isRun returns true if the container is a run-length-encoded container. func (c *Container) isRun() bool { - return c.typ == containerRun -} - -// unmapArray ensures that the container is not using mmapped storage. -func (c *Container) unmapArray() { - if !c.mapped { - return + if roaringParanoia { + if c == nil { + panic("calling isRun on nil container") + } } - array := c.array() - tmp := make([]uint16, c.len) - copy(tmp, array) - h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) - c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) - runtime.KeepAlive(&tmp) - c.mapped = false -} - -// unmapBitmap ensures that the container is not using mmapped storage. -func (c *Container) unmapBitmap() { - if !c.mapped { - return - } - bitmap := c.bitmap() - tmp := make([]uint64, c.len) - copy(tmp, bitmap) - h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) - c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) - runtime.KeepAlive(&tmp) - c.mapped = false -} - -// unmapRun ensures that the container is not using mmapped storage. -func (c *Container) unmapRun() { - if !c.mapped { - return - } - runs := c.runs() - tmp := make([]interval16, c.len) - copy(tmp, runs) - h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) - c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) - c.mapped = false + return c.typeID == containerRun } diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index 5934a1244..c2dbc01f5 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -15,6 +15,7 @@ package roaring import ( + "fmt" "io" ) @@ -42,7 +43,7 @@ func NewBTreeBitmap(a ...uint64) *Bitmap { func (btc *bTreeContainers) Get(key uint64) *Container { // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { + if key == btc.lastKey { return btc.lastContainer } @@ -57,11 +58,15 @@ func (btc *bTreeContainers) Get(key uint64) *Container { } func (btc *bTreeContainers) Put(key uint64, c *Container) { + // If we don't do this, a Put on a container we just got from + // Get can result in the tree containing a different container + // than we'll get on next lookup. + btc.lastKey, btc.lastContainer = key, c // If a mapped container is added to the tree, reset the // lastContainer cache so that the cache is not pointing // at a read-only mmap. if c.Mapped() { - btc.lastContainer = nil + btc.lastKey = ^uint64(0) } btc.tree.Set(key, c) } @@ -69,13 +74,13 @@ func (btc *bTreeContainers) Put(key uint64, c *Container) { func (u updater) update(oldV *Container, exists bool) (*Container, bool) { // update the existing container if exists { - oldV.Update(u.typ, u.n, u.mapped) - return oldV, false + oldV = oldV.UpdateOrMake(u.typ, u.n, u.mapped) + return oldV, true } cont := NewContainer() - cont.typ = u.typ - cont.n = u.n - cont.mapped = u.mapped + cont.setTyp(u.typ) + cont.setN(u.n) + cont.setMapped(u.mapped) return cont, true } @@ -98,7 +103,7 @@ func (btc *bTreeContainers) Remove(key uint64) { func (btc *bTreeContainers) GetOrCreate(key uint64) *Container { // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { + if key == btc.lastKey { return btc.lastContainer } @@ -119,7 +124,7 @@ func (btc *bTreeContainers) Count() (n uint64) { e, _ := btc.tree.Seek(0) _, c, err := e.Next() for err != io.EOF { - n += uint64(c.n) + n += uint64(c.N()) _, c, err = e.Next() } return n @@ -142,6 +147,23 @@ func (btc *bTreeContainers) Clone() Containers { return nbtc } +func (btc *bTreeContainers) Freeze() Containers { + nbtc := newBTreeContainers() + + itr, err := btc.tree.SeekFirst() + if err == io.EOF { + return nbtc + } + for { + k, v, err := itr.Next() + if err == io.EOF { + break + } + nbtc.tree.Set(k, v.Freeze()) + } + return nbtc +} + func (btc *bTreeContainers) Last() (key uint64, c *Container) { if btc.tree.Len() == 0 { return 0, nil @@ -156,7 +178,10 @@ func (btc *bTreeContainers) Size() int { func (btc *bTreeContainers) Reset() { btc.tree = treeNew() - btc.lastKey = 0 + // use a definitely-invalid key, so we can distinguish between "you + // just looked that up, and it was a nil container" and "you have + // never looked that up before." + btc.lastKey = ^uint64(0) btc.lastContainer = nil } @@ -180,6 +205,23 @@ func (btc *bTreeContainers) Repair() { } } +// Update calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (btc *bTreeContainers) Update(key uint64, fn func(*Container, bool) (*Container, bool)) { + btc.tree.Put(key, fn) +} + +// UpdateEvery calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (btc *bTreeContainers) UpdateEvery(fn func(*Container, bool) (*Container, bool)) { + e, _ := btc.tree.Seek(0) + // currently not handling the error from this, but in practice it has + // to be io.EOF. + _ = e.Every(fn) +} + type btcIterator struct { e *enumerator key uint64 @@ -187,19 +229,20 @@ type btcIterator struct { } func (i *btcIterator) Next() bool { - k, v, err := i.e.Next() if err == io.EOF { return false } + if roaringParanoia { + if v == nil { + panic(fmt.Sprintf("got nil container for key %d", k)) + } + } i.key = k i.val = v return true } func (i *btcIterator) Value() (uint64, *Container) { - if i.val == nil { - return 0, nil - } return i.key, i.val } diff --git a/roaring/containers_slice.go b/roaring/containers_slice.go index dc5d4118d..cbff4f179 100644 --- a/roaring/containers_slice.go +++ b/roaring/containers_slice.go @@ -50,15 +50,21 @@ func (sc *sliceContainers) PutContainerValues(key uint64, typ byte, n int, mappe i := search64(sc.keys, key) if i < 0 { c := NewContainer() - c.typ = typ - c.n = int32(n) - c.mapped = mapped + c.setTyp(typ) + c.setN(int32(n)) + c.setMapped(mapped) sc.insertAt(key, c, -i-1) } else { - c := sc.containers[i] - c.typ = typ - c.n = int32(n) - c.mapped = mapped + // if the container already exists, and is frozen, this may + // result in copying its data, which is sort of pointless + // because PutContainerValues almost always gets called + // because we're reading new data from a file -- but also + // that means this case probably never happens. + c := sc.containers[i].Thaw() + c.setTyp(typ) + c.setN(int32(n)) + c.setMapped(mapped) + sc.containers[i] = c } } @@ -114,6 +120,17 @@ func (sc *sliceContainers) Clone() Containers { return other } +func (sc *sliceContainers) Freeze() Containers { + other := newSliceContainers() + other.keys = make([]uint64, len(sc.keys)) + other.containers = make([]*Container, len(sc.containers)) + copy(other.keys, sc.keys) + for i, c := range sc.containers { + other.containers[i] = c.Freeze() + } + return other +} + func (sc *sliceContainers) Last() (key uint64, c *Container) { if len(sc.keys) == 0 { return 0, nil @@ -129,7 +146,7 @@ func (sc *sliceContainers) Size() int { func (sc *sliceContainers) Count() uint64 { n := uint64(0) for i := range sc.containers { - n += uint64(sc.containers[i].n) + n += uint64(sc.containers[i].N()) } return n } @@ -162,6 +179,42 @@ func (sc *sliceContainers) Repair() { } } +// Update calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (sc *sliceContainers) Update(key uint64, fn func(*Container, bool) (*Container, bool)) { + i, found := sc.seek(key) + var nc *Container + var write bool + if found { + nc, write = fn(sc.containers[i], true) + if write { + sc.containers[i] = nc + } + } else { + nc, write = fn(nil, false) + // don't expand the slice just to add a nil container, we + // could return that anyway + if write && nc != nil { + sc.containers = append(sc.containers, nil) + copy(sc.containers[i+1:], sc.containers[i:]) + sc.containers[i] = nc + } + } +} + +// UpdateEvery calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (sc *sliceContainers) UpdateEvery(fn func(*Container, bool) (*Container, bool)) { + for i, c := range sc.containers { + nc, write := fn(c, true) + if write { + sc.containers[i] = nc + } + } +} + type sliceIterator struct { e *sliceContainers i int @@ -170,14 +223,20 @@ type sliceIterator struct { } func (si *sliceIterator) Next() bool { - if si.e == nil || si.i > len(si.e.keys)-1 { + if si.e == nil { return false } - si.key = si.e.keys[si.i] - si.value = si.e.containers[si.i] - si.i++ - - return true + // discard nil containers from iteration. we don't always + // actually remove them because copying is expensive. + for si.i < len(si.e.keys) { + si.key = si.e.keys[si.i] + si.value = si.e.containers[si.i] + si.i++ + if si.value != nil { + return true + } + } + return false } func (si *sliceIterator) Value() (uint64, *Container) { diff --git a/roaring/containers_test.go b/roaring/containers_test.go index ad95f2f79..abe5dbd99 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -43,14 +43,14 @@ func testContainersIterator(cs Containers, t *testing.T) { if !itr.Next() { t.Fatalf("one should be next, but got false") } - if key, val := itr.Value(); key != 1 || val.n != 1 { - t.Fatalf("Wrong k/v, exp: 1,1 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 1 || val.N() != 1 { + t.Fatalf("Wrong k/v, exp: 1,1 got: %v,%v", key, val.N()) } if !itr.Next() { t.Fatalf("two should be next, but got false") } - if key, val := itr.Value(); key != 2 || val.n != 2 { - t.Fatalf("Wrong k/v, exp: 2,2 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 2 || val.N() != 2 { + t.Fatalf("Wrong k/v, exp: 2,2 got: %v,%v", key, val.N()) } if itr.Next() { @@ -68,14 +68,14 @@ func testContainersIterator(cs Containers, t *testing.T) { if !found { t.Fatalf("should have found 3") } - if key, val := itr.Value(); key != 3 || val.n != 3 { - t.Fatalf("Wrong k/v, exp: 3,3 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 3 || val.N() != 3 { + t.Fatalf("Wrong k/v, exp: 3,3 got: %v,%v", key, val.N()) } if !itr.Next() { t.Fatalf("5 should be next, but got false") } - if key, val := itr.Value(); key != 5 || val.n != 5 { - t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 5 || val.N() != 5 { + t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.N()) } itr, found = cs.Iterator(4) @@ -85,14 +85,14 @@ func testContainersIterator(cs Containers, t *testing.T) { if !itr.Next() { t.Fatalf("5 should be next, but got false") } - if key, val := itr.Value(); key != 5 || val.n != 5 { - t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 5 || val.N() != 5 { + t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.N()) } if !itr.Next() { t.Fatalf("6 should be next, but got false") } - if key, val := itr.Value(); key != 6 || val.n != 6 { - t.Fatalf("Wrong k/v, exp: 6,6 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 6 || val.N() != 6 { + t.Fatalf("Wrong k/v, exp: 6,6 got: %v,%v", key, val.N()) } if itr.Next() { diff --git a/roaring/roaring.go b/roaring/roaring.go index 2d1a2d08a..cd7833a75 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -62,9 +62,10 @@ const ( ) const ( - containerArray byte = iota + 1 // slice of bit position values - containerBitmap // slice of 1024 uint64s - containerRun // container of run-encoded bits + containerNil byte = iota // no container + containerArray // slice of bit position values + containerBitmap // slice of 1024 uint64s + containerRun // container of run-encoded bits ) // map used for a more descriptive print @@ -74,6 +75,8 @@ var containerTypeNames = map[byte]string{ containerRun: "run", } +var fullContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}).Freeze() + type Containers interface { // Get returns nil if the key does not exist. Get(key uint64) *Container @@ -95,12 +98,27 @@ type Containers interface { // Clone does a deep copy of Containers, including cloning all containers contained. Clone() Containers + // Freeze creates a shallow copy of Containers, freezing all the containers + // contained. The new copy is a distinct Containers, but the individual containers + // are shared (but marked as frozen). + Freeze() Containers + // Last returns the highest key and associated container. Last() (key uint64, c *Container) // Size returns the number of containers stored. Size() int + // Update calls fn (existing-container, existed), and expects + // (new-container, write). If write is true, the container is used to + // replace the given container. + Update(key uint64, fn func(*Container, bool) (*Container, bool)) + + // UpdateEvery calls fn (existing-container, existed), and expects + // (new-container, write). If write is true, the container is used to + // replace the given container. + UpdateEvery(fn func(*Container, bool) (*Container, bool)) + // Iterator returns a Contiterator which after a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a // container is found at key. @@ -149,6 +167,21 @@ func NewBitmap(a ...uint64) *Bitmap { return b } +// NewSliceBitmap makes a new bitmap, explicitly selecting the slice containers +// type, which performs better in cases where we expect a contiguous block of +// containers added in ascending order, such as when extracting a range from +// another bitmap. +func NewSliceBitmap(a ...uint64) *Bitmap { + b := &Bitmap{ + Containers: newSliceContainers(), + } + // TODO: We have no way to report this. We aren't in a server context + // so we haven't got a logger, nothing is checking for nil returns + // from this... + _, _ = b.AddN(a...) + return b +} + // NewFileBitmap returns a Bitmap with an initial set of values, used for file storage. // By default, this is a copy of NewBitmap, but is replaced with B+Tree in server/enterprise.go var NewFileBitmap func(a ...uint64) *Bitmap = NewBTreeBitmap @@ -168,6 +201,23 @@ func (b *Bitmap) Clone() *Bitmap { return other } +// Freeze returns a shallow copy of the bitmap. The new bitmap +// is a distinct bitmap, with a new Containers object, but the +// actual containers it holds are the same as the parent's +// containers, but have been frozen. +func (b *Bitmap) Freeze() *Bitmap { + if b == nil { + return nil + } + + // Create a copy of the bitmap structure. + other := &Bitmap{ + Containers: b.Containers.Freeze(), + } + + return other +} + // Add adds values to the bitmap. TODO(2.0) deprecate - use the more general // AddN (though be aware that it modifies 'a' in place). func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { @@ -235,7 +285,7 @@ func (b *Bitmap) DirectRemoveN(a ...uint64) (changed int) { // container level operation across a list of values and return the number of // trues while modifying the list of values in place to contain the // true-returning values in order. -func (b *Bitmap) directOpN(op func(c *Container, v uint16) bool, a ...uint64) (changed int) { +func (b *Bitmap) directOpN(op func(c *Container, v uint16) (*Container, bool), a ...uint64) (changed int) { hb := uint64(0xFFFFFFFFFFFFFFFF) // impossible sentinel value var cont *Container for _, v := range a { @@ -243,10 +293,15 @@ func (b *Bitmap) directOpN(op func(c *Container, v uint16) bool, a ...uint64) (c hb = newhb cont = b.Containers.GetOrCreate(hb) } - if op(cont, lowbits(v)) { + newC, change := op(cont, lowbits(v)) + if change { a[changed] = v changed++ } + if newC != cont { + b.Containers.Put(hb, newC) + cont = newC + } } return changed } @@ -255,7 +310,11 @@ func (b *Bitmap) directOpN(op func(c *Container, v uint16) bool, a ...uint64) (c // deprecate in favor of DirectAddN. func (b *Bitmap) DirectAdd(v uint64) bool { cont := b.Containers.GetOrCreate(highbits(v)) - return cont.add(lowbits(v)) + newC, changed := cont.add(lowbits(v)) + if newC != cont { + b.Containers.Put(highbits(v), newC) + } + return changed } // Contains returns true if v is in the bitmap. @@ -313,11 +372,11 @@ func (b *Bitmap) RemoveN(a ...uint64) (changed int, err error) { func (b *Bitmap) remove(v uint64) bool { c := b.Containers.Get(highbits(v)) - if c == nil { - return false + newC, changed := c.remove(lowbits(v)) + if newC != c { + b.Containers.Put(highbits(v), newC) } - // TODO - do nil check inside c.remove? - return c.remove(lowbits(v)) + return changed } // Max returns the highest value in the bitmap. @@ -345,7 +404,7 @@ func (b *Bitmap) Any() bool { // container should be removed from the bitmap though. for iter.Next() { _, c := iter.Value() - if c.n > 0 { + if c.N() > 0 { return true } } @@ -359,7 +418,6 @@ func (b *Bitmap) Size() int { for citer.Next() { _, c := citer.Value() numbytes += c.size() - } return numbytes } @@ -392,7 +450,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { continue } if k < ekey { - n += uint64(c.n) + n += uint64(c.N()) continue } if k == ekey { @@ -448,6 +506,8 @@ func (b *Bitmap) ForEachRange(start, end uint64, fn func(uint64)) { } // OffsetRange returns a new bitmap with a containers offset by start. +// The containers themselves are shared, so they get frozen so it will +// be safe to interact with them. func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { if lowbits(offset) != 0 { panic("offset must not contain low bits") @@ -462,13 +522,13 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) citer, _ := b.Containers.Iterator(hi0) - other := NewBitmap() + other := NewSliceBitmap() for citer.Next() { k, c := citer.Value() if k >= hi1 { break } - other.Containers.Put(off+(k-hi0), c) + other.Containers.Put(off+(k-hi0), c.Freeze()) } return other } @@ -537,7 +597,10 @@ func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { b.unionIntoTargetSingle(output, others[0]) return output } - output := b.Clone() + // It may seem counterintuitive to freeze this, but the result is + // a new bitmap which can be safely modified, but postponing any + // allocations until an actual write to any given container. + output := b.Freeze() output.UnionInPlace(others...) return output } @@ -556,11 +619,11 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - target.Containers.Put(ki, ci.Clone()) + target.Containers.Put(ki, ci.Freeze()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - target.Containers.Put(kj, cj.Clone()) + target.Containers.Put(kj, cj.Freeze()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -693,11 +756,11 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { tContainer := target.Containers.Get(iKey) // if the target's full, short-circuit out. if tContainer != nil { - if tContainer.n == maxContainerVal+1 { + if tContainer.N() == maxContainerVal+1 { bitmapIters.markItersWithKeyAsHandled(i, iKey) continue } - expectedN = int64(tContainer.n) + expectedN = int64(tContainer.N()) } // Check i and later iters for any max-range containers, and // find out how many there are. @@ -707,7 +770,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // range that a container can store, so instead of calculating a // union we can generate an RLE container that represents the entire // range. - tContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}) + tContainer = fullContainer target.Containers.Put(iKey, tContainer) bitmapIters.markItersWithKeyAsHandled(i, iKey) continue @@ -722,9 +785,9 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // No existing target container. if summaryStats.c == 1 { // There's no target and we have only one container, we - // can just clone it instead of unioning. + // can just reuse it instead of unioning. statsHit("unionInPlace/reuse") - target.Containers.Put(iKey, iContainer.Clone()) + target.Containers.Put(iKey, iContainer.Freeze()) bitmapIters[i].handled = true continue } @@ -734,17 +797,19 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // first other container, but for some cases, that will // result in cloning a non-bitmap, then converting it // to a bitmap, and this will be expensive... - if expectedN >= 512 && iContainer.typ != containerBitmap { + if expectedN >= 512 && iContainer.typ() != containerBitmap { // copying the non-bitmap, then converting it, // is expensive. statsHit("unionInPlace/newBitmap") - tContainer = NewContainerBitmap(0, nil) + tContainer = NewContainerBitmapN(nil, 0) itersToUnion = bitmapIters[i:] } else { // either N will be small or iContainer is a // bitmap, so we can skip one union op by copying it. + // And we can just freeze it, and the copy will + // happen later if it's needed... statsHit("unionInPlace/clone") - tContainer = iContainer.Clone() + tContainer = iContainer.Freeze() itersToUnion = bitmapIters[i+1:] } } else { @@ -753,13 +818,13 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // convert it preemptively, because union into a // bitmap is nearly always faster. itersToUnion = bitmapIters[i:] - if expectedN >= 512 && tContainer.typ != containerBitmap { + if expectedN >= 512 && tContainer.typ() != containerBitmap { statsHit("unionInPlace/convertToBitmap") - switch tContainer.typ { + switch tContainer.typ() { case containerArray: - tContainer.arrayToBitmap() + tContainer = tContainer.arrayToBitmap() case containerRun: - tContainer.runToBitmap() + tContainer = tContainer.runToBitmap() } } } @@ -770,6 +835,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { jKey, jContainer := iter.iter.Value() if iKey == jKey { + tContainer = tContainer.Thaw() tContainer.unionInPlace(jContainer) // "iter" is a local copy from the range // loop, not the actual slice member. @@ -806,7 +872,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.Containers.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Freeze()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { @@ -833,11 +899,11 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.Containers.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Freeze()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.Containers.Put(kj, cj.Clone()) + output.Containers.Put(kj, cj.Freeze()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -865,7 +931,7 @@ func (b *Bitmap) Shift(n int) (*Bitmap, error) { if lastCarry { o.add(0) } - if o.n > 0 { + if o.N() > 0 { output.Containers.Put(ki, o) } lastCarry = carry @@ -886,7 +952,7 @@ func (b *Bitmap) removeEmptyContainers() { citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() - if c.n == 0 { + if c.N() == 0 { b.Containers.Remove(k) } } @@ -896,7 +962,7 @@ func (b *Bitmap) countEmptyContainers() int { citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - if c.n == 0 { + if c.N() == 0 { result++ } } @@ -987,10 +1053,10 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { // TODO: instead of commenting this out, we need to make it a configuration option //count := c.count() //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) - if c.n > 0 { + if c.N() > 0 { ew.WriteUint64(byte8, key) - ew.WriteUint16(byte2, uint16(c.typ)) - ew.WriteUint16(byte2, uint16(c.n-1)) + ew.WriteUint16(byte2, uint16(c.typ())) + ew.WriteUint16(byte2, uint16(c.N()-1)) } } @@ -1001,7 +1067,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { citer, _ = b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - if c.n > 0 { + if c.N() > 0 { ew.WriteUint32(byte4, offset) offset += uint32(c.size()) } @@ -1017,7 +1083,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { citer, _ = b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - if c.n > 0 { + if c.N() > 0 { nn, err := c.WriteTo(w) n += nn if err != nil { @@ -1074,13 +1140,17 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Map byte slice directly to the container data. citer.Next() _, c := citer.Value() - switch c.typ { + // this shouldn't happen, since we don't normally store nils. + if c == nil { + continue + } + switch c.typ() { case containerRun: runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount]) opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size case containerArray: - c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n:c.n]) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) @@ -1228,6 +1298,11 @@ func (itr *Iterator) Seek(seek uint64) { return // eof } itr.key, itr.c = itr.citer.Value() + if roaringParanoia { + if itr.c == nil { + panic("seeking iterator got a nil container when Next() was true") + } + } // Move to the correct value index inside the container. lb := lowbits(seek) @@ -1283,7 +1358,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Iterate over containers until we find the next value or EOF. for { if itr.c.isArray() { - if itr.j >= itr.c.n-1 { + if itr.j >= itr.c.N()-1 { // Reached end of array, move to the next container. if !itr.citer.Next() { itr.c = nil @@ -1419,6 +1494,9 @@ func (c *Container) count() (n int32) { // countRange counts the number of bits set between [start, end). func (c *Container) countRange(start, end int32) (n int32) { + if c == nil { + return 0 + } if c.isArray() { return c.arrayCountRange(start, end) } else if c.isRun() { @@ -1503,73 +1581,78 @@ func (c *Container) runCountRange(start, end int32) (n int32) { } // add adds a value to the container. -func (c *Container) add(v uint16) (added bool) { - +func (c *Container) add(v uint16) (newC *Container, added bool) { + if c == nil { + return NewContainerArray([]uint16{v}), true + } if c.isArray() { - added = c.arrayAdd(v) + return c.arrayAdd(v) } else if c.isRun() { - added = c.runAdd(v) + return c.runAdd(v) } else { - added = c.bitmapAdd(v) + return c.bitmapAdd(v) } - if added { - c.n++ - } - return added } -func (c *Container) arrayAdd(v uint16) bool { +func (c *Container) arrayAdd(v uint16) (*Container, bool) { // Optimize appending to the end of an array container. array := c.array() - if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && array[c.n-1] < v { + if c.N() > 0 && c.N() < ArrayMaxSize && c.isArray() && array[c.N()-1] < v { statsHit("arrayAdd/append") - c.unmapArray() + c = c.Thaw() array = append(c.array(), v) c.setArray(array) - return true + return c, true } // Find index of the integer in the container. Exit if it already exists. i := search32(array, v) if i >= 0 { - return false + return c, false } // Convert to a bitmap container if too many values are in an array container. - if c.n >= ArrayMaxSize { + if c.N() >= ArrayMaxSize { statsHit("arrayAdd/arrayToBitmap") - c.arrayToBitmap() + c = c.arrayToBitmap() return c.bitmapAdd(v) } // Otherwise insert into array. statsHit("arrayAdd/insert") - c.unmapArray() + c = c.Thaw() i = -i - 1 array = append(c.array(), 0) copy(array[i+1:], array[i:]) array[i] = v c.setArray(array) - return true + return c, true } -func (c *Container) bitmapAdd(v uint16) bool { - if c.bitmapContains(v) { - return false +func (c *Container) bitmapAdd(v uint16) (*Container, bool) { + if c == nil { + c = NewContainerBitmapN(nil, 1) + c.bitmap()[v/64] |= (1 << uint64(v%64)) + return c, true } - c.unmapBitmap() + if c.bitmapContains(v) { + return c, false + } + c = c.Thaw() c.bitmap()[v/64] |= (1 << uint64(v%64)) - return true + c.setN(c.N() + 1) + return c, true } -func (c *Container) runAdd(v uint16) bool { +func (c *Container) runAdd(v uint16) (*Container, bool) { runs := c.runs() if len(runs) == 0 { - c.unmapRun() + c = c.Thaw() c.setRuns([]interval16{{start: v, last: v}}) - return true + c.setN(1) + return c, true } i := sort.Search(len(runs), @@ -1581,10 +1664,10 @@ func (c *Container) runAdd(v uint16) bool { iv := runs[i] if v >= iv.start && iv.last >= v { - return false + return c, false } - c.unmapRun() + c = c.Thaw() runs = c.runs() if iv.last < v { if iv.last == v-1 { @@ -1598,7 +1681,8 @@ func (c *Container) runAdd(v uint16) bool { runs[i-1].last = iv.last runs = append(runs[:i], runs[i+1:]...) c.setRuns(runs) - return true + c.setN(c.N() + 1) + return c, true } // just before an interval runs[i].start-- @@ -1611,11 +1695,15 @@ func (c *Container) runAdd(v uint16) bool { runs = append(runs[:i], append([]interval16{newIv}, runs[i:]...)...) } c.setRuns(runs) - return true + c.setN(c.N() + 1) + return c, true } // Contains returns true if v is in the container. func (c *Container) Contains(v uint16) bool { + if c == nil { + return false + } if c.isArray() { return c.arrayContains(v) } else if c.isRun() { @@ -1669,17 +1757,17 @@ func (c *Container) countRuns() (r int32) { // optimize converts the container to the type which will take up the least // amount of space. -func (c *Container) optimize() { - if c.n == 0 { +func (c *Container) optimize() *Container { + if c.N() == 0 { statsHit("optimize/empty") - return + return nil } runs := c.countRuns() var newType byte - if runs <= runMaxSize && runs <= c.n/2 { + if runs <= runMaxSize && runs <= c.N()/2 { newType = containerRun - } else if c.n < ArrayMaxSize { + } else if c.N() < ArrayMaxSize { newType = containerArray } else { newType = containerBitmap @@ -1689,75 +1777,94 @@ func (c *Container) optimize() { if c.isArray() { if newType == containerBitmap { statsHit("optimize/arrayToBitmap") - c.arrayToBitmap() + c = c.arrayToBitmap() } else if newType == containerRun { statsHit("optimize/arrayToRun") - c.arrayToRun(runs) + c = c.arrayToRun(runs) } else { statsHit("optimize/arrayUnchanged") } } else if c.isBitmap() { if newType == containerArray { statsHit("optimize/bitmapToArray") - c.bitmapToArray() + c = c.bitmapToArray() } else if newType == containerRun { statsHit("optimize/bitmapToRun") - c.bitmapToRun(runs) + c = c.bitmapToRun(runs) } else { statsHit("optimize/bitmapUnchanged") } } else if c.isRun() { if newType == containerBitmap { statsHit("optimize/runToBitmap") - c.runToBitmap() + c = c.runToBitmap() } else if newType == containerArray { statsHit("optimize/runToArray") - c.runToArray() + c = c.runToArray() } else { statsHit("optimize/runUnchanged") } } + return c } // unionInPlace does not necessarily preserve container's N; it's expected // to be used when running a sequence of unions, after which you should // call Repair(). (As of this writing, that only matters for bitmaps.) -func (c *Container) unionInPlace(other *Container) { - switch c.typ { +// +// If called on a frozen container, or a container of the wrong sort, +// it is possible that the returned container will not actually be the +// original container; in-place is a suggestion. +func (c *Container) unionInPlace(other *Container) *Container { + if c == nil { + return other.Freeze() + } + if other == nil { + return c + } + // short-circuit the trivial cases + if c.N() == maxContainerVal+1 || other.N() == maxContainerVal+1 { + return fullContainer + } + switch c.typ() { case containerBitmap: - switch other.typ { + switch other.typ() { case containerBitmap: - unionBitmapBitmapInPlace(c, other) + return unionBitmapBitmapInPlace(c, other) case containerArray: - unionBitmapArrayInPlace(c, other) + return unionBitmapArrayInPlace(c, other) case containerRun: - unionBitmapRunInPlace(c, other) + return unionBitmapRunInPlace(c, other) } case containerArray: - switch other.typ { + switch other.typ() { case containerBitmap: - c.arrayToBitmap() - unionBitmapBitmapInPlace(c, other) + c = c.arrayToBitmap() + return unionBitmapBitmapInPlace(c, other) case containerArray: - unionArrayArrayInPlace(c, other) + return unionArrayArrayInPlace(c, other) case containerRun: - c.arrayToBitmap() - unionBitmapRunInPlace(c, other) + c = c.arrayToBitmap() + return unionBitmapRunInPlace(c, other) } case containerRun: - switch other.typ { + switch other.typ() { case containerBitmap: - c.runToBitmap() - unionBitmapBitmapInPlace(c, other) + c = c.runToBitmap() + return unionBitmapBitmapInPlace(c, other) case containerArray: - c.runToBitmap() - unionBitmapArrayInPlace(c, other) + c = c.runToBitmap() + return unionBitmapArrayInPlace(c, other) case containerRun: - c.runToBitmap() - unionBitmapRunInPlace(c, other) + c = c.runToBitmap() + return unionBitmapRunInPlace(c, other) } } + if roaringParanoia { + panic(fmt.Sprintf("invalid union op: unknown types %d/%d", c.typ(), other.typ())) + } + return c } func (c *Container) arrayContains(v uint16) bool { @@ -1788,58 +1895,71 @@ func (c *Container) runContains(v uint16) bool { } // remove removes a value from the container. -func (c *Container) remove(v uint16) (removed bool) { - if c.isArray() { - removed = c.arrayRemove(v) - } else if c.isRun() { - removed = c.runRemove(v) - } else { - removed = c.bitmapRemove(v) +func (c *Container) remove(v uint16) (newC *Container, removed bool) { + if c == nil { + return nil, false + } + if c.isArray() { + return c.arrayRemove(v) + } else if c.isRun() { + return c.runRemove(v) + } else { + return c.bitmapRemove(v) } - return removed } -func (c *Container) arrayRemove(v uint16) bool { +func (c *Container) arrayRemove(v uint16) (*Container, bool) { array := c.array() i := search32(array, v) if i < 0 { - return false + return c, false } - c.unmapArray() + // removing the last item? we can just return the empty container. + if c.N() == 1 { + return nil, true + } + c = c.Thaw() array = c.array() array = append(array[:i], array[i+1:]...) - c.n-- c.setArray(array) - return true + return c, true } -func (c *Container) bitmapRemove(v uint16) bool { +func (c *Container) bitmapRemove(v uint16) (*Container, bool) { if !c.bitmapContains(v) { - return false + return c, false } - c.unmapBitmap() + // removing the last item? we can just return the empty container. + if c.N() == 1 { + return nil, true + } + c = c.Thaw() // Lower count and remove element. c.bitmap()[v/64] &^= (uint64(1) << uint(v%64)) - c.n-- + c.setN(c.N() - 1) // Convert to array if we go below the threshold. - if c.n == ArrayMaxSize { + if c.N() == ArrayMaxSize { statsHit("bitmapRemove/bitmapToArray") - c.bitmapToArray() + c = c.bitmapToArray() } - return true + return c, true } // runRemove removes v from a run container, and returns true if v was removed. -func (c *Container) runRemove(v uint16) bool { +func (c *Container) runRemove(v uint16) (*Container, bool) { runs := c.runs() i, contains := binSearchRuns(v, runs) if !contains { - return false + return c, false } - c.unmapRun() + // removing the last item? we can just return the empty container. + if c.N() == 1 { + return nil, true + } + c = c.Thaw() runs = c.runs() if v == runs[i].last && v == runs[i].start { runs = append(runs[:i], runs[i+1:]...) @@ -1855,13 +1975,17 @@ func (c *Container) runRemove(v uint16) bool { runs[i+1] = interval16{start: v + 1, last: last} // runs = append(runs[:i+1], append([]interval16{{start: v + 1, last: last}}, runs[i+1:]...)...) } - c.n-- + c.setN(c.N() - 1) c.setRuns(runs) - return true + return c, true } // max returns the maximum value in the container. func (c *Container) max() uint16 { + if c == nil || c.N() == 0 { + // probably wrong, but prevents a crash elsewhere + return 0 + } if c.isArray() { return c.arrayMax() } else if c.isRun() { @@ -1873,9 +1997,6 @@ func (c *Container) max() uint16 { func (c *Container) arrayMax() uint16 { array := c.array() - if len(array) == 0 { - return 0 // probably hiding some ugly bug but it prevents a crash - } return array[len(array)-1] } @@ -1903,26 +2024,33 @@ func (c *Container) runMax() uint16 { } // bitmapToArray converts from bitmap format to array format. -func (c *Container) bitmapToArray() { +func (c *Container) bitmapToArray() *Container { statsHit("bitmapToArray") - bitmap := c.bitmap() - c.setBitmap(nil) - c.typ = containerArray - c.mapped = false - - // return early if empty - if c.n == 0 { - c.setArray(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for bitmapToArray") + } + return nil } + // If c is frozen, we'll be making a new array container. Otherwise, + // we'll convert this container. + if c.N() == 0 { + if c.frozen() { + return NewContainerArray(nil) + } + c.setTyp(containerArray) + c.setArray(nil) + return c + } + bitmap := c.bitmap() n := int32(0) - array := make([]uint16, c.n) + array := make([]uint16, c.N()) for i, word := range bitmap { for word != 0 { t := word & -word if roaringParanoia { - if n >= c.n { + if n >= c.N() { panic("bitmap has more bits set than container.n") } } @@ -1932,71 +2060,112 @@ func (c *Container) bitmapToArray() { } } if roaringParanoia { - if n != c.n { + if n != c.N() { panic("bitmap has fewer bits set than container.n") } } + if c.frozen() { + return NewContainerArray(array) + } + c.setTyp(containerArray) + c.setMapped(false) c.setArray(array) + return c } // arrayToBitmap converts from array format to bitmap format. -func (c *Container) arrayToBitmap() { +func (c *Container) arrayToBitmap() *Container { statsHit("arrayToBitmap") - array := c.array() - c.typ = containerBitmap - bitmap := make([]uint64, bitmapN) - c.setBitmap(bitmap) - c.mapped = false + if c == nil { + if roaringParanoia { + panic("nil container for arrayToBitmap") + } + return nil + } // return early if empty - if c.n == 0 { - return + if c.N() == 0 { + if c.frozen() { + return NewContainerBitmap(0, nil) + } + c.setTyp(containerBitmap) + c.setBitmap(make([]uint64, bitmapN)) + return c } - for _, v := range array { + bitmap := make([]uint64, bitmapN) + for _, v := range c.array() { bitmap[int(v)/64] |= (uint64(1) << uint(v%64)) } + if c.frozen() { + return NewContainerBitmapN(bitmap, c.N()) + } + c.setTyp(containerBitmap) + c.setMapped(false) + c.setBitmap(bitmap) + return c } // runToBitmap converts from RLE format to bitmap format. -func (c *Container) runToBitmap() { +func (c *Container) runToBitmap() *Container { statsHit("runToBitmap") - runs := c.runs() - bitmap := make([]uint64, bitmapN) - c.typ = containerBitmap - c.setBitmap(bitmap) - - c.mapped = false - - // return early if empty - if c.n == 0 { - return + if c == nil { + if roaringParanoia { + panic("nil container for runToBitmap") + } + return nil } - for _, r := range runs { + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerBitmap(0, nil) + } + c.setTyp(containerBitmap) + c.setBitmap(make([]uint64, bitmapN)) + return c + } + bitmap := make([]uint64, bitmapN) + for _, r := range c.runs() { // TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits //note v must be int or will overflow for v := int(r.start); v <= int(r.last); v++ { bitmap[v/64] |= (uint64(1) << uint(v%64)) } } + if c.frozen() { + return NewContainerBitmapN(bitmap, c.N()) + } + c.setTyp(containerBitmap) + c.setMapped(false) + c.setBitmap(bitmap) + return c } // bitmapToRun converts from bitmap format to RLE format. -func (c *Container) bitmapToRun(numRuns int32) { +func (c *Container) bitmapToRun(numRuns int32) *Container { statsHit("bitmapToRun") - bitmap := c.bitmap() - c.mapped = false - c.typ = containerRun - // return early if empty - if c.n == 0 { - c.setRuns(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for bitmapToRun") + } + return nil } + + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerRun(nil) + } + c.setTyp(containerRun) + c.setRuns(nil) + return c + } + + bitmap := c.bitmap() if numRuns == 0 { numRuns = bitmapCountRuns(bitmap) } - runs := make([]interval16, 0, numRuns) current := bitmap[0] @@ -2036,20 +2205,37 @@ func (c *Container) bitmapToRun(numRuns int32) { // pad LSBs with 0s current = current & (current + 1) } + if c.frozen() { + return NewContainerRunN(runs, c.N()) + } + c.setTyp(containerRun) c.setRuns(runs) + c.setMapped(false) + return c } // arrayToRun converts from array format to RLE format. -func (c *Container) arrayToRun(numRuns int32) { +func (c *Container) arrayToRun(numRuns int32) *Container { statsHit("arrayToRun") - array := c.array() - c.typ = containerRun - c.mapped = false - // return early if empty - if c.n == 0 { - c.setRuns(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for arrayToRun") + } + return nil } + + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerRun(nil) + } + c.setTyp(containerRun) + c.setRuns(nil) + return c + } + + array := c.array() + if numRuns == 0 { numRuns = arrayCountRuns(array) } @@ -2064,24 +2250,39 @@ func (c *Container) arrayToRun(numRuns int32) { } } // append final run - runs = append(runs, interval16{start, array[c.n-1]}) + runs = append(runs, interval16{start, array[c.N()-1]}) + if c.frozen() { + return NewContainerRunN(runs, c.N()) + } + c.setTyp(containerRun) + c.setMapped(false) c.setRuns(runs) + return c } // runToArray converts from RLE format to array format. -func (c *Container) runToArray() { +func (c *Container) runToArray() *Container { statsHit("runToArray") - runs := c.runs() - c.typ = containerArray - c.mapped = false - - // return early if empty - if c.n == 0 { - c.setArray(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for runToArray") + } + return nil } - array := make([]uint16, c.n) + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerArray(nil) + } + c.setTyp(containerArray) + c.setArray(nil) + return c + } + + runs := c.runs() + + array := make([]uint16, c.N()) n := int32(0) for _, r := range runs { for v := int(r.start); v <= int(r.last); v++ { @@ -2090,38 +2291,43 @@ func (c *Container) runToArray() { } } if roaringParanoia { - if n != c.n { + if n != c.N() { panic("run has fewer bits set than container.n") } } + if c.frozen() { + return NewContainerArray(array) + } + c.setTyp(containerArray) + c.setMapped(false) c.setArray(array) + return c } // Clone returns a copy of c. func (c *Container) Clone() (out *Container) { statsHit("Container/Clone") - switch c.typ { + if c == nil { + return nil + } + switch c.typ() { case containerArray: statsHit("Container/Clone/Array") - cArray := c.array() - array := make([]uint16, len(cArray)) - copy(array, cArray) - out = NewContainerArray(array) + out = NewContainerArrayCopy(c.array()) case containerBitmap: statsHit("Container/Clone/Bitmap") - other := NewContainerBitmap(c.n, nil) + other := NewContainerBitmapN(nil, c.N()) copy(other.bitmap(), c.bitmap()) out = other case containerRun: statsHit("Container/Clone/Run") - cRuns := c.runs() - runs := make([]interval16, len(cRuns)) - copy(runs, cRuns) - out = NewContainerRun(runs) + out = NewContainerRunCopy(c.runs()) + default: + panic(fmt.Sprintf("cloning a container of unknown type %d", c.typ())) } // this should probably never happen if roaringParanoia { - if out.n != out.count() { + if out.N() != out.count() { panic("cloned container has wrong n") } } @@ -2130,6 +2336,9 @@ func (c *Container) Clone() (out *Container) { // WriteTo writes c to w. func (c *Container) WriteTo(w io.Writer) (n int64, err error) { + if c == nil { + return 0, nil + } if c.isArray() { return c.arrayWriteTo(w) } else if c.isRun() { @@ -2153,7 +2362,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { //} // Write sizeof(uint16) * cardinality bytes. - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&array[0]))[: 2*c.n : 2*c.n]) + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&array[0]))[: 2*c.N() : 2*c.N()]) return int64(nn), err } @@ -2194,7 +2403,12 @@ func (c *Container) size() int { // info returns the current stats about the container. func (c *Container) info() containerInfo { - info := containerInfo{N: c.n} + info := containerInfo{N: c.N()} + if c == nil { + info.Type = "nil" + info.Alloc = 0 + return info + } if c.isArray() { info.Type = "array" @@ -2207,7 +2421,7 @@ func (c *Container) info() containerInfo { info.Alloc = len(c.bitmap()) * 8 // sizeof(uint64) } - if c.mapped { + if c.Mapped() { if c.isArray() { info.Pointer = unsafe.Pointer(&c.array()[0]) } else if c.isRun() { @@ -2224,24 +2438,27 @@ func (c *Container) info() containerInfo { func (c *Container) check() error { var a ErrorList + if c == nil { + return nil + } if c.isArray() { array := c.array() - if int32(len(array)) != c.n { - a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.n)) + if int32(len(array)) != c.N() { + a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.N())) } } else if c.isRun() { n := c.runCountRange(0, maxContainerVal+1) - if n != c.n { - a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.n)) + if n != c.N() { + a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.N())) } } else if c.isBitmap() { - if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.n { - a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.n)) + if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.N() { + a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.N())) } } else { a.Append(fmt.Errorf("empty container")) - if c.n != 0 { - a.Append(fmt.Errorf("empty container with nonzero count: n=%d", c.n)) + if c.N() != 0 { + a.Append(fmt.Errorf("empty container with nonzero count: n=%d", c.N())) } } @@ -2254,6 +2471,11 @@ func (c *Container) check() error { // Repair repairs the cardinality of c if it has been corrupted by // optimized operations. func (c *Container) Repair() { + // a frozen container can't have had n or contents changed, so we + // don't need to recount it. + if c.frozen() { + return + } if c.isBitmap() { c.bitmapRepair() } @@ -2271,7 +2493,7 @@ func (c *Container) bitmapRepair() { n += int32(popcount(bitmap[i+2])) n += int32(popcount(bitmap[i+3])) } - c.n = n + c.setN(n) } // containerInfo represents a point-in-time snapshot of container stats. @@ -2299,20 +2521,20 @@ func flipArray(b *Container) *Container { statsHit("flipArray") // TODO: actually implement this x := b.Clone() - x.arrayToBitmap() + x = x.arrayToBitmap() return flipBitmap(x) } func flipBitmap(b *Container) *Container { statsHit("flipBitmap") - other := NewContainerBitmap(0, nil) + other := NewContainerBitmapN(nil, 0) bitmap := b.bitmap() otherBitmap := other.bitmap() for i, word := range bitmap { otherBitmap[i] = ^word } - other.n = other.count() + other.setN(other.count()) return other } @@ -2320,11 +2542,20 @@ func flipRun(b *Container) *Container { statsHit("flipRun") // TODO: actually implement this x := b.Clone() - x.runToBitmap() + x = x.runToBitmap() return flipBitmap(x) } func intersectionCount(a, b *Container) int32 { + if a.N() == maxContainerVal+1 { + return b.N() + } + if b.N() == maxContainerVal+1 { + return a.N() + } + if a.N() == 0 || b.N() == 0 { + return 0 + } if a.isArray() { if b.isArray() { return intersectionCountArrayArray(a, b) @@ -2356,9 +2587,6 @@ func intersectionCountArrayArray(a, b *Container) (n int32) { statsHit("intersectionCount/ArrayArray") ca, cb := a.array(), b.array() na, nb := len(ca), len(cb) - if na == 0 || nb == 0 { - return 0 - } if na > nb { ca, cb = cb, ca na, nb = nb, na // nolint: ineffassign @@ -2458,6 +2686,15 @@ func intersectionCountBitmapBitmap(a, b *Container) (n int32) { } func intersect(a, b *Container) *Container { + if a.N() == maxContainerVal+1 { + return b.Freeze() + } + if b.N() == maxContainerVal+1 { + return a.Freeze() + } + if a.N() == 0 || b.N() == 0 { + return nil + } if a.isArray() { if b.isArray() { return intersectArrayArray(a, b) @@ -2532,6 +2769,7 @@ func intersectRunRun(a, b *Container) *Container { output := NewContainerRun(nil) ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) + n := int32(0) for i, j := 0, 0; i < na && j < nb; { va, vb := ra[i], rb[j] if va.last < vb.start { @@ -2542,24 +2780,25 @@ func intersectRunRun(a, b *Container) *Container { j++ } else if va.last > vb.last && va.start >= vb.start { // |--vb-|-|-va--| - output.n += output.runAppendInterval(interval16{start: va.start, last: vb.last}) + n += output.runAppendInterval(interval16{start: va.start, last: vb.last}) j++ } else if va.last > vb.last && va.start < vb.start { // |--va|--vb--|--| - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } else if va.last <= vb.last && va.start >= vb.start { // |--vb|--va--|--| - output.n += output.runAppendInterval(va) + n += output.runAppendInterval(va) i++ } else if va.last <= vb.last && va.start < vb.start { // |--va-|-|-vb--| - output.n += output.runAppendInterval(interval16{start: vb.start, last: va.last}) + n += output.runAppendInterval(interval16{start: vb.start, last: va.last}) i++ } } + output.setN(n) runs := output.runs() - if output.n < ArrayMaxSize && int32(len(runs)) > output.n/2 { + if n < ArrayMaxSize && int32(len(runs)) > n/2 { output.runToArray() } else if len(runs) > runMaxSize { output.runToBitmap() @@ -2573,9 +2812,9 @@ func intersectBitmapRun(a, b *Container) *Container { statsHit("intersect/BitmapRun") var output *Container runs := b.runs() - if b.n <= ArrayMaxSize || a.n <= ArrayMaxSize { + if b.N() <= ArrayMaxSize || a.N() <= ArrayMaxSize { // output is array container - array := make([]uint16, 0, b.n) + array := make([]uint16, 0, b.N()) for _, iv := range runs { for i := iv.start; i <= iv.last; i++ { if a.bitmapContains(i) { @@ -2596,6 +2835,7 @@ func intersectBitmapRun(a, b *Container) *Container { output = NewContainerBitmap(0, nil) bitmap := output.bitmap() aBitmap := a.bitmap() + n := int32(0) for j := 0; j < len(runs); j++ { vb := runs[j] i := vb.start >> 6 // index into a @@ -2604,22 +2844,22 @@ func intersectBitmapRun(a, b *Container) *Container { for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b bitmap[i] = aBitmap[i] - output.n += int32(popcount(aBitmap[i])) + n += int32(popcount(aBitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) bits := aBitmap[i] & mask bitmap[i] |= bits - output.n += int32(popcount(bits)) + n += int32(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) bits := (aBitmap[i] >> offset) << offset bitmap[i] |= bits - output.n += int32(popcount(bits)) + n += int32(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) bits := (aBitmap[i] << offset) >> offset bitmap[i] |= bits - output.n += int32(popcount(bits)) + n += int32(popcount(bits)) } // update loop vars i++ @@ -2627,6 +2867,7 @@ func intersectBitmapRun(a, b *Container) *Container { valast = vastart + 63 } } + output.setN(n) } return output } @@ -2662,11 +2903,14 @@ func intersectBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) + output := NewContainerBitmapN(ob, n) return output } func union(a, b *Container) *Container { + if a.N() == maxContainerVal+1 || b.N() == maxContainerVal+1 { + return fullContainer + } if a.isArray() { if b.isArray() { return unionArrayArray(a, b) @@ -2736,7 +2980,7 @@ func unionArrayArray(a, b *Container) *Container { // unionArrayArrayInPlace does what it sounds like -- tries to combine // the two arrays in-place. It does not try to ensure that the result is // of a good array size, so it could be up to twice that size, temporarily. -func unionArrayArrayInPlace(a, b *Container) { +func unionArrayArrayInPlace(a, b *Container) *Container { statsHit("union/ArrayArrayInPlace") aa, ab := a.array(), b.array() na, nb := len(aa), len(ab) @@ -2771,25 +3015,26 @@ func unionArrayArrayInPlace(a, b *Container) { j++ } } - a.setArray(output[:outN]) - a.n = int32(outN) - if a.n > ArrayMaxSize { - a.optimize() + // a union can't omit anything that was previously in a, so if + // the output is the same length, nothing changed. + if len(output) != int(a.N()) { + a = a.Thaw() + a.setArray(output[:outN]) + a = a.optimize() } + return a } // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *Container) *Container { statsHit("union/ArrayRun") - if b.n == maxContainerVal+1 { - return b.Clone() - } output := NewContainerRun(nil) aa, rb := a.array(), b.runs() na, nb := len(aa), len(rb) var vb interval16 var va uint16 + n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na { va = aa[i] @@ -2798,17 +3043,18 @@ func unionArrayRun(a, b *Container) *Container { vb = rb[j] } if i < na && (j >= nb || va < vb.start) { - output.n += output.runAppendInterval(interval16{start: va, last: va}) + n += output.runAppendInterval(interval16{start: va, last: va}) i++ } else { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } } - if output.n < ArrayMaxSize { - output.runToArray() + output.setN(n) + if n < ArrayMaxSize { + output = output.runToArray() } else if len(output.runs()) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -2845,16 +3091,11 @@ func (c *Container) runAppendInterval(v interval16) int32 { func unionRunRun(a, b *Container) *Container { statsHit("union/RunRun") - if a.n == maxContainerVal+1 { - return a.Clone() - } - if b.n == maxContainerVal+1 { - return b.Clone() - } ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) output := NewContainerRun(make([]interval16, 0, na+nb)) var va, vb interval16 + n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na { va = ra[i] @@ -2863,13 +3104,14 @@ func unionRunRun(a, b *Container) *Container { vb = rb[j] } if i < na && (j >= nb || va.start < vb.start) { - output.n += output.runAppendInterval(va) + n += output.runAppendInterval(va) i++ } else { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } } + output.setN(n) if len(output.runs()) > runMaxSize { output.runToBitmap() } @@ -2878,55 +3120,52 @@ func unionRunRun(a, b *Container) *Container { func unionBitmapRun(a, b *Container) *Container { statsHit("union/BitmapRun") - if b.n == maxContainerVal+1 { - return b.Clone() - } - if a.n == maxContainerVal+1 { - return a.Clone() - } output := a.Clone() - bitmap := output.bitmap() for _, run := range b.runs() { - output.bitmapSetRange(bitmap, uint64(run.start), uint64(run.last)+1) + output.bitmapSetRange(uint64(run.start), uint64(run.last)+1) } return output } // unions the run b into the bitmap a, mutating a in place. The n value of // a will need to be repaired after the fact. -func unionBitmapRunInPlace(a, b *Container) { - a.unmapBitmap() +func unionBitmapRunInPlace(a, b *Container) *Container { + a = a.Thaw() bitmap := a.bitmap() statsHit("union/BitmapRun") for _, run := range b.runs() { bitmapSetRangeIgnoreN(bitmap, uint64(run.start), uint64(run.last)+1) } + return a } const maxBitmap = 0xFFFFFFFFFFFFFFFF // sets all bits in [i, j) (c must be a bitmap container, and bitmap must // be its bitmap). -func (c *Container) bitmapSetRange(bitmap []uint64, i, j uint64) { +func (c *Container) bitmapSetRange(i, j uint64) { + bitmap := c.bitmap() x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) xcnt := popcount(X) ycnt := popcount(Y) + n := int32(c.N()) if x == y { - c.n += int32((j - i) - popcount(bitmap[x]&(X&Y))) + n += int32((j - i) - popcount(bitmap[x]&(X&Y))) bitmap[x] |= (X & Y) } else { - c.n += int32(xcnt - popcount(bitmap[x]&X)) + n += int32(xcnt - popcount(bitmap[x]&X)) bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int32(64 - popcount(bitmap[i])) + n += int32(64 - popcount(bitmap[i])) bitmap[i] = maxBitmap } - c.n += int32(ycnt - popcount(bitmap[y]&Y)) + n += int32(ycnt - popcount(bitmap[y]&Y)) bitmap[y] |= Y } + c.setN(n) } // sets all bits in [i, j) without updating any corresponding n value. @@ -2954,23 +3193,25 @@ func (c *Container) bitmapXorRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) bitmap := c.bitmap() + n := c.N() if x == y { cnt := popcount(bitmap[x]) bitmap[x] ^= (X & Y) //// flip - c.n += int32(popcount(bitmap[x]) - cnt) + n += int32(popcount(bitmap[x]) - cnt) } else { cnt := popcount(bitmap[x]) bitmap[x] ^= X - c.n += int32(popcount(bitmap[x]) - cnt) + n += int32(popcount(bitmap[x]) - cnt) for i := x + 1; i < y; i++ { cnt = popcount(bitmap[i]) bitmap[i] ^= maxBitmap - c.n += int32(popcount(bitmap[i]) - cnt) + n += int32(popcount(bitmap[i]) - cnt) } cnt = popcount(bitmap[y]) bitmap[y] ^= Y - c.n += int32(popcount(bitmap[y]) - cnt) + n += int32(popcount(bitmap[y]) - cnt) } + c.setN(n) } // zeroes all bits in [i, j) (c must be a bitmap container) @@ -2980,26 +3221,34 @@ func (c *Container) bitmapZeroRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) bitmap := c.bitmap() + n := c.N() if x == y { - c.n -= int32(popcount(bitmap[x] & (X & Y))) + n -= int32(popcount(bitmap[x] & (X & Y))) bitmap[x] &= ^(X & Y) } else { - c.n -= int32(popcount(bitmap[x] & X)) + n -= int32(popcount(bitmap[x] & X)) bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int32(popcount(bitmap[i])) + n -= int32(popcount(bitmap[i])) bitmap[i] = 0 } - c.n -= int32(popcount(bitmap[y] & Y)) + n -= int32(popcount(bitmap[y] & Y)) bitmap[y] &= ^Y } + c.setN(n) } +// equals reports whether two containers are equal. func (c *Container) equals(c2 *Container) bool { - if c.mapped != c2.mapped || c.typ != c2.typ || c.n != c2.n { + if c == nil || c2 == nil { + if c != c2 { + return false + } + } + if c.Mapped() != c2.Mapped() || c.typ() != c2.typ() || c.N() != c2.N() { return false } - if c.typ == containerArray { + if c.typ() == containerArray { ca, c2a := c.array(), c2.array() if len(ca) != len(c2a) { return false @@ -3009,7 +3258,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.typ == containerBitmap { + } else if c.typ() == containerBitmap { cb, c2b := c.bitmap(), c2.bitmap() if len(cb) != len(c2b) { return false @@ -3019,7 +3268,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.typ == containerRun { + } else if c.typ() == containerRun { cr, c2r := c.runs(), c2.runs() if len(cr) != len(c2r) { return false @@ -3030,7 +3279,7 @@ func (c *Container) equals(c2 *Container) bool { } } } else { - panic(fmt.Sprintf("unknown container type: %v", c.typ)) + panic(fmt.Sprintf("unknown container type: %v", c.typ())) } return true } @@ -3038,23 +3287,26 @@ func (c *Container) equals(c2 *Container) bool { func unionArrayBitmap(a, b *Container) *Container { output := b.Clone() bitmap := output.bitmap() + n := output.N() for _, v := range a.array() { if !output.bitmapContains(v) { bitmap[v/64] |= (1 << uint64(v%64)) - output.n++ + n++ } } + output.setN(n) return output } // unions array b into bitmap a, mutating a in place. The n value // of a will need to be repaired after the fact. -func unionBitmapArrayInPlace(a, b *Container) { - a.unmapBitmap() +func unionBitmapArrayInPlace(a, b *Container) *Container { + a = a.Thaw() bitmap := a.bitmap() for _, v := range b.array() { bitmap[v>>6] |= (uint64(1) << (v % 64)) } + return a } func unionBitmapBitmap(a, b *Container) *Container { @@ -3074,15 +3326,14 @@ func unionBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) + output := NewContainerBitmapN(ob, n) return output } // unions bitmap b into bitmap a, mutating a in place. The n value of // a will need to be repaired after the fact. -func unionBitmapBitmapInPlace(a, b *Container) { - - a.unmapBitmap() +func unionBitmapBitmapInPlace(a, b *Container) *Container { + a = a.Thaw() // local variables added to prevent BCE checks in loop // see https://go101.org/article/bounds-check-elimination.html @@ -3099,9 +3350,16 @@ func unionBitmapBitmapInPlace(a, b *Container) { ab[i+2] |= bb[i+2] ab[i+3] |= bb[i+3] } + return a } func difference(a, b *Container) *Container { + if a.N() == 0 || b.N() == maxContainerVal+1 { + return nil + } + if b.N() == 0 { + return a.Freeze() + } if a.isArray() { if b.isArray() { return differenceArrayArray(a, b) @@ -3160,12 +3418,7 @@ func differenceArrayArray(a, b *Container) *Container { func differenceArrayRun(a, b *Container) *Container { statsHit("difference/ArrayRun") // func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { - - if a.n == 0 || b.n == 0 { - return a.Clone() - } - - output := NewContainerArray(make([]uint16, 0, a.n)) + output := make([]uint16, 0, a.N()) // cardinality upper bound: card(A) i := 0 // array index @@ -3173,11 +3426,11 @@ func differenceArrayRun(a, b *Container) *Container { aa, rb := a.array(), b.runs() // handle overlap - for i < int(a.n) { + for i < len(aa) { // keep all array elements before beginning of runs if aa[i] < rb[j].start { - output.add(aa[i]) + output = append(output, aa[i]) i++ continue } @@ -3201,29 +3454,14 @@ func differenceArrayRun(a, b *Container) *Container { // keep all array elements after end of runs // It's possible that output was converted from array to bitmap in output.add() // so check container type before proceeding. - if output.typ == containerArray { - array := output.array() - array = append(array, aa[i:]...) - output.setArray(array) - // TODO: consider handling container.n mutations in one place - // like we do with container.add(). - output.n += int32(len(aa[i:])) - } else { - for _, v := range aa[i:] { - output.add(v) - } - } + output = append(output, aa[i:]...) } - return output + return NewContainerArray(output) } // differenceBitmapRun computes the difference of an bitmap from a run. func differenceBitmapRun(a, b *Container) *Container { statsHit("difference/BitmapRun") - if a.n == 0 || b.n == 0 { - return a.Clone() - } - output := a.Clone() for _, run := range b.runs() { output.bitmapZeroRange(uint64(run.start), uint64(run.last)+1) @@ -3235,9 +3473,6 @@ func differenceBitmapRun(a, b *Container) *Container { // container. func differenceRunArray(a, b *Container) *Container { statsHit("difference/RunArray") - if a.n == 0 || b.n == 0 { - return a.Clone() - } ra, ab := a.runs(), b.array() runs := make([]interval16, 0, len(ra)) @@ -3296,18 +3531,12 @@ func differenceRunBitmap(a, b *Container) *Container { if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 { return flipBitmap(b) } - output := NewContainerRun(nil) - runs := output.runs() - if len(ra) == 0 { - return NewContainerRun(nil) - } - output.n = a.n + runs := make([]interval16, 0, len(ra)) for _, inputRun := range ra { run := inputRun add := true for bit := inputRun.start; bit <= inputRun.last; bit++ { if b.bitmapContains(bit) { - output.n-- if run.start == bit { if bit == 65535 { //overflow add = false @@ -3340,11 +3569,11 @@ func differenceRunBitmap(a, b *Container) *Container { } } - output.setRuns(runs) - if output.n < ArrayMaxSize && int32(len(runs)) > output.n/2 { - output.runToArray() + output := NewContainerRun(runs) + if output.N() < ArrayMaxSize && int32(len(runs)) > output.N()/2 { + output = output.runToArray() } else if len(runs) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -3352,9 +3581,6 @@ func differenceRunBitmap(a, b *Container) *Container { // differenceRunRun computes the difference of two runs. func differenceRunRun(a, b *Container) *Container { statsHit("difference/RunRun") - if a.n == 0 || b.n == 0 { - return a.Clone() - } ra, rb := a.runs(), b.runs() apos := 0 // current a-run index @@ -3415,7 +3641,7 @@ func differenceRunRun(a, b *Container) *Container { func differenceArrayBitmap(a, b *Container) *Container { statsHit("difference/ArrayBitmap") - output := make([]uint16, 0, a.n) + output := make([]uint16, 0, a.N()) bitmap := b.bitmap() for _, va := range a.array() { bmidx := va / 64 @@ -3435,14 +3661,16 @@ func differenceBitmapArray(a, b *Container) *Container { output := a.Clone() bitmap := output.bitmap() + n := output.N() for _, v := range b.array() { if output.bitmapContains(v) { bitmap[v/64] &^= (uint64(1) << uint(v%64)) - output.n-- + n-- } } - if output.n < ArrayMaxSize { - output.bitmapToArray() + output.setN(n) + if n < ArrayMaxSize { + output = output.bitmapToArray() } return output } @@ -3465,14 +3693,20 @@ func differenceBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) - if output.n < ArrayMaxSize { - output.bitmapToArray() + output := NewContainerBitmapN(ob, n) + if output.N() < ArrayMaxSize { + output = output.bitmapToArray() } return output } func xor(a, b *Container) *Container { + if a.N() == 0 { + return b.Freeze() + } + if b.N() == 0 { + return a.Freeze() + } if a.isArray() { if b.isArray() { return xorArrayArray(a, b) @@ -3502,33 +3736,33 @@ func xor(a, b *Container) *Container { func xorArrayArray(a, b *Container) *Container { statsHit("xor/ArrayArray") - output := NewContainerArray(nil) + output := make([]uint16, 0) aa, ab := a.array(), b.array() na, nb := len(aa), len(ab) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { - output.add(aa[i]) + output = append(output, aa[i]) i++ continue } else if i >= na && j < nb { - output.add(ab[j]) + output = append(output, ab[j]) j++ continue } va, vb := aa[i], ab[j] if va < vb { - output.add(va) + output = append(output, va) i++ } else if va > vb { - output.add(vb) + output = append(output, vb) j++ } else { //== i++ j++ } } - return output + return NewContainerArray(output) } func xorArrayBitmap(a, b *Container) *Container { @@ -3536,16 +3770,16 @@ func xorArrayBitmap(a, b *Container) *Container { output := b.Clone() for _, v := range a.array() { if b.bitmapContains(v) { - output.remove(v) + output, _ = output.remove(v) } else { - output.add(v) + output, _ = output.add(v) } } // It's possible that output was converted from bitmap to array in output.remove() // so we only do this conversion if output is still a bitmap container. - if output.typ == containerBitmap && output.count() < ArrayMaxSize { - output.bitmapToArray() + if output.typ() == containerBitmap && output.count() < ArrayMaxSize { + output = output.bitmapToArray() } return output @@ -3569,9 +3803,9 @@ func xorBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) - if output.count() < ArrayMaxSize { - output.bitmapToArray() + output := NewContainerBitmapN(ob, n) + if n < ArrayMaxSize { + output = output.bitmapToArray() } return output } @@ -3580,6 +3814,9 @@ func xorBitmapBitmap(a, b *Container) *Container { // the new container and a bool indicating whether a // carry bit was shifted out. func shift(c *Container) (*Container, bool) { + if c.N() == 0 { + return nil, false + } if c.isArray() { return shiftArray(c) } else if c.isRun() { @@ -3607,23 +3844,18 @@ func shiftArray(a *Container) (*Container, bool) { // shiftBitmap is a bitmap-specific implementation of shift(). func shiftBitmap(a *Container) (*Container, bool) { statsHit("shift/Bitmap") - carry := false - output := NewContainerBitmap(a.n, nil) + carry := uint64(0) + output := NewContainerBitmapN(nil, 0) ba, bo := a.bitmap(), output.bitmap() - lastCarry := false + lastCarry := uint64(0) for i, v := range ba { - carry = (v & (1 << 63)) != 0 - v = v << 1 - if lastCarry { - v |= 1 - } + carry = v >> 63 + v = v<<1 | lastCarry bo[i] = v lastCarry = carry } - if carry { - output.n-- - } - return output, carry + output.setN(a.N() - int32(carry)) + return output, carry != 0 } // shiftRun is a run-specific implementation of shift(). @@ -3905,6 +4137,7 @@ func xorArrayRun(a, b *Container) *Container { var vb interval16 var va uint16 lastI, lastJ := -1, -1 + n := int32((0)) for i, j := 0, 0; i < na || j < nb; { if i < na && i != lastI { va = aa[i] @@ -3916,14 +4149,14 @@ func xorArrayRun(a, b *Container) *Container { lastJ = j if i < na && (j >= nb || va < vb.start) { //before - output.n += output.runAppendInterval(interval16{start: va, last: va}) + n += output.runAppendInterval(interval16{start: va, last: va}) i++ } else if j < nb && (i >= na || va > vb.last) { //after - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } else if va > vb.start { if va < vb.last { - output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) + n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) i++ vb.start = va + 1 @@ -3931,12 +4164,12 @@ func xorArrayRun(a, b *Container) *Container { j++ } } else if va > vb.last { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } else { // va == vb.last vb.last-- if vb.start <= vb.last { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) } j++ i++ @@ -3954,10 +4187,11 @@ func xorArrayRun(a, b *Container) *Container { i++ } } - if output.n < ArrayMaxSize { - output.runToArray() + output.setN(n) + if n < ArrayMaxSize { + output = output.runToArray() } else if len(output.runs()) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -4061,18 +4295,13 @@ func xorRunRun(a, b *Container) *Container { statsHit("xor/RunRun") ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) - if na == 0 { - return b.Clone() - } - if nb == 0 { - return a.Clone() - } output := NewContainerRun(nil) lastI, lastJ := -1, -1 state := &xorstm{} + n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na && lastI != i { state.va = ra[i] @@ -4087,7 +4316,7 @@ func xorRunRun(a, b *Container) *Container { r1, ok := xorCompare(state) if ok { - output.n += output.runAppendInterval(r1) + n += output.runAppendInterval(r1) } if !state.vaValid { i++ @@ -4099,10 +4328,11 @@ func xorRunRun(a, b *Container) *Container { } l := len(output.runs()) - if output.n < ArrayMaxSize && int32(l) > output.n/2 { - output.runToArray() + output.setN(n) + if n < ArrayMaxSize && int32(l) > n/2 { + output = output.runToArray() } else if l > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -4289,13 +4519,13 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error { // Map byte slice directly to the container data. citer.Next() _, c := citer.Value() - switch c.typ { + switch c.typ() { case containerArray: - c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n:c.n]) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) default: - return fmt.Errorf("unsupported container type %d", c.typ) + return fmt.Errorf("unsupported container type %d", c.typ()) } } return nil @@ -4306,7 +4536,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { for i := 0; i < int(keyN); i++ { citer.Next() _, c := citer.Value() - switch c.typ { + switch c.typ() { case containerRun: runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount:runCount]) @@ -4317,8 +4547,8 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { } pos += int((runCount * interval16Size) + runCountHeaderSize) case containerArray: - c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n:c.n]) - pos += int(c.n * 2) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.N():c.N()]) + pos += int(c.N() * 2) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[pos]))[:bitmapN:bitmapN]) pos += bitmapN * 8 @@ -4373,9 +4603,9 @@ func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummarySta if key == currKey { summary.c++ - summary.n += int64(currContainer.n) + summary.n += int64(currContainer.N()) - if currContainer.n == maxContainerVal+1 { + if currContainer.N() == maxContainerVal+1 { summary.hasMaxRange = true summary.n = maxContainerVal + 1 return summary diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 8c8bbd9ac..41d4059fb 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -253,8 +253,7 @@ func doContainer(typ byte, data interface{}) *Container { case containerArray: return NewContainerArray(data.([]uint16)) case containerBitmap: - c := NewContainerBitmap(0, data.([]uint64)) - c.n = c.count() + c := NewContainerBitmap(-1, data.([]uint64)) return c case containerRun: return NewContainerRun(data.([]interval16)) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 8c3e7a930..95c8d1b0b 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -32,10 +32,6 @@ func (iv interval16) String() string { return fmt.Sprintf("[%d, %d]", iv.start, iv.last) } -func (c *Container) String() string { - return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%s", c.info().Type, c.n, len(c.array()), len(c.runs()), len(c.bitmap()), containerTypeNames[c.typ]) -} - func TestRunAppendInterval(t *testing.T) { a := NewContainerRun(nil) tests := []struct { @@ -102,15 +98,15 @@ func TestContainerRunAdd(t *testing.T) { {8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}}, } for _, test := range tests { - c.mapped = true - ret := c.add(test.op) - if !ret { + c.setMapped(true) + c, changed := c.add(test.op) + if !changed { t.Fatalf("result of adding new bit should be true: %v", c.runs()) } if !reflect.DeepEqual(c.runs(), test.exp) { t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs(), test.op) } - if c.mapped { + if c.Mapped() { t.Fatalf("container should not be mapped after adding bit %v", test.op) } } @@ -118,14 +114,14 @@ func TestContainerRunAdd(t *testing.T) { func TestContainerRunAdd2(t *testing.T) { c := NewContainerRun(nil) - ret := c.add(0) + c, ret := c.add(0) if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs()) } if !reflect.DeepEqual(c.runs(), []interval16{{start: 0, last: 0}}) { t.Fatalf("should have 1 run of length 1, but have %v", c.runs()) } - ret = c.add(0) + c, ret = c.add(0) if ret { t.Fatalf("result of adding existing bit should be false: %v", c.runs()) } @@ -237,23 +233,23 @@ func TestBitmapCountRange(t *testing.T) { } func TestIntersectionCountArrayBitmap3(t *testing.T) { - a, b := NewContainerBitmap(maxContainerVal+1, getFullBitmap()), NewContainerBitmap(maxContainerVal+1, getFullBitmap()) + a, b := NewContainerBitmapN(getFullBitmap(), maxContainerVal+1), NewContainerBitmapN(getFullBitmap(), maxContainerVal+1) res := intersectBitmapBitmap(a, b) - if res.n != res.count() || res.n != maxContainerVal+1 { - t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != maxContainerVal+1 { + t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) } - a.bitmapToRun(0) + a = a.bitmapToRun(0) res = intersectBitmapRun(b, a) - if res.n != res.count() || res.n != maxContainerVal+1 { - t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != maxContainerVal+1 { + t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) } b.bitmapToRun(0) res = intersectRunRun(a, b) n := intersectionCountRunRun(a, b) - if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int32(n) { - t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != maxContainerVal+1 || res.N() != int32(n) { + t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) } } @@ -321,15 +317,16 @@ func TestRunRemove(t *testing.T) { } for i, test := range tests { - c.mapped = true - ret := c.remove(test.op) + c = c.Freeze() + var ret bool + c, ret = c.remove(test.op) if ret != test.expRet || !reflect.DeepEqual(c.runs(), test.exp) { t.Fatalf("test #%v Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", i, test.op, test.expRet, ret, test.exp, c.runs()) } - if ret && c.mapped { + if ret && c.frozen() { t.Fatalf("test #%v container was not unmapped although bit %v was removed", i, test.op) } - if !ret && !c.mapped { + if !ret && !c.frozen() { t.Fatalf("test #%v container was unmapped although bit %v was not removed", i, test.op) } } @@ -370,7 +367,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap(), b.runs(), ret) } - a = NewContainerBitmap(29, []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}) + a = NewContainerBitmap(-1, []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}) b = NewContainerRun([]interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}) ret = intersectionCountBitmapRun(a, b) @@ -380,8 +377,6 @@ func TestIntersectionCountBitmapRun(t *testing.T) { } func TestIntersectionCountRunRun(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -419,10 +414,8 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { - a.typ = containerRun - b.typ = containerRun - a.setRuns(test.aruns) - b.setRuns(test.bruns) + a := NewContainerRun(test.aruns) + b := NewContainerRun(test.bruns) ret := intersectionCountRunRun(a, b) if ret != test.exp { t.Fatalf("test #%v failed intersecting %v with %v should be %v, but got %v", i, test.aruns, test.bruns, test.exp, ret) @@ -524,8 +517,8 @@ func TestIntersectRunRun(t *testing.T) { a.setRuns(test.aruns) b.setRuns(test.bruns) ret := intersectRunRun(a, b) - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } if test.exp != nil { if !reflect.DeepEqual(ret.runs(), test.exp) { @@ -538,8 +531,6 @@ func TestIntersectRunRun(t *testing.T) { } func TestIntersectBitmapRunBitmap(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -578,13 +569,11 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap(), test.bitmap) - b.setRuns(test.runs) - b.n = 4097 // ;) exp := make([]uint64, bitmapN) copy(exp, test.exp) - a.typ = containerBitmap - b.typ = containerRun + a := NewContainerBitmap(-1, test.bitmap) + b := NewContainerRun(test.runs) + b.setN(4097) ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -592,8 +581,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { if !reflect.DeepEqual(ret.bitmap(), exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap()) } - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } } @@ -646,8 +635,8 @@ func TestIntersectBitmapRunArray(t *testing.T) { if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } } @@ -738,8 +727,7 @@ func TestDifferenceMixed(t *testing.T) { b := NewContainerArray([]uint16{0, 2, 4, 6, 8, 10, 12}) - c := NewContainerBitmap(0, MakeBitmap([]uint64{0x64})) - c.n = c.countRange(0, 100) + c := NewContainerBitmap(-1, MakeBitmap([]uint64{0x64})) d := NewContainerArray([]uint16{1, 3, 5, 7, 9, 11, 12}) @@ -780,13 +768,13 @@ func TestDifferenceMixed(t *testing.T) { } res = difference(b, b) - if res.n != 0 { - t.Fatalf("test #8 expected 0, but got %d", res.n) + if res.N() != 0 { + t.Fatalf("test #8 expected 0, but got %d", res.N()) } res = difference(c, c) - if res.n != 0 { - t.Fatalf("test #9 expected 0, but got %d", res.n) + if res.N() != 0 { + t.Fatalf("test #9 expected 0, but got %d", res.N()) } res = difference(d, b) @@ -901,7 +889,6 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -926,21 +913,18 @@ func TestBitmapSetRange(t *testing.T) { } for i, test := range tests { - bitmap := c.bitmap() - copy(bitmap, test.bitmap) - c.n = c.countRange(0, 65535) - c.bitmapSetRange(bitmap, test.start, test.last+1) + c := NewContainerBitmap(-1, test.bitmap) + c.bitmapSetRange(test.start, test.last+1) if !reflect.DeepEqual(c.bitmap()[:len(test.exp)], test.exp) { t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap()[:len(test.bitmap)]) } - if test.expN != c.n { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + if test.expN != c.N() { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.N()) } } } func TestArrayToBitmap(t *testing.T) { - a := NewContainerArray(nil) tests := []struct { array []uint16 exp []uint64 @@ -958,19 +942,15 @@ func TestArrayToBitmap(t *testing.T) { for i, test := range tests { exp := make([]uint64, bitmapN) copy(exp, test.exp) - - a.setArray(test.array) - a.n = int32(len(test.array)) - a.arrayToBitmap() + a := NewContainerArray(test.array) + a = a.arrayToBitmap() if !reflect.DeepEqual(a.bitmap(), exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap()) } - a.bitmapToArray() } } func TestBitmapToArray(t *testing.T) { - a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []uint16 @@ -985,25 +965,16 @@ func TestBitmapToArray(t *testing.T) { }, } for i, test := range tests { - a.setBitmap(make([]uint64, bitmapN)) - bitmap := a.bitmap() - n := int32(0) - for i, v := range test.bitmap { - bitmap[i] = v - n += int32(popcount(v)) - } - a.n = n + a := NewContainerBitmap(-1, test.bitmap) - a.bitmapToArray() + a = a.bitmapToArray() if !reflect.DeepEqual(a.array(), test.exp) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, a.array()) } - a.arrayToBitmap() } } func TestRunToBitmap(t *testing.T) { - a := NewContainerRun(nil) tests := []struct { runs []interval16 exp []uint64 @@ -1037,11 +1008,8 @@ func TestRunToBitmap(t *testing.T) { exp[i] = v n += int(popcount(v)) } - - a.typ = containerRun - a.setRuns(test.runs) - a.n = int32(n) - a.runToBitmap() + a := NewContainerRun(test.runs) + a = a.runToBitmap() if !reflect.DeepEqual(a.bitmap(), exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap()) } @@ -1058,7 +1026,6 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []interval16 @@ -1116,15 +1083,8 @@ func TestBitmapToRun(t *testing.T) { tests[8].bitmap[1023] = 0xFFFFFFFFFFFFFFFF for i, test := range tests { - a.setBitmap(make([]uint64, bitmapN)) - bitmap := a.bitmap() - n := 0 - for i, v := range test.bitmap { - bitmap[i] = v - n += int(popcount(v)) - } - a.n = int32(n) - x := bitmap + a := NewContainerBitmap(-1, test.bitmap) + x := a.bitmap() a.bitmapToRun(0) if !reflect.DeepEqual(a.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs()) @@ -1137,7 +1097,6 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := NewContainerArray(nil) tests := []struct { array []uint16 exp []interval16 @@ -1161,10 +1120,8 @@ func TestArrayToRun(t *testing.T) { } for i, test := range tests { - a.typ = containerArray - a.setArray(test.array) - a.n = int32(len(test.array)) - a.arrayToRun(0) + a := NewContainerArray(test.array) + a = a.arrayToRun(0) if !reflect.DeepEqual(a.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs()) } @@ -1172,7 +1129,6 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := NewContainerRun(nil) tests := []struct { runs []interval16 exp []uint16 @@ -1196,10 +1152,8 @@ func TestRunToArray(t *testing.T) { } for i, test := range tests { - a.typ = containerRun - a.setRuns(test.runs) - a.n = int32(len(test.exp)) - a.runToArray() + a := NewContainerRun(test.runs) + a = a.runToArray() if !reflect.DeepEqual(a.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array()) } @@ -1207,7 +1161,6 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -1230,17 +1183,16 @@ func TestBitmapZeroRange(t *testing.T) { expN: 13, }, } - bitmap := c.bitmap() for i, test := range tests { - copy(bitmap, test.bitmap) - c.n = c.countRange(0, 65535) + c := NewContainerBitmap(-1, test.bitmap) + bitmap := c.bitmap() c.bitmapZeroRange(test.start, test.last+1) if !reflect.DeepEqual(bitmap[:len(test.exp)], test.exp) { t.Fatalf("test %#v expected %x, got %x", i, test.exp, bitmap[:len(test.bitmap)]) } - if test.expN != c.n { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + if test.expN != c.N() { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.N()) } for i := range test.bitmap { bitmap[i] = 0 @@ -1250,8 +1202,6 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -1266,20 +1216,18 @@ func TestUnionBitmapRun(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap(), test.bitmap) - a.n = a.bitmapCountRange(0, 65535) - b.setRuns(test.runs) - b.n = b.runCountRange(0, 65535) + a := NewContainerBitmap(-1, test.bitmap) + b := NewContainerRun(test.runs) ret := unionBitmapRun(a, b) if ret.isArray() { - ret.arrayToBitmap() + ret = ret.arrayToBitmap() } bitmap := ret.bitmap() if !reflect.DeepEqual(bitmap[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected %x, but got %x", i, test.exp, bitmap[:len(test.exp)]) } - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } for i := range test.bitmap { a.bitmap()[i] = 0 @@ -1288,7 +1236,6 @@ func TestUnionBitmapRun(t *testing.T) { } func TestBitmapCountRuns(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp int32 @@ -1310,10 +1257,10 @@ func TestBitmapCountRuns(t *testing.T) { exp: 10, }, } + var c *Container for i, test := range tests { - copy(c.bitmap(), test.bitmap) - + c = NewContainerBitmap(-1, test.bitmap) ret := c.bitmapCountRuns() if ret != test.exp { t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret) @@ -1377,8 +1324,6 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := NewContainerArray(nil) - b := NewContainerRun(nil) tests := []struct { array []uint16 runs []interval16 @@ -1391,10 +1336,8 @@ func TestDifferenceArrayRun(t *testing.T) { }, } for i, test := range tests { - a.setArray(test.array) - a.n = int32(len(a.array())) - b.setRuns(test.runs) - b.n = b.runCountRange(0, 100) + a := NewContainerArray(test.array) + b := NewContainerRun(test.runs) ret := differenceArrayRun(a, b) if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) @@ -1403,8 +1346,6 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerArray(nil) tests := []struct { runs []interval16 array []uint16 @@ -1457,10 +1398,8 @@ func TestDifferenceRunArray(t *testing.T) { }, } for i, test := range tests { - a.setRuns(test.runs) - a.n = a.runCountRange(0, 100) - b.setArray(test.array) - b.n = int32(len(b.array())) + a := NewContainerRun(test.runs) + b := NewContainerArray(test.array) ret := differenceRunArray(a, b) if !reflect.DeepEqual(ret.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) @@ -1480,8 +1419,6 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerBitmap(0, nil) tests := []struct { runs []interval16 bitmap []uint64 @@ -1529,10 +1466,8 @@ func TestDifferenceRunBitmap(t *testing.T) { }, } for i, test := range tests { - a.setRuns(test.runs) - a.n = a.runCountRange(0, 65536) - copy(b.bitmap(), test.bitmap) - b.n = b.bitmapCountRange(0, 65536) + a := NewContainerRun(test.runs) + b := NewContainerBitmap(-1, test.bitmap) ret := differenceRunBitmap(a, b) if !reflect.DeepEqual(ret.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) @@ -1541,8 +1476,6 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -1610,10 +1543,8 @@ func TestDifferenceBitmapRun(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap(), test.bitmap) - a.n = a.bitmapCountRange(0, 65536) - b.setRuns(test.runs) - b.n = b.runCountRange(0, 65536) + a := NewContainerBitmap(-1, test.bitmap) + b := NewContainerRun(test.runs) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap()[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap()[:len(test.exp)]) @@ -1622,8 +1553,6 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := NewContainerBitmap(0, nil) - a := NewContainerArray(nil) tests := []struct { bitmap []uint64 array []uint16 @@ -1661,9 +1590,8 @@ func TestDifferenceBitmapArray(t *testing.T) { }, } for i, test := range tests { - b.bitmap()[0] = test.bitmap[0] - b.n = b.count() - a.setArray(test.array) + b := NewContainerBitmap(-1, test.bitmap[:1]) + a := NewContainerArray(test.array) ret := differenceBitmapArray(b, a) if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret.array()) @@ -1672,8 +1600,6 @@ func TestDifferenceBitmapArray(t *testing.T) { } func TestDifferenceBitmapBitmap(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerBitmap(0, nil) tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1691,9 +1617,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) { }, } for i, test := range tests { - a.bitmap()[0] = test.abitmap[0] - b.bitmap()[0] = test.bbitmap[0] - + a := NewContainerBitmap(-1, test.abitmap) + b := NewContainerBitmap(-1, test.bbitmap) ret := differenceBitmapBitmap(a, b) if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array()) @@ -1702,8 +1627,6 @@ func TestDifferenceBitmapBitmap(t *testing.T) { } func TestDifferenceRunRun(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -1721,16 +1644,14 @@ func TestDifferenceRunRun(t *testing.T) { }, } for i, test := range tests { - a.setRuns(test.aruns) - a.n = a.runCountRange(0, 100) - b.setRuns(test.bruns) - b.n = b.runCountRange(0, 100) + a := NewContainerRun(test.aruns) + b := NewContainerRun(test.bruns) ret := differenceRunRun(a, b) if !reflect.DeepEqual(ret.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } - if ret.n != test.expn { - t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.n) + if ret.N() != test.expn { + t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.N()) } } } @@ -1756,7 +1677,7 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := NewContainerBitmap(129*32, nil) + cb := NewContainerBitmapN(nil, 129*32) for i := 0; i < 129; i++ { cb.bitmap()[i] = 0x5555555555555555 } @@ -1779,7 +1700,7 @@ func TestWriteReadBitmap(t *testing.T) { func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := NewContainerBitmap(65536, nil) + cb := NewContainerBitmapN(nil, 65536) for i := 0; i < bitmapN; i++ { cb.bitmap()[i] = 0xffffffffffffffff } @@ -1802,11 +1723,11 @@ func TestWriteReadFullBitmap(t *testing.T) { t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap(), bb2.Containers.Get(0).bitmap()) } - if bb2.Containers.Get(0).n != cb.n { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n) + if bb2.Containers.Get(0).N() != cb.N() { + t.Fatalf("bitmap test expected count %x, but got %x", cb.N(), bb2.Containers.Get(0).N()) } if bb2.Containers.Get(0).count() != cb.count() { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n) + t.Fatalf("bitmap test expected count %x, but got %x", cb.N(), bb2.Containers.Get(0).N()) } } @@ -1855,8 +1776,8 @@ func TestXorArrayRun(t *testing.T) { } for i, test := range tests { - test.a.n = test.a.count() - test.b.n = test.b.count() + test.a.setN(test.a.count()) + test.b.setN(test.b.count()) ret := xor(test.a, test.b) if !reflect.DeepEqual(ret.array(), test.exp.array()) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret) @@ -1982,7 +1903,6 @@ func TestXorRunRun(t *testing.T) { } func TestBitmapXorRange(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -2035,14 +1955,13 @@ func TestBitmapXorRange(t *testing.T) { } for i, test := range tests { - copy(c.bitmap(), test.bitmap) - c.n = c.countRange(0, 65535) + c := NewContainerBitmap(-1, test.bitmap) c.bitmapXorRange(test.start, test.last+1) if !reflect.DeepEqual(c.bitmap()[:len(test.exp)], test.exp) { t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap()[:len(test.bitmap)]) } - if test.expN != c.n { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + if test.expN != c.N() { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.N()) } } } @@ -2060,15 +1979,21 @@ func TestXorBitmapRun(t *testing.T) { }, } for i, test := range tests { - a := NewContainerBitmap(0, test.bitmap) - e := NewContainerBitmap(0, test.exp) + a := NewContainerBitmap(-1, test.bitmap) + e := NewContainerBitmap(-1, test.exp) b := NewContainerRun(test.runs) //xorBitmapRun ret := xor(a, b) + if ret.isRun() { + ret = ret.runToBitmap() + } if !reflect.DeepEqual(ret.bitmap(), e.bitmap()) { t.Fatalf("test #%v expected %v, but got %v", i, e.bitmap(), ret.bitmap()) } ret = xor(b, a) + if ret.isRun() { + ret = ret.runToBitmap() + } if !reflect.DeepEqual(ret.bitmap(), e.bitmap()) { t.Fatalf("test #%v.1 expected %v, but got %v", i, e.bitmap(), ret.bitmap()) } @@ -2397,15 +2322,14 @@ func TestRunBinSearch(t *testing.T) { } func TestBitmap_RemoveEmptyContainers(t *testing.T) { bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) - if _, err := bm1.Remove(2 << 16); err != nil { - t.Fatalf("removing a bit: %v", err) - } - if bm1.countEmptyContainers() != 1 { + bm2 := NewFileBitmap(1<<16, 2<<16+1, 3<<16) + bm3 := bm1.Intersect(bm2) + if bm3.countEmptyContainers() != 1 { t.Fatalf("Should be 1 empty container ") } - bm1.removeEmptyContainers() + bm3.removeEmptyContainers() - if bm1.countEmptyContainers() != 0 { + if bm3.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") } } @@ -2510,7 +2434,6 @@ func TestSearch64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := NewContainerArray(nil), NewContainerBitmap(0, nil) tests := []struct { array []uint16 bitmap []uint64 @@ -2554,8 +2477,8 @@ func TestIntersectArrayBitmap(t *testing.T) { } for i, test := range tests { - a.setArray(test.array) - copy(b.bitmap(), test.bitmap) + a := NewContainerArray(test.array) + b := NewContainerBitmap(-1, test.bitmap) ret := intersectArrayBitmap(a, b).array() if len(ret) == 0 && len(test.exp) == 0 { continue @@ -3274,42 +3197,48 @@ func TestContainerCombinations(t *testing.T) { for _, ct := range containerTypes { clone := ret.Clone() if ct == containerArray { - if clone.isBitmap() { - clone.bitmapToArray() + if clone == nil { + clone = NewContainerArray(nil) + } else if clone.isBitmap() { + clone = clone.bitmapToArray() } else if clone.isRun() { - clone.runToArray() + clone = clone.runToArray() } - if clone.n != cts[ct][exp].n { - t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + if clone.N() != cts[ct][exp].N() { + t.Errorf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) } // Because xorRunRun resulting in an empty container returns an array container with a // nil slice array, then we need to check len() on array first (look for 0). if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) { - t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) + t.Errorf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) } } else if ct == containerBitmap { - if clone.isArray() { - clone.arrayToBitmap() + if clone == nil { + clone = NewContainerBitmap(0, nil) + } else if clone.isArray() { + clone = clone.arrayToBitmap() } else if clone.isRun() { - clone.runToBitmap() + clone = clone.runToBitmap() } - if clone.n != cts[ct][exp].n { - t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + if clone.N() != cts[ct][exp].N() { + t.Errorf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) } if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) { - t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) + t.Errorf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) } } else if ct == containerRun { - if clone.isArray() { - clone.arrayToRun(0) + if clone == nil { + clone = NewContainerRun(nil) + } else if clone.isArray() { + clone = clone.arrayToRun(0) } else if clone.isBitmap() { - clone.bitmapToRun(0) + clone = clone.bitmapToRun(0) } - if clone.n != cts[ct][exp].n { - t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + if clone.N() != cts[ct][exp].N() { + t.Errorf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) } if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) { - t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) + t.Errorf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) } } } @@ -3405,7 +3334,6 @@ func TestEquals(t *testing.T) { } */ func TestShiftArray(t *testing.T) { - a := NewContainerArray(nil) tests := []struct { array []uint16 exp []uint16 @@ -3429,10 +3357,16 @@ func TestShiftArray(t *testing.T) { } for i, test := range tests { - a.setArray(test.array) - a.n = int32(len(a.array())) + a := NewContainerArray(test.array) ret1, _ := shift(a) // test generic shift function ret2, _ := shiftArray(a) // test array-specific shift function + // accept nil *Container as valid substitute for empty array + if ret1 == nil { + ret1 = NewContainerArray(nil) + } + if ret2 == nil { + ret2 = NewContainerArray(nil) + } if !reflect.DeepEqual(ret1.array(), test.exp) { t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array()) } else if !reflect.DeepEqual(ret2.array(), test.exp) { @@ -3443,7 +3377,6 @@ func TestShiftArray(t *testing.T) { func TestShiftBitmap(t *testing.T) { // note, bitmaps are provided for us by the ensuing tests - a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []uint64 @@ -3463,11 +3396,10 @@ func TestShiftBitmap(t *testing.T) { } for i, test := range tests { - a.setBitmap(test.bitmap) - a.n = 1 + a := NewContainerBitmap(-1, test.bitmap) ret1, _ := shift(a) // test generic shift function ret2, _ := shiftBitmap(a) // test bitmap-specific shift function - e := NewContainerBitmap(1, test.exp) + e := NewContainerBitmap(-1, test.exp) if !reflect.DeepEqual(ret1.bitmap(), e.bitmap()) { t.Fatalf("test #%v shift() expected %v, but got %v", i, e.bitmap(), ret1.bitmap()) } else if !reflect.DeepEqual(ret2.bitmap(), test.exp) { @@ -3476,8 +3408,6 @@ func TestShiftBitmap(t *testing.T) { } } func TestShiftRun(t *testing.T) { - a := NewContainerRun(nil) - tests := []struct { runs []interval16 n int32 @@ -3509,14 +3439,13 @@ func TestShiftRun(t *testing.T) { } for i, test := range tests { - a.setRuns(test.runs) - a.n = test.n + a := NewContainerRun(test.runs) ret1, c1 := shift(a) // test generic shift function ret2, c2 := shiftRun(a) // test run-specific shift function - if !reflect.DeepEqual(ret1.runs(), test.exp) && c1 == test.carry && ret1.n == test.en { - t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs(), ret1.n) - } else if !reflect.DeepEqual(ret2.runs(), test.exp) && c2 == test.carry && ret2.n == test.en { - t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs(), ret2.n) + if !reflect.DeepEqual(ret1.runs(), test.exp) && c1 == test.carry && ret1.N() == test.en { + t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs(), ret1.N()) + } else if !reflect.DeepEqual(ret2.runs(), test.exp) && c2 == test.carry && ret2.N() == test.en { + t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs(), ret2.N()) } } } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index b4f792629..c6ea63eb0 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1087,14 +1087,15 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { } // Remove all values in random order. - for _, i := range rand.Perm(len(a)) { - removed, _ := bm.Remove(a[i]) + for i, item := range rand.Perm(len(a)) { + removed, _ := bm.Remove(a[item]) if removed { manual_count-- } //check count if manual_count != bm.Count() { - t.Fatalf("expected bitmap Remove count to be: %d got: %d", manual_count, bm.Count()) + t.Fatalf("removing %d/%d [%d] from bitmap: expected bitmap Remove count to be %d, got %d", + i, len(a), a[item], manual_count, bm.Count()) } } diff --git a/row.go b/row.go index a2e938434..79c918311 100644 --- a/row.go +++ b/row.go @@ -57,6 +57,12 @@ func (r *Row) IsEmpty() bool { return true } +func (r *Row) Freeze() { + for _, s := range r.segments { + s.Freeze() + } +} + // Merge merges data from other into r. func (r *Row) Merge(other *Row) { var segments []rowSegment @@ -210,15 +216,6 @@ func (r *Row) SetBit(i uint64) (changed bool) { return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) } -// clearBit clears the i-th column of the row. -func (r *Row) clearBit(i uint64) (changed bool) { // nolint: unparam - s := r.segment(i / ShardWidth) - if s == nil { - return false - } - return s.ClearBit(i) -} - // Segments returns a list of all segments in the row. func (r *Row) Segments() []rowSegment { return r.segments @@ -246,12 +243,12 @@ func (r *Row) createSegmentIfNotExists(shard uint64) *rowSegment { } // Insert new segment. - r.segments = append(r.segments, rowSegment{data: *roaring.NewBitmap()}) + r.segments = append(r.segments, rowSegment{data: roaring.NewSliceBitmap()}) if i < len(r.segments) { copy(r.segments[i+1:], r.segments[i:]) } r.segments[i] = rowSegment{ - data: *roaring.NewBitmap(), + data: roaring.NewSliceBitmap(), shard: shard, writable: true, } @@ -312,13 +309,17 @@ type rowSegment struct { // Underlying raw bitmap implementation. // This is an mmapped bitmap if writable is false. Otherwise // it is a heap allocated bitmap which can be manipulated. - data roaring.Bitmap + data *roaring.Bitmap writable bool // Bit count n uint64 } +func (s *rowSegment) Freeze() { + s.data.Freeze() +} + // Merge adds chunks from other to s. // Chunks in s are overwritten if they exist in other. func (s *rowSegment) Merge(other *rowSegment) { @@ -332,50 +333,58 @@ func (s *rowSegment) Merge(other *rowSegment) { // IntersectionCount returns the number of intersections between s and other. func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { - return s.data.IntersectionCount(&other.data) + return s.data.IntersectionCount(other.data) } // Intersect returns the itersection of s and other. func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { - data := s.data.Intersect(&other.data) + data := s.data.Intersect(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } // Union returns the bitwise union of s and other. func (s *rowSegment) Union(other *rowSegment) *rowSegment { - data := s.data.Union(&other.data) + data := s.data.Union(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } // Difference returns the diff of s and other. func (s *rowSegment) Difference(other *rowSegment) *rowSegment { - data := s.data.Difference(&other.data) + data := s.data.Difference(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } // Xor returns the xor of s and other. func (s *rowSegment) Xor(other *rowSegment) *rowSegment { - data := s.data.Xor(&other.data) + data := s.data.Xor(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } @@ -386,11 +395,13 @@ func (s *rowSegment) Shift() (*rowSegment, error) { if err != nil { return nil, errors.Wrap(err, "shifting roaring data") } + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, }, nil } @@ -439,7 +450,11 @@ func (s *rowSegment) ensureWritable() { return } - s.data = *s.data.Clone() + // This doesn't actually clone all the containers, but does clone + // the bitmap itself -- we get a new bitmap, but it just marks the + // containers as frozen and shares them. It's now safe to write to + // this bitmap, but the actual containers are copy-on-write. + s.data = s.data.Freeze() s.writable = true } From 1515ddaf14b8d71a3af43af28f4dd12bf1db53ea Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 30 May 2019 11:10:03 -0500 Subject: [PATCH 68/73] fixed swapped order of flags and file version bytes on unmarshal also fix tests to use correct flags for bsi fields --- fragment_internal_test.go | 21 +++++++++++++++------ roaring/roaring.go | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index d815f41bd..19dc3f587 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -708,12 +708,12 @@ func BenchmarkFragment_ImportValue(b *testing.B) { depths := []uint{4, 8, 16} for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) b.Run(name+"_Sparse", func(b *testing.B) { benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) }) f.Clean(b) - f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f = mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) b.Run(name+"_Dense", func(b *testing.B) { benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) }) @@ -832,7 +832,7 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { b.Run(fmt.Sprintf("Updates%dVals%dOpN%d", numUpdates, valsPerUpdate, opN), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, CacheTypeNone) + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = opN err := f.importValue(initialCols, initialVals, 21, false) if err != nil { @@ -2457,6 +2457,15 @@ func (f *fragment) CleanKeep(t testing.TB) { // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { + return mustOpenFragmentFlags(index, field, view, shard, cacheType, 0) +} + +func mustOpenBSIFragment(index, field, view string, shard uint64) *fragment { + return mustOpenFragmentFlags(index, field, view, shard, "", 1) +} + +// mustOpenFragment returns a new instance of Fragment with a temporary path. +func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) *fragment { file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-") if err != nil { panic(err) @@ -2467,7 +2476,7 @@ func mustOpenFragment(index, field, view string, shard uint64, cacheType string) cacheType = DefaultCacheType } - f := newFragment(file.Name(), index, field, view, shard, 0) + f := newFragment(file.Name(), index, field, view, shard, flags) f.CacheType = cacheType f.RowAttrStore = &memAttrStore{ store: make(map[uint64]map[string]interface{}), @@ -3227,7 +3236,7 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) { } func TestImportValueConcurrent(t *testing.T) { - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) eg := &errgroup.Group{} for i := 0; i < 4; i++ { i := i @@ -3267,7 +3276,7 @@ func TestImportMultipleValues(t *testing.T) { for i, test := range tests { for _, maxOpN := range []int{0, 10000} { // test small/large write t.Run(fmt.Sprintf("%dLowOpN", i), func(t *testing.T) { - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, CacheTypeNone) + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = maxOpN defer f.Clean(t) err := f.importValue(test.cols, test.vals, test.depth, false) diff --git a/roaring/roaring.go b/roaring/roaring.go index cd7833a75..8930981e0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1103,8 +1103,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion. fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) - b.Flags = data[2] - fileVersion := uint32(data[3]) + fileVersion := uint32(data[2]) + b.Flags = data[3] if fileMagic != MagicNumber { return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) } From 636f1325649e1b505dd6bc6e6c8563da5aed0ac5 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 31 May 2019 15:39:13 -0500 Subject: [PATCH 69/73] add a test case which breaks the rowcache code It turns out that frozen containers which have mmapped data are only safe *until the data gets unmapped*. Which it does on a snapshot. --- fragment_internal_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 19dc3f587..c159bb1a1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -26,6 +26,7 @@ import ( "os" "reflect" "sort" + "sync/atomic" "testing" "testing/quick" @@ -104,6 +105,44 @@ func TestFragment_ClearBit(t *testing.T) { } } +// What about rowcache timing. +func TestFragment_RowcacheMap(t *testing.T) { + var done int64 + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + ch := make(chan struct{}) + + for i := 0; i < f.MaxOpN; i++ { + f.setBit(0, uint64(i*32)) + } + // force snapshot so we get a mmapped row... + f.snapshot() + row := f.row(0) + segment := row.Segments()[0] + bitmap := segment.data + + // request information from the frozen bitmap we got back + go func() { + for atomic.LoadInt64(&done) == 0 { + for i := 0; i < f.MaxOpN; i++ { + _ = bitmap.Contains(uint64(i * 32)) + } + } + close(ch) + }() + + // modify the original bitmap, until it causes a snapshot, which + // then invalidates the other map... + for j := 0; j < 5; j++ { + for i := 0; i < f.MaxOpN; i++ { + f.setBit(0, uint64(i*32+j+1)) + } + } + atomic.StoreInt64(&done, 1) + <-ch +} + // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") From 973579e662da41344880a4ff4a558cd89c94e845 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 31 May 2019 16:17:06 -0500 Subject: [PATCH 70/73] on freeze, unmap mapped containers It turns out that calling syscall.Munmap() is a thing which can change any container holding a pointer into the mapped space, but which wouldn't detect frozen containers. So we need to copy storage for such things. This negates some of the memory wins of the rowcache code, but makes it not crashy. --- roaring/container_stash.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 46725238f..fe03f9d2e 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -256,11 +256,20 @@ func (c *Container) setMapped(mapped bool) { } // Freeze returns an unmodifiable container identical to c. This might -// be c, now marked unmodifiable, or might be a new container. +// be c, now marked unmodifiable, or might be a new container. If c +// is currently marked as "mapped", referring to a backing store that's +// not a conventional Go pointer, the storage may be copied. func (c *Container) Freeze() *Container { if c == nil { return nil } + // don't need to freeze + if c.flags&flagFrozen != 0 { + return c + } + // unmapOrClone should unmap-in-place because the existing + // container isn't frozen (or we'd already have returned it). + c = c.unmapOrClone() c.flags |= flagFrozen return c } From 372c369e7c67f344dca3d6ac55efbfe9e505b39a Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 4 Jun 2019 08:55:35 -0500 Subject: [PATCH 71/73] Optimize needs to use the new container logic When calling `.optimize`, need to grab the new container which may be different from the original container. --- roaring/roaring.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 8930981e0..f09667104 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -971,11 +971,9 @@ func (b *Bitmap) countEmptyContainers() int { // Optimize converts array and bitmap containers to run containers as necessary. func (b *Bitmap) Optimize() { - citer, _ := b.Containers.Iterator(0) - for citer.Next() { - _, c := citer.Value() - c.optimize() - } + b.Containers.UpdateEvery(func(c *Container, existed bool) (*Container, bool) { + return c.optimize(), true + }) } type errWriter struct { @@ -3519,7 +3517,7 @@ RUNLOOP: } } output := NewContainerRun(runs) - output.optimize() + output = output.optimize() return output } From 77cd21e89f36cddc47a09988fc083d33f5f00e29 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 4 Jun 2019 09:16:01 -0500 Subject: [PATCH 72/73] don't check errors we don't care about in a test --- fragment_internal_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index c159bb1a1..e45d3cce8 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -114,10 +114,10 @@ func TestFragment_RowcacheMap(t *testing.T) { ch := make(chan struct{}) for i := 0; i < f.MaxOpN; i++ { - f.setBit(0, uint64(i*32)) + _, _ = f.setBit(0, uint64(i*32)) } // force snapshot so we get a mmapped row... - f.snapshot() + _ = f.snapshot() row := f.row(0) segment := row.Segments()[0] bitmap := segment.data @@ -136,7 +136,7 @@ func TestFragment_RowcacheMap(t *testing.T) { // then invalidates the other map... for j := 0; j < 5; j++ { for i := 0; i < f.MaxOpN; i++ { - f.setBit(0, uint64(i*32+j+1)) + _, _ = f.setBit(0, uint64(i*32+j+1)) } } atomic.StoreInt64(&done, 1) From 388efd0e73d4d9c7cb15fd105ff13c88bfc717f2 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 4 Jun 2019 09:44:59 -0500 Subject: [PATCH 73/73] use os.Rename semantically correctly So it's true that Rename's arguments are called oldname/newname, and you want to rename from the previous name to the new name. And it's true that we're calling Rename on oldPath and newPath. But in our case, oldPath is the name the fragment file had before the operation, and newPath is the name of the temporary file created during the operation. Use tmpPath and frag.path to make the semantics clearer. --- view.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/view.go b/view.go index eee1ab810..bd917f407 100644 --- a/view.go +++ b/view.go @@ -437,12 +437,11 @@ func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) { } ok = true // mark as upgraded, requires reload - oldPath := frag.path - if newPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil { + if tmpPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil { return ok, errors.Wrap(err, "upgrading bsi v2") } else if err := frag.closeStorage(); err != nil { return ok, errors.Wrap(err, "closing after bsi v2 upgrade") - } else if err := os.Rename(oldPath, newPath); err != nil { + } else if err := os.Rename(tmpPath, frag.path); err != nil { return ok, errors.Wrap(err, "renaming after bsi v2 upgrade") } else if err := frag.openStorage(); err != nil { return ok, errors.Wrap(err, "re-opening after bsi v2 upgrade")