diff --git a/.circleci/config.yml b/.circleci/config.yml index c696d5691..aa6da2fda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,12 @@ jobs: - *fast-checkout - run: sudo apt-get install lsof - run: make test + test-golang-1.12-shard22: &base-test + <<: *defaults + steps: + - *fast-checkout + - run: sudo apt-get install lsof + - run: make test SHARD_WIDTH=22 test-golang-1.12-race: <<: *defaults steps: diff --git a/Makefile b/Makefile index 45b1df78c..456456a7b 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) +SHARD_WIDTH = 20 LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(if $(ENTERPRISE_ENABLED),1)" GO_VERSION=latest ENTERPRISE ?= 0 @@ -14,6 +15,7 @@ RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) +BUILD_TAGS += shardwidth$(SHARD_WIDTH) export GO111MODULE=on # Run tests and compile Pilosa @@ -29,7 +31,7 @@ vendor: go.mod # Run test suite test: - go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) + go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) bench: go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) diff --git a/cluster_internal_test.go b/cluster_internal_test.go index a6cf437b2..fa40c48b7 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -157,15 +157,15 @@ func TestFragSources(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 1300000, nil) + _, err = field.SetBit(1, ShardWidth+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 2600000, nil) + _, err = field.SetBit(1, ShardWidth*2+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 3900000, nil) + _, err = field.SetBit(1, ShardWidth*3+1, nil) if err != nil { t.Fatal(err) } @@ -755,7 +755,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Fatal(err) } tc.SetBit("i", "f", 1, 101, nil) - tc.SetBit("i", "f", 1, 1300000, nil) + tc.SetBit("i", "f", 1, ShardWidth+1, nil) // Before starting the resize, get the CheckSum to use for // comparison later. diff --git a/executor_test.go b/executor_test.go index a36ddcf72..2dc96223e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2226,11 +2226,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Remote SetBit", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil { - t.Fatalf("quuerying remote: %v", err) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil { + t.Fatalf("querying remote: %v", err) } - if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) { + if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{pilosa.ShardWidth + 1}) { t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns()) } }) @@ -2241,11 +2241,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil { t.Fatalf("quuerying remote: %v", err) } - if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) { + if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{pilosa.ShardWidth + 1}) { t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns()) } }) diff --git a/fragment.go b/fragment.go index 4769ce700..1fdce2ee5 100644 --- a/fragment.go +++ b/fragment.go @@ -48,8 +48,9 @@ import ( const ( // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. - shardWidthExponent = 20 - ShardWidth = 1 << shardWidthExponent + // shardWidthExponent = 20 // set in shardwidthNN.go files + + ShardWidth = 1 << shardWidthExponent // shardVsContainerExponent is the power of 2 of ShardWith minus the power // of two of roaring container width (which is 16). diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 5186d6da1..3930e98e2 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -137,14 +137,14 @@ func TestFragment_SetRow(t *testing.T) { rowID := uint64(1000) // Set bits on the fragment. - if _, err := f.setBit(rowID, 8000001); err != nil { + if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(rowID, 8065536); err != nil { + } else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil { t.Fatal(err) } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) { + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) { t.Fatalf("unexpected columns: %+v", cols) } // Verify count on row. @@ -153,7 +153,7 @@ func TestFragment_SetRow(t *testing.T) { } // Set row (overwrite existing data). - row := NewRow(8000002, 8065537, 8131074) + row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000) if changed, err := f.unprotectedSetRow(row, rowID); err != nil { t.Fatal(err) } else if !changed { @@ -161,7 +161,7 @@ func TestFragment_SetRow(t *testing.T) { } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) { + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) { t.Fatalf("unexpected columns after set row: %+v", cols) } // Verify count on row. @@ -1914,7 +1914,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(b) // Generate some intersecting data. - maxX := 1048576 / 2 + maxX := ShardWidth / 2 sz := maxX rows := make([]uint64, sz) cols := make([]uint64, sz) @@ -1950,7 +1950,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { b.StopTimer() - maxX := 1048576 * 5 * 2 + maxX := ShardWidth * 5 * 2 sz := maxX rows := make([]uint64, sz) cols := make([]uint64, sz) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 9d0b3c8f8..ebf88a212 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -299,10 +299,10 @@ func TestBitmap_Max(t *testing.T) { // Ensure CountRange is correct even if rangekey is prior to initial container. func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { - s := uint64(2009 * 1048576) - e := uint64(2010 * 1048576) + s := uint64(2009 * pilosa.ShardWidth) + e := uint64(2010 * pilosa.ShardWidth) - start := s + (39314024 % 1048576) + start := s + (39314024 % pilosa.ShardWidth) bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i++ { if (i+1)%4096 == 0 { diff --git a/server/cluster_test.go b/server/cluster_test.go index 617bf6cad..b284e49df 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -85,10 +85,10 @@ func TestMain_SendReceiveMessage(t *testing.T) { } // Write data on first node. - if _, err := m0.Query("i", "", ` + if _, err := m0.Query("i", "", fmt.Sprintf(` Set(1, f=1) - Set(2400000, f=1) - `); err != nil { + Set(%d, f=1) + `, 2*pilosa.ShardWidth+1)); err != nil { t.Fatal(err) } diff --git a/server/handler_test.go b/server/handler_test.go index 701d019ae..2a6b9b8c2 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "io" "io/ioutil" gohttp "net/http" @@ -109,8 +110,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - target := `{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]} -` + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} +`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } @@ -294,7 +295,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" { + } else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" { t.Fatalf("unexpected body: %s", body) } }) @@ -311,10 +312,11 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ColumnAttrs_JSON", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)"))) + exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n" if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) + } else if body := w.Body.String(); body != exp { + t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp) } }) diff --git a/shardwidth20.go b/shardwidth20.go new file mode 100644 index 000000000..fd99f0a84 --- /dev/null +++ b/shardwidth20.go @@ -0,0 +1,5 @@ +// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32 + +package pilosa + +const shardWidthExponent = 20 diff --git a/shardwidth22.go b/shardwidth22.go new file mode 100644 index 000000000..57727a9a5 --- /dev/null +++ b/shardwidth22.go @@ -0,0 +1,5 @@ +// +build shardwidth22 + +package pilosa + +const shardWidthExponent = 22