Merge pull request #1921 from jaffee/shardwidth22

add support to modify shard width at build time

.circleci/config.yml

@@ -44,6 +44,12 @@ jobs:
       - *fast-checkout
       - run: sudo apt-get install lsof
       - run: make test
+  test-golang-1.12-shard22: &base-test
+    <<: *defaults
+    steps:
+      - *fast-checkout
+      - run: sudo apt-get install lsof
+      - run: make test SHARD_WIDTH=22
   test-golang-1.12-race:
     <<: *defaults
     steps:

Makefile

@@ -6,6 +6,7 @@ VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
 BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
+SHARD_WIDTH = 20
 LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
 GO_VERSION=latest
 ENTERPRISE ?= 0
@@ -14,6 +15,7 @@ RELEASE ?= 0
 RELEASE_ENABLED = $(subst 0,,$(RELEASE))
 BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
+BUILD_TAGS += shardwidth$(SHARD_WIDTH)
 export GO111MODULE=on
 
 # Run tests and compile Pilosa
@@ -29,7 +31,7 @@ vendor: go.mod
 
 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS)
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 
 
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)

cluster_internal_test.go

@@ -157,15 +157,15 @@ func TestFragSources(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 1300000, nil)
+	_, err = field.SetBit(1, ShardWidth+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 2600000, nil)
+	_, err = field.SetBit(1, ShardWidth*2+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 3900000, nil)
+	_, err = field.SetBit(1, ShardWidth*3+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -755,7 +755,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 			t.Fatal(err)
 		}
 		tc.SetBit("i", "f", 1, 101, nil)
-		tc.SetBit("i", "f", 1, 1300000, nil)
+		tc.SetBit("i", "f", 1, ShardWidth+1, nil)
 
 		// Before starting the resize, get the CheckSum to use for
 		// comparison later.

executor.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/pql"
+	"github.com/pilosa/pilosa/shardwidth"
 	"github.com/pilosa/pilosa/tracing"
 	"github.com/pkg/errors"
 )
@@ -1234,7 +1235,7 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
 	if columnID, ok, err := c.UintArg("column"); err != nil {
 		return nil, err
 	} else if ok {
-		colShard := columnID >> shardWidthExponent
+		colShard := columnID >> shardwidth.Exponent
 		if colShard != shard {
 			return rowIDs, nil
 		}

executor_test.go

@@ -2237,11 +2237,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
 	})
 
 	t.Run("Remote SetBit", func(t *testing.T) {
-		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil {
-			t.Fatalf("quuerying remote: %v", err)
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil {
+			t.Fatalf("querying remote: %v", err)
 		}
 
-		if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) {
+		if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{pilosa.ShardWidth + 1}) {
 			t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns())
 		}
 	})
@@ -2252,11 +2252,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
 			t.Fatalf("creating field: %v", err)
 		}
 
-		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil {
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil {
 			t.Fatalf("quuerying remote: %v", err)
 		}
 
-		if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) {
+		if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{pilosa.ShardWidth + 1}) {
 			t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns())
 		}
 	})

fragment.go

@@ -40,6 +40,7 @@ import (
 	"github.com/pilosa/pilosa/logger"
 	"github.com/pilosa/pilosa/pql"
 	"github.com/pilosa/pilosa/roaring"
+	"github.com/pilosa/pilosa/shardwidth"
 	"github.com/pilosa/pilosa/stats"
 	"github.com/pilosa/pilosa/syswrap"
 	"github.com/pilosa/pilosa/tracing"
@@ -48,8 +49,9 @@ import (
 
 const (
 	// ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16.
-	shardWidthExponent = 20
-	ShardWidth         = 1 << shardWidthExponent
+	// shardWidthExponent = 20 // set in shardwidthNN.go files
+
+	ShardWidth = 1 << shardwidth.Exponent
 
 	// shardVsContainerExponent is the power of 2 of ShardWith minus the power
 	// of two of roaring container width (which is 16).
@@ -59,7 +61,7 @@ const (
 	// which a given container is in means dividing by the number of rows per
 	// container which is performantly expressed as a right shift by this
 	// exponent.
-	shardVsContainerExponent = shardWidthExponent - 16
+	shardVsContainerExponent = shardwidth.Exponent - 16
 
 	// width of roaring containers is 2^16
 	containerWidth = 1 << 16

fragment_internal_test.go

@@ -137,14 +137,14 @@ func TestFragment_SetRow(t *testing.T) {
 	rowID := uint64(1000)
 
 	// Set bits on the fragment.
-	if _, err := f.setBit(rowID, 8000001); err != nil {
+	if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil {
 		t.Fatal(err)
-	} else if _, err := f.setBit(rowID, 8065536); err != nil {
+	} else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify data on row.
-	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) {
+	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) {
 		t.Fatalf("unexpected columns: %+v", cols)
 	}
 	// Verify count on row.
@@ -153,7 +153,7 @@ func TestFragment_SetRow(t *testing.T) {
 	}
 
 	// Set row (overwrite existing data).
-	row := NewRow(8000002, 8065537, 8131074)
+	row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000)
 	if changed, err := f.unprotectedSetRow(row, rowID); err != nil {
 		t.Fatal(err)
 	} else if !changed {
@@ -161,7 +161,7 @@ func TestFragment_SetRow(t *testing.T) {
 	}
 
 	// Verify data on row.
-	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) {
+	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) {
 		t.Fatalf("unexpected columns after set row: %+v", cols)
 	}
 	// Verify count on row.
@@ -1914,7 +1914,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
 	f := mustOpenFragment("i", "f", viewStandard, 0, "")
 	defer f.Clean(b)
 	// Generate some intersecting data.
-	maxX := 1048576 / 2
+	maxX := ShardWidth / 2
 	sz := maxX
 	rows := make([]uint64, sz)
 	cols := make([]uint64, sz)
@@ -1950,7 +1950,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
 
 func BenchmarkFragment_Import(b *testing.B) {
 	b.StopTimer()
-	maxX := 1048576 * 5 * 2
+	maxX := ShardWidth * 5 * 2
 	sz := maxX
 	rows := make([]uint64, sz)
 	cols := make([]uint64, sz)

roaring/roaring_test.go

@@ -299,10 +299,10 @@ func TestBitmap_Max(t *testing.T) {
 
 // Ensure CountRange is correct even if rangekey is prior to initial container.
 func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
-	s := uint64(2009 * 1048576)
-	e := uint64(2010 * 1048576)
+	s := uint64(2009 * pilosa.ShardWidth)
+	e := uint64(2010 * pilosa.ShardWidth)
 
-	start := s + (39314024 % 1048576)
+	start := s + (39314024 % pilosa.ShardWidth)
 	bm0 := roaring.NewFileBitmap()
 	for i := uint64(0); i < 65536; i++ {
 		if (i+1)%4096 == 0 {

server/cluster_test.go

@@ -85,10 +85,10 @@ func TestMain_SendReceiveMessage(t *testing.T) {
 	}
 
 	// Write data on first node.
-	if _, err := m0.Query("i", "", `
+	if _, err := m0.Query("i", "", fmt.Sprintf(`
             Set(1, f=1)
-            Set(2400000, f=1)
-        `); err != nil {
+            Set(%d, f=1)
+        `, 2*pilosa.ShardWidth+1)); err != nil {
 		t.Fatal(err)
 	}
 

server/handler_test.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	gohttp "net/http"
@@ -109,8 +110,8 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 		body := w.Body.String()
-		target := `{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}
-`
+		target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}
+`, pilosa.ShardWidth)
 		if body != target {
 			t.Fatalf("%s != %s", target, body)
 		}
@@ -298,7 +299,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")))
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
-		} else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" {
+		} else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" {
 			t.Fatalf("unexpected body: %s", body)
 		}
 	})
@@ -315,10 +316,11 @@ func TestHandler_Endpoints(t *testing.T) {
 	t.Run("ColumnAttrs_JSON", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)")))
+		exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n"
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String())
-		} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" {
-			t.Fatalf("unexpected body: %s", body)
+		} else if body := w.Body.String(); body != exp {
+			t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp)
 		}
 	})
 

shardwidth/16.go

@@ -0,0 +1,5 @@
+// +build shardwidth16
+
+package shardwidth
+
+const Exponent = 16

shardwidth/17.go

@@ -0,0 +1,5 @@
+// +build shardwidth17
+
+package shardwidth
+
+const Exponent = 17

shardwidth/18.go

@@ -0,0 +1,5 @@
+// +build shardwidth18
+
+package shardwidth
+
+const Exponent = 18

shardwidth/19.go

@@ -0,0 +1,5 @@
+// +build shardwidth19
+
+package shardwidth
+
+const Exponent = 19

shardwidth/20.go

@@ -0,0 +1,5 @@
+// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32
+
+package shardwidth
+
+const Exponent = 20

shardwidth/21.go

@@ -0,0 +1,5 @@
+// +build shardwidth21
+
+package shardwidth
+
+const Exponent = 21

shardwidth/22.go

@@ -0,0 +1,5 @@
+// +build shardwidth22
+
+package shardwidth
+
+const Exponent = 22

shardwidth/23.go

@@ -0,0 +1,5 @@
+// +build shardwidth23
+
+package shardwidth
+
+const Exponent = 23

shardwidth/24.go

@@ -0,0 +1,5 @@
+// +build shardwidth24
+
+package shardwidth
+
+const Exponent = 24

shardwidth/25.go

@@ -0,0 +1,5 @@
+// +build shardwidth25
+
+package shardwidth
+
+const Exponent = 25

shardwidth/26.go

@@ -0,0 +1,5 @@
+// +build shardwidth26
+
+package shardwidth
+
+const Exponent = 26

shardwidth/27.go

@@ -0,0 +1,5 @@
+// +build shardwidth27
+
+package shardwidth
+
+const Exponent = 27

shardwidth/28.go

@@ -0,0 +1,5 @@
+// +build shardwidth28
+
+package shardwidth
+
+const Exponent = 28

shardwidth/29.go

@@ -0,0 +1,5 @@
+// +build shardwidth29
+
+package shardwidth
+
+const Exponent = 29

shardwidth/30.go

@@ -0,0 +1,5 @@
+// +build shardwidth30
+
+package shardwidth
+
+const Exponent = 30

shardwidth/31.go

@@ -0,0 +1,5 @@
+// +build shardwidth31
+
+package shardwidth
+
+const Exponent = 31

shardwidth/32.go

@@ -0,0 +1,5 @@
+// +build shardwidth32
+
+package shardwidth
+
+const Exponent = 32